Code commits

Wrangler Transform

  • Terence Yim <885032+chtyim@users.noreply.github.com>

    Terence Yim <885032+chtyim@users.noreply.github.com> 5b3dddbf29cecbd9b747e39230228a7babf5e84d

    Merge pull request #536 from data-integrations/feature/remote-directive-perf
    Remote directives execution optimization

  • Terence Yim

    Terence Yim 7077d7213520a07c41f7909bfd973eaee225f1ac m

    - Remote task execution optimization   - Fix UDD Classloader creation to be per artifact, not per plugin   - Identify system directives on the service side   - Avoid using reflections to find all system directives in the task   - Avoid loading all UDD artifacts in the task   - Use the new CDAP API to keep task around if no UDD was executed   - No need to execute remote task if there is no directives   - Avoid task restart if no UDD nor JEXL expressions in the directives - Code cleanup and bug fixes   - Fixed the missing #pragma bug in the V2 API   - Bunch of code cleanup based on suggestions from IDE

    • wrangler-api/src/main/java/io/cdap/wrangler/api/DirectiveConfig.java (version 7077d7213520a07c41f7909bfd973eaee225f1ac)
    • wrangler-api/src/main/java/io/cdap/wrangler/api/DirectiveParseException.java (version 7077d7213520a07c41f7909bfd973eaee225f1ac)
    • wrangler-api/src/main/java/io/cdap/wrangler/api/RecipeParser.java (version 7077d7213520a07c41f7909bfd973eaee225f1ac)
    • wrangler-api/src/main/java/io/cdap/wrangler/api/RecipePipeline.java (version 7077d7213520a07c41f7909bfd973eaee225f1ac)
    • wrangler-api/src/main/java/io/cdap/wrangler/api/RecipeSymbol.java (version 7077d7213520a07c41f7909bfd973eaee225f1ac)
    • wrangler-api/src/main/java/io/cdap/wrangler/api/Row.java (version 7077d7213520a07c41f7909bfd973eaee225f1ac)
    • wrangler-api/src/main/java/io/cdap/wrangler/api/parser/UsageDefinition.java (version 7077d7213520a07c41f7909bfd973eaee225f1ac)
    • wrangler-core/src/main/java/io/cdap/directives/aggregates/DefaultTransientStore.java (version 7077d7213520a07c41f7909bfd973eaee225f1ac)
    • wrangler-core/src/main/java/io/cdap/directives/aggregates/IncrementTransientVariable.java (version 7077d7213520a07c41f7909bfd973eaee225f1ac)
    • wrangler-core/src/main/java/io/cdap/directives/aggregates/SetTransientVariable.java (version 7077d7213520a07c41f7909bfd973eaee225f1ac)
    • wrangler-core/src/main/java/io/cdap/directives/row/Fail.java (version 7077d7213520a07c41f7909bfd973eaee225f1ac)
    • wrangler-core/src/main/java/io/cdap/directives/row/RecordConditionFilter.java (version 7077d7213520a07c41f7909bfd973eaee225f1ac)
    • wrangler-core/src/main/java/io/cdap/directives/row/SendToError.java (version 7077d7213520a07c41f7909bfd973eaee225f1ac)
    • wrangler-core/src/main/java/io/cdap/directives/row/SendToErrorAndContinue.java (version 7077d7213520a07c41f7909bfd973eaee225f1ac)
    • wrangler-core/src/main/java/io/cdap/directives/transformation/ColumnExpression.java (version 7077d7213520a07c41f7909bfd973eaee225f1ac)
    • wrangler-core/src/main/java/io/cdap/wrangler/executor/RecipePipelineExecutor.java (version 7077d7213520a07c41f7909bfd973eaee225f1ac)
    • wrangler-core/src/main/java/io/cdap/wrangler/expression/EL.java (version 7077d7213520a07c41f7909bfd973eaee225f1ac)
    • wrangler-core/src/main/java/io/cdap/wrangler/parser/ConfigDirectiveContext.java (version 7077d7213520a07c41f7909bfd973eaee225f1ac)
    • wrangler-core/src/main/java/io/cdap/wrangler/parser/DirectiveClass.java (version 7077d7213520a07c41f7909bfd973eaee225f1ac)
    • wrangler-core/src/main/java/io/cdap/wrangler/parser/GrammarBasedParser.java (version 7077d7213520a07c41f7909bfd973eaee225f1ac)
    • wrangler-core/src/main/java/io/cdap/wrangler/parser/GrammarWalker.java (version 7077d7213520a07c41f7909bfd973eaee225f1ac)
    • wrangler-core/src/main/java/io/cdap/wrangler/parser/RecipeCompiler.java (version 7077d7213520a07c41f7909bfd973eaee225f1ac)
    • wrangler-core/src/main/java/io/cdap/wrangler/parser/SimpleTextParser.java (version 7077d7213520a07c41f7909bfd973eaee225f1ac)
    • wrangler-core/src/main/java/io/cdap/wrangler/registry/CompositeDirectiveRegistry.java (version 7077d7213520a07c41f7909bfd973eaee225f1ac)
    • wrangler-core/src/main/java/io/cdap/wrangler/registry/DirectiveInfo.java (version 7077d7213520a07c41f7909bfd973eaee225f1ac)
    • wrangler-core/src/main/java/io/cdap/wrangler/registry/DirectiveScope.java (version 7077d7213520a07c41f7909bfd973eaee225f1ac)
    • wrangler-core/src/main/java/io/cdap/wrangler/registry/SystemDirectiveRegistry.java (version 7077d7213520a07c41f7909bfd973eaee225f1ac)
    • wrangler-core/src/main/java/io/cdap/wrangler/registry/UserDirectiveRegistry.java (version 7077d7213520a07c41f7909bfd973eaee225f1ac)
    • wrangler-core/src/main/java/io/cdap/wrangler/utils/ArtifactSummaryComparator.java (version 7077d7213520a07c41f7909bfd973eaee225f1ac)
    • wrangler-core/src/main/java/io/cdap/wrangler/utils/ObjectSerDe.java (version 7077d7213520a07c41f7909bfd973eaee225f1ac)
    • wrangler-core/src/test/java/io/cdap/wrangler/TestingRig.java (version 7077d7213520a07c41f7909bfd973eaee225f1ac)
    • wrangler-core/src/test/java/io/cdap/wrangler/config/DirectiveConfigTest.java (version 7077d7213520a07c41f7909bfd973eaee225f1ac)
    • wrangler-core/src/test/java/io/cdap/wrangler/expression/ELTest.java (version 7077d7213520a07c41f7909bfd973eaee225f1ac)
    • wrangler-core/src/test/java/io/cdap/wrangler/lineage/LineageOperationsTest.java (version 7077d7213520a07c41f7909bfd973eaee225f1ac)
    • wrangler-core/src/test/java/io/cdap/wrangler/parser/ConfigDirectiveContextTest.java (version 7077d7213520a07c41f7909bfd973eaee225f1ac)
    • wrangler-core/src/test/java/io/cdap/wrangler/parser/GrammarBasedParserTest.java (version 7077d7213520a07c41f7909bfd973eaee225f1ac)
    • wrangler-core/src/test/java/io/cdap/wrangler/registry/CompositeDirectiveRegistryTest.java (version 7077d7213520a07c41f7909bfd973eaee225f1ac)
    • wrangler-proto/src/main/java/io/cdap/wrangler/proto/workspace/v2/Workspace.java (version 7077d7213520a07c41f7909bfd973eaee225f1ac)
    • wrangler-service/src/main/java/io/cdap/wrangler/service/common/AbstractWranglerHandler.java (version 7077d7213520a07c41f7909bfd973eaee225f1ac)
    • wrangler-service/src/main/java/io/cdap/wrangler/service/directive/AbstractDirectiveHandler.java (version 7077d7213520a07c41f7909bfd973eaee225f1ac)
    • wrangler-service/src/main/java/io/cdap/wrangler/service/directive/CommonDirectiveExecutor.java (version 7077d7213520a07c41f7909bfd973eaee225f1ac)
    • wrangler-service/src/main/java/io/cdap/wrangler/service/directive/DirectivesHandler.java (version 7077d7213520a07c41f7909bfd973eaee225f1ac)
    • wrangler-service/src/main/java/io/cdap/wrangler/service/directive/RemoteDirectiveRequest.java (version 7077d7213520a07c41f7909bfd973eaee225f1ac)
    • wrangler-service/src/main/java/io/cdap/wrangler/service/directive/RemoteExecutionTask.java (version 7077d7213520a07c41f7909bfd973eaee225f1ac)
    • wrangler-service/src/main/java/io/cdap/wrangler/service/directive/UserDirectivesCollector.java (version 7077d7213520a07c41f7909bfd973eaee225f1ac)
    • wrangler-service/src/main/java/io/cdap/wrangler/service/directive/WorkspaceHandler.java (version 7077d7213520a07c41f7909bfd973eaee225f1ac)
    • wrangler-test/src/main/java/io/cdap/wrangler/test/TestingRig.java (version 7077d7213520a07c41f7909bfd973eaee225f1ac)
    • wrangler-transform/src/main/java/io/cdap/wrangler/Wrangler.java (version 7077d7213520a07c41f7909bfd973eaee225f1ac)
    • wrangler-transform/src/main/java/io/cdap/wrangler/WranglerPipelineContext.java (version 7077d7213520a07c41f7909bfd973eaee225f1ac)