Build #1,387

Build: #1387 was successful Child of CDAP-DRC-5385

Code commits

caskdata/cdap

  • Fernando Velasquez

    Fernando Velasquez 7c604765f33445e9e07954692ed87a02cbdd0cb0

    Merge pull request #13343 from cdapio/CDAP-17864-implement-full-sql-engine-integration
    Tie SQL Engine integration into Spark Pipeline Runner and RDD Collection.

  • Fernando Velasquez

    Fernando Velasquez 91e0dd9ddd4f43bfafd14c504873a81dbc823a73 m

    Tie SQL Engine integration into Spark Pipeline Runner and RDD Collection.
    Added support for stage metrics for SQL engine jobs.

    Added unit test for Joins using the SQL engine.

    Implemented new SQLEngineCollection class to represent records residing in the SQL engine. This class will pull and delegate to the RDDCOllection as needed.

    Added note detailing that SQL join operations can only be performed after all involved datasets have been pushed to the SQL engine beforehand.

    Added test for logic that decides wether we should push to the SQL engine or not.

    • cdap-app-templates/cdap-etl/cdap-data-pipeline-base/src/test/java/io/cdap/cdap/datapipeline/AutoJoinerTest.java (version 91e0dd9ddd4f43bfafd14c504873a81dbc823a73)
    • cdap-app-templates/cdap-etl/cdap-etl-api/src/main/java/io/cdap/cdap/etl/api/engine/sql/BatchSQLEngine.java (version 91e0dd9ddd4f43bfafd14c504873a81dbc823a73)
    • cdap-app-templates/cdap-etl/cdap-etl-api/src/main/java/io/cdap/cdap/etl/api/engine/sql/SQLEngine.java (version 91e0dd9ddd4f43bfafd14c504873a81dbc823a73)
    • cdap-app-templates/cdap-etl/cdap-etl-api/src/main/java/io/cdap/cdap/etl/api/engine/sql/SQLEngineException.java (version 91e0dd9ddd4f43bfafd14c504873a81dbc823a73)
    • cdap-app-templates/cdap-etl/cdap-etl-api/src/main/java/io/cdap/cdap/etl/api/engine/sql/SQLOperationResult.java (version 91e0dd9ddd4f43bfafd14c504873a81dbc823a73)
    • cdap-app-templates/cdap-etl/cdap-etl-api/src/main/java/io/cdap/cdap/etl/api/engine/sql/SQLOperationStatus.java (version 91e0dd9ddd4f43bfafd14c504873a81dbc823a73)
    • cdap-app-templates/cdap-etl/cdap-etl-api/src/main/java/io/cdap/cdap/etl/api/engine/sql/dataset/SQLDataset.java (version 91e0dd9ddd4f43bfafd14c504873a81dbc823a73)
    • cdap-app-templates/cdap-etl/cdap-etl-api/src/main/java/io/cdap/cdap/etl/api/engine/sql/dataset/SQLPullDataset.java (version 91e0dd9ddd4f43bfafd14c504873a81dbc823a73)
    • cdap-app-templates/cdap-etl/cdap-etl-api/src/main/java/io/cdap/cdap/etl/api/engine/sql/dataset/SQLPushDataset.java (version 91e0dd9ddd4f43bfafd14c504873a81dbc823a73)
    • cdap-app-templates/cdap-etl/cdap-etl-api/src/main/java/io/cdap/cdap/etl/api/engine/sql/request/SQLJoinRequest.java (version 91e0dd9ddd4f43bfafd14c504873a81dbc823a73)
    • cdap-app-templates/cdap-etl/cdap-etl-api/src/main/java/io/cdap/cdap/etl/api/engine/sql/request/SQLPullRequest.java (version 91e0dd9ddd4f43bfafd14c504873a81dbc823a73)
    • cdap-app-templates/cdap-etl/cdap-etl-api/src/main/java/io/cdap/cdap/etl/api/engine/sql/request/SQLPushRequest.java (version 91e0dd9ddd4f43bfafd14c504873a81dbc823a73)
    • cdap-app-templates/cdap-etl/cdap-etl-batch/pom.xml (version 91e0dd9ddd4f43bfafd14c504873a81dbc823a73)
    • cdap-app-templates/cdap-etl/cdap-etl-batch/src/test/java/io/cdap/cdap/etl/spark/batch/BatchSparkPipelineDriverTest.java (version 91e0dd9ddd4f43bfafd14c504873a81dbc823a73)
    • cdap-app-templates/cdap-etl/cdap-etl-core/src/main/java/io/cdap/cdap/etl/batch/PipelinePluginInstantiator.java (version 91e0dd9ddd4f43bfafd14c504873a81dbc823a73)
    • cdap-app-templates/cdap-etl/cdap-etl-core/src/main/java/io/cdap/cdap/etl/batch/mapreduce/MapReduceStageStatisticsCollector.java (version 91e0dd9ddd4f43bfafd14c504873a81dbc823a73)
    • cdap-app-templates/cdap-etl/cdap-etl-core/src/main/java/io/cdap/cdap/etl/common/NoopStageStatisticsCollector.java (version 91e0dd9ddd4f43bfafd14c504873a81dbc823a73)
    • cdap-app-templates/cdap-etl/cdap-etl-core/src/main/java/io/cdap/cdap/etl/common/StageStatisticsCollector.java (version 91e0dd9ddd4f43bfafd14c504873a81dbc823a73)
    • cdap-app-templates/cdap-etl/cdap-etl-core/src/main/java/io/cdap/cdap/etl/engine/SQLEngineJob.java (version 91e0dd9ddd4f43bfafd14c504873a81dbc823a73)
    • cdap-app-templates/cdap-etl/cdap-etl-core/src/main/java/io/cdap/cdap/etl/engine/SQLEngineJobKey.java (version 91e0dd9ddd4f43bfafd14c504873a81dbc823a73)
    • cdap-app-templates/cdap-etl/cdap-etl-core/src/main/java/io/cdap/cdap/etl/engine/SQLEngineJobStatus.java (version 91e0dd9ddd4f43bfafd14c504873a81dbc823a73)
    • cdap-app-templates/cdap-etl/cdap-etl-core/src/main/java/io/cdap/cdap/etl/engine/SQLEngineJobType.java (version 91e0dd9ddd4f43bfafd14c504873a81dbc823a73)
    • cdap-app-templates/cdap-etl/cdap-etl-core/src/test/java/io/cdap/cdap/etl/spec/PipelineSpecGeneratorTest.java (version 91e0dd9ddd4f43bfafd14c504873a81dbc823a73)
    • cdap-app-templates/cdap-etl/hydrator-spark-core-base/src/main/java/io/cdap/cdap/etl/spark/SparkPipelineRunner.java (version 91e0dd9ddd4f43bfafd14c504873a81dbc823a73)
    • cdap-app-templates/cdap-etl/hydrator-spark-core-base/src/main/java/io/cdap/cdap/etl/spark/SparkStageStatisticsCollector.java (version 91e0dd9ddd4f43bfafd14c504873a81dbc823a73)
    • cdap-app-templates/cdap-etl/hydrator-spark-core-base/src/main/java/io/cdap/cdap/etl/spark/batch/BatchSQLEngineAdapter.java (version 91e0dd9ddd4f43bfafd14c504873a81dbc823a73)
    • cdap-app-templates/cdap-etl/hydrator-spark-core-base/src/main/java/io/cdap/cdap/etl/spark/batch/BatchSparkPipelineDriver.java (version 91e0dd9ddd4f43bfafd14c504873a81dbc823a73)
    • cdap-app-templates/cdap-etl/hydrator-spark-core-base/src/main/java/io/cdap/cdap/etl/spark/batch/RDDCollection.java (version 91e0dd9ddd4f43bfafd14c504873a81dbc823a73)
    • cdap-app-templates/cdap-etl/hydrator-spark-core-base/src/main/java/io/cdap/cdap/etl/spark/batch/SQLEngineCollection.java (version 91e0dd9ddd4f43bfafd14c504873a81dbc823a73)
    • cdap-app-templates/cdap-etl/hydrator-spark-core-base/src/main/java/io/cdap/cdap/etl/spark/join/JoinExpressionRequest.java (version 91e0dd9ddd4f43bfafd14c504873a81dbc823a73)
    • cdap-app-templates/cdap-etl/hydrator-spark-core-base/src/main/java/io/cdap/cdap/etl/spark/join/JoinRequest.java (version 91e0dd9ddd4f43bfafd14c504873a81dbc823a73)
    • cdap-app-templates/cdap-etl/hydrator-test/src/main/java/io/cdap/cdap/etl/mock/batch/MockPullDataset.java (version 91e0dd9ddd4f43bfafd14c504873a81dbc823a73)
    • cdap-app-templates/cdap-etl/hydrator-test/src/main/java/io/cdap/cdap/etl/mock/batch/MockPushDataset.java (version 91e0dd9ddd4f43bfafd14c504873a81dbc823a73)
    • cdap-app-templates/cdap-etl/hydrator-test/src/main/java/io/cdap/cdap/etl/mock/batch/MockSQLEngine.java (version 91e0dd9ddd4f43bfafd14c504873a81dbc823a73)
    • cdap-app-templates/cdap-etl/hydrator-test/src/main/java/io/cdap/cdap/etl/mock/batch/SerializableTransform.java (version 91e0dd9ddd4f43bfafd14c504873a81dbc823a73)
    • cdap-app-templates/cdap-etl/hydrator-test/src/main/java/io/cdap/cdap/etl/mock/test/HydratorTestBase.java (version 91e0dd9ddd4f43bfafd14c504873a81dbc823a73)