Build: #3112 was successful Child of CDAP-DRC-5146
Code commits
caskdata/cdap
-
Terence Yim <885032+chtyim@users.noreply.github.com> 57e5469d89a279aebeacb59b4c1caab3a5f6b0bc
Merge pull request #12879 from cdapio/feature/CDAP-17439-hadoop-3
(CDAP-17439) Added runtime support for Hadoop + Spark 3 -
Terence Yim 4570295c99bce9aab75fb2d1929193c7f777140c m
(CDAP-17439) Added runtime support for Hadoop + Spark 3
This change added support for Hadoop 3 and Spark 3, yet maintaining compatibility with
Hadoop 2 and Spark 1 and Spark 2.
Hadoop YARN
* Don’t use YARN localization to expand program jar file due to YARN bug YARN-9591
** In twill containers, expand the jar locally
** In Spark, renames the jar to .zip before localization
Guava depdendency
* Add missing methods to the Guava library
** Various missing Preconditions.checkArgument() methods
** Various missing Preconditions.checkState() methods
** MoreExecutors.directExecutor() method
Hadoop 3
* Upgrade to Avro 1.8.2
** SpecificData constructor becomes public
* Remove usage of ConcurrentHashSet from the jetty library as Hadoop 3 uses an incompatible version, and CDAP shouldn’t be using that library outside of the cdap-security module
Spark 3
* Use Reflection to alter SparkListenerApplicationStart event
** Spark 2 and 3 are binary incompatible for that class
Also, there is some small refactoring to modernize the code to use the standard Java library instead of Guava- cdap-app-fabric/src/main/java/io/cdap/cdap/app/program/ManifestFields.java (version 4570295c99bce9aab75fb2d1929193c7f777140c)
- cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/ProgramClassLoader.java (version 4570295c99bce9aab75fb2d1929193c7f777140c)
- cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/ProgramOptionConstants.java (version 4570295c99bce9aab75fb2d1929193c7f777140c)
- cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/distributed/AbstractProgramTwillRunnable.java (version 4570295c99bce9aab75fb2d1929193c7f777140c)
- cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/distributed/DistributedProgramRunner.java (version 4570295c99bce9aab75fb2d1929193c7f777140c)
- cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/plugin/PluginClassLoader.java (version 4570295c99bce9aab75fb2d1929193c7f777140c)
- cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/service/http/HttpHandlerGenerator.java (version 4570295c99bce9aab75fb2d1929193c7f777140c)
- cdap-common/src/main/java/io/cdap/cdap/common/app/MainClassLoader.java (version 4570295c99bce9aab75fb2d1929193c7f777140c)
- cdap-common/src/main/java/io/cdap/cdap/common/lang/ClassLoaders.java (version 4570295c99bce9aab75fb2d1929193c7f777140c)
- cdap-common/src/main/java/io/cdap/cdap/common/lang/GuavaClassRewriter.java (version 4570295c99bce9aab75fb2d1929193c7f777140c)
- cdap-common/src/main/java/io/cdap/cdap/common/lang/PackageFilterClassLoader.java (version 4570295c99bce9aab75fb2d1929193c7f777140c)
- cdap-common/src/main/java/io/cdap/cdap/internal/asm/Classes.java (version 4570295c99bce9aab75fb2d1929193c7f777140c)
- cdap-common/src/main/java/io/cdap/cdap/internal/asm/Methods.java (version 4570295c99bce9aab75fb2d1929193c7f777140c)
- cdap-common/src/test/java/io/cdap/cdap/common/lang/ClassLoaderTest.java (version 4570295c99bce9aab75fb2d1929193c7f777140c)
- cdap-gateway/src/main/java/io/cdap/cdap/gateway/router/RouterAuditLookUp.java (version 4570295c99bce9aab75fb2d1929193c7f777140c)
- cdap-runtime-ext-dataproc/src/main/java/io/cdap/cdap/runtime/spi/common/DataprocUtils.java (version 4570295c99bce9aab75fb2d1929193c7f777140c)
- cdap-runtime-ext-dataproc/src/main/java/io/cdap/cdap/runtime/spi/runtimejob/DataprocJobMain.java (version 4570295c99bce9aab75fb2d1929193c7f777140c)
- cdap-runtime-ext-dataproc/src/main/java/io/cdap/cdap/runtime/spi/runtimejob/DataprocRuntimeEnvironment.java (version 4570295c99bce9aab75fb2d1929193c7f777140c)
- cdap-spark-core-base/src/main/java/io/cdap/cdap/app/runtime/spark/SparkRuntimeContextProvider.java (version 4570295c99bce9aab75fb2d1929193c7f777140c)
- cdap-spark-core-base/src/main/java/io/cdap/cdap/app/runtime/spark/SparkRuntimeService.java (version 4570295c99bce9aab75fb2d1929193c7f777140c)
- cdap-spark-core-base/src/main/java/io/cdap/cdap/app/runtime/spark/SparkRuntimeUtils.java (version 4570295c99bce9aab75fb2d1929193c7f777140c)
- cdap-spark-core-base/src/main/java/io/cdap/cdap/app/runtime/spark/distributed/SparkContainerLauncher.java (version 4570295c99bce9aab75fb2d1929193c7f777140c)
- cdap-spark-core-base/src/main/scala/io/cdap/cdap/app/runtime/spark/SparkRuntimeEnv.scala (version 4570295c99bce9aab75fb2d1929193c7f777140c)
- cdap-watchdog/src/main/java/io/cdap/cdap/logging/appender/LogAppenderInitializer.java (version 4570295c99bce9aab75fb2d1929193c7f777140c)
- pom.xml (version 4570295c99bce9aab75fb2d1929193c7f777140c)