Build #644

Build: #644 failed Manual run by Derek Wood

Stages & jobs

  1. Variables

  2. Spinup and Test

Logs

The following logs have been generated by the jobs in this plan.
  • Expand all
  • Collapse all
Job Logs
Collapse Spark 2.2 Release 2 Spinup and Test Download or View
20-Mar-2020 18:45:05 2020-03-20 18:45:05 +0000 DEBUG: Response: {"id"=>195, "name"=>"First Run", "startTime"=>"2020-03-20T18:44:34.738Z", "endTime"=>"2020-03-20T18:45:04.901Z", "active"=>false, "success"=>true, "resultMessage"=>"Finished First Run of the following services successfully: hbase-00053556.", "children"=>{"items"=>[{"id"=>203, "name"=>"GlobalWait", "startTime"=>"2020-03-20T18:44:37.206Z", "endTime"=>"2020-03-20T18:44:37.226Z", "active"=>false, "success"=>true, "resultMessage"=>"Finished waiting"}, {"id"=>196, "name"=>"DeployClusterClientConfig", "startTime"=>"2020-03-20T18:44:34.912Z", "endTime"=>"2020-03-20T18:44:37.205Z", "active"=>false, "success"=>true, "resultMessage"=>"Successfully deployed all client configurations.", "clusterRef"=>{"clusterName"=>"00053556"}}, {"id"=>206, "name"=>"Start", "startTime"=>"2020-03-20T18:44:42.309Z", "endTime"=>"2020-03-20T18:45:04.901Z", "active"=>false, "success"=>true, "resultMessage"=>"Successfully started service.", "serviceRef"=>{"clusterName"=>"00053556", "serviceName"=>"hbase-00053556"}}, {"id"=>204, "name"=>"CreateRootDir", "startTime"=>"2020-03-20T18:44:37.246Z", "endTime"=>"2020-03-20T18:44:42.274Z", "active"=>false, "success"=>true, "resultMessage"=>"Successfully created HDFS directory.", "serviceRef"=>{"clusterName"=>"00053556", "serviceName"=>"hbase-00053556"}}]}, "canRetry"=>false}
20-Mar-2020 18:45:05 2020-03-20 18:45:05 +0000 INFO: ------------ Start HIVE FirstRun --------------
20-Mar-2020 18:45:05 2020-03-20 18:45:05 +0000 DEBUG: Request: clusters/00053556/services/hive-00053556/commands/firstRun
20-Mar-2020 18:45:05 2020-03-20 18:45:05 +0000 DEBUG: Response: {"id"=>217, "name"=>"First Run", "startTime"=>"2020-03-20T18:45:05.428Z", "active"=>true, "children"=>{"items"=>[]}}
20-Mar-2020 18:45:05 2020-03-20 18:45:05 +0000 INFO: ------------ Wait for HIVE FirstRun --------------
20-Mar-2020 18:45:05 2020-03-20 18:45:05 +0000 INFO: Waiting for command 217 to complete
20-Mar-2020 18:45:52 .......................
20-Mar-2020 18:45:52 2020-03-20 18:45:52 +0000 DEBUG: Response: {"id"=>217, "name"=>"First Run", "startTime"=>"2020-03-20T18:45:05.428Z", "endTime"=>"2020-03-20T18:45:52.189Z", "active"=>false, "success"=>true, "resultMessage"=>"Finished First Run of the following services successfully: hive-00053556.", "children"=>{"items"=>[{"id"=>225, "name"=>"GlobalWait", "startTime"=>"2020-03-20T18:45:07.693Z", "endTime"=>"2020-03-20T18:45:07.710Z", "active"=>false, "success"=>true, "resultMessage"=>"Finished waiting"}, {"id"=>230, "name"=>"CreateHiveUserDir", "startTime"=>"2020-03-20T18:45:20.168Z", "endTime"=>"2020-03-20T18:45:25.025Z", "active"=>false, "success"=>true, "resultMessage"=>"Successfully created HDFS directory.", "serviceRef"=>{"clusterName"=>"00053556", "serviceName"=>"hive-00053556"}}, {"id"=>218, "name"=>"DeployClusterClientConfig", "startTime"=>"2020-03-20T18:45:05.580Z", "endTime"=>"2020-03-20T18:45:07.692Z", "active"=>false, "success"=>true, "resultMessage"=>"Successfully deployed all client configurations.", "clusterRef"=>{"clusterName"=>"00053556"}}, {"id"=>235, "name"=>"Start", "startTime"=>"2020-03-20T18:45:29.713Z", "endTime"=>"2020-03-20T18:45:52.188Z", "active"=>false, "success"=>true, "resultMessage"=>"Successfully started service.", "serviceRef"=>{"clusterName"=>"00053556", "serviceName"=>"hive-00053556"}}, {"id"=>226, "name"=>"HiveCreateMetastoreTables", "startTime"=>"2020-03-20T18:45:07.755Z", "endTime"=>"2020-03-20T18:45:10.731Z", "active"=>false, "success"=>true, "resultMessage"=>"Created Hive Metastore Database Tables successfully.", "serviceRef"=>{"clusterName"=>"00053556", "serviceName"=>"hive-00053556"}, "roleRef"=>{"clusterName"=>"00053556", "serviceName"=>"hive-00053556", "roleName"=>"hive-00053556-HIVEMETASTORE-542ac06e21d3ff036b38e440ce84bb57"}}, {"id"=>232, "name"=>"CreateHiveWarehouseDir", "startTime"=>"2020-03-20T18:45:25.042Z", "endTime"=>"2020-03-20T18:45:29.648Z", "active"=>false, "success"=>true, "resultMessage"=>"Successfully created HDFS directory.", "serviceRef"=>{"clusterName"=>"00053556", "serviceName"=>"hive-00053556"}}, {"id"=>228, "name"=>"HiveValidateMetastore", "startTime"=>"2020-03-20T18:45:10.761Z", "endTime"=>"2020-03-20T18:45:20.146Z", "active"=>false, "success"=>true, "resultMessage"=>"Validated Hive Metastore Schema successfully.", "serviceRef"=>{"clusterName"=>"00053556", "serviceName"=>"hive-00053556"}, "roleRef"=>{"clusterName"=>"00053556", "serviceName"=>"hive-00053556", "roleName"=>"hive-00053556-HIVEMETASTORE-542ac06e21d3ff036b38e440ce84bb57"}}]}, "canRetry"=>false}
20-Mar-2020 18:45:52 2020-03-20 18:45:52 +0000 INFO: ------------ Start SPARK2_ON_YARN FirstRun --------------
20-Mar-2020 18:45:52 2020-03-20 18:45:52 +0000 DEBUG: Request: clusters/00053556/services/spark2_on_yarn-00053556/commands/firstRun
20-Mar-2020 18:45:52 2020-03-20 18:45:52 +0000 DEBUG: Response: {"id"=>240, "name"=>"First Run", "startTime"=>"2020-03-20T18:45:52.359Z", "active"=>true, "children"=>{"items"=>[]}}
20-Mar-2020 18:45:52 2020-03-20 18:45:52 +0000 INFO: ------------ Wait for SPARK2_ON_YARN FirstRun --------------
20-Mar-2020 18:45:52 2020-03-20 18:45:52 +0000 INFO: Waiting for command 240 to complete
20-Mar-2020 18:46:27 .................
20-Mar-2020 18:46:27 2020-03-20 18:46:27 +0000 DEBUG: Response: {"id"=>240, "name"=>"First Run", "startTime"=>"2020-03-20T18:45:52.359Z", "endTime"=>"2020-03-20T18:46:26.500Z", "active"=>false, "success"=>true, "resultMessage"=>"Finished First Run of the following services successfully: spark2_on_yarn-00053556.", "children"=>{"items"=>[{"id"=>249, "name"=>"CreateSparkUserDirCommand", "startTime"=>"2020-03-20T18:45:54.838Z", "endTime"=>"2020-03-20T18:45:59.459Z", "active"=>false, "success"=>true, "resultMessage"=>"Successfully created HDFS directory.", "serviceRef"=>{"clusterName"=>"00053556", "serviceName"=>"spark2_on_yarn-00053556"}}, {"id"=>253, "name"=>"Start", "startTime"=>"2020-03-20T18:46:04.210Z", "endTime"=>"2020-03-20T18:46:26.500Z", "active"=>false, "success"=>true, "resultMessage"=>"Successfully started service.", "serviceRef"=>{"clusterName"=>"00053556", "serviceName"=>"spark2_on_yarn-00053556"}}, {"id"=>251, "name"=>"CreateSparkHistoryDirCommand", "startTime"=>"2020-03-20T18:45:59.477Z", "endTime"=>"2020-03-20T18:46:04.115Z", "active"=>false, "success"=>true, "resultMessage"=>"Successfully created HDFS directory.", "serviceRef"=>{"clusterName"=>"00053556", "serviceName"=>"spark2_on_yarn-00053556"}}, {"id"=>248, "name"=>"GlobalWait", "startTime"=>"2020-03-20T18:45:54.792Z", "endTime"=>"2020-03-20T18:45:54.807Z", "active"=>false, "success"=>true, "resultMessage"=>"Finished waiting"}, {"id"=>241, "name"=>"DeployClusterClientConfig", "startTime"=>"2020-03-20T18:45:52.594Z", "endTime"=>"2020-03-20T18:45:54.792Z", "active"=>false, "success"=>true, "resultMessage"=>"Successfully deployed all client configurations.", "clusterRef"=>{"clusterName"=>"00053556"}}]}, "canRetry"=>false}
20-Mar-2020 18:46:27 2020-03-20 18:46:27 +0000 INFO: ------------ Configure CDAP Service --------------
20-Mar-2020 18:46:27 2020-03-20 18:46:27 +0000 DEBUG: Request: {"items"=>[{"name"=>"security_store_provider", "value"=>"kms"}, {"name"=>"cdap-site.xml_service_safety_valve", "value"=>"<property><name>security.authorization.extension.config.sentry.site.url</name><value>/etc/sentry/conf/sentry-site.xml</value></property><property><name>cdap.ugi.cache.expiration.ms</name><value>60000</value></property><property><name>cdap.http.client.read.timeout.ms</name><value>120000</value></property><property><name>hive.server2.jdbc.url</name><value>jdbc:hive2://it-cmspark2-cmspk22r2-644-53556-1000.coopr.sh:2181,it-cmspark2-cmspk22r2-644-53556-1001.coopr.sh:2181,it-cmspark2-cmspk22r2-644-53556-1002.coopr.sh:2181/;serviceDiscoveryMode=zooKeeper;zooKeeperNamespace=hiveserver2</value></property>"}, {"name"=>"CDAP_service_env_safety_valve", "value"=>"\nSPARK_MAJOR_VERSION=2"}]}
20-Mar-2020 18:46:27 2020-03-20 18:46:27 +0000 DEBUG: Response {"items"=>[{"name"=>"CDAP_service_env_safety_valve", "value"=>"\nSPARK_MAJOR_VERSION=2"}, {"name"=>"cdap-site.xml_service_safety_valve", "value"=>"<property><name>security.authorization.extension.config.sentry.site.url</name><value>/etc/sentry/conf/sentry-site.xml</value></property><property><name>cdap.ugi.cache.expiration.ms</name><value>60000</value></property><property><name>cdap.http.client.read.timeout.ms</name><value>120000</value></property><property><name>hive.server2.jdbc.url</name><value>jdbc:hive2://it-cmspark2-cmspk22r2-644-53556-1000.coopr.sh:2181,it-cmspark2-cmspk22r2-644-53556-1001.coopr.sh:2181,it-cmspark2-cmspk22r2-644-53556-1002.coopr.sh:2181/;serviceDiscoveryMode=zooKeeper;zooKeeperNamespace=hiveserver2</value></property>"}, {"name"=>"csd_compatibility_check_enabled", "value"=>"false"}, {"name"=>"enable_unrecoverable_reset", "value"=>"true"}, {"name"=>"hbase_service", "value"=>"hbase-00053556"}, {"name"=>"hdfs_service", "value"=>"hdfs-00053556"}, {"name"=>"hive_service", "value"=>"hive-00053556"}, {"name"=>"kerberos_auth_enabled", "value"=>"true"}, {"name"=>"security_authorization_cache_ttl_secs", "value"=>"3"}, {"name"=>"security_authorization_extension_jar_path", "value"=>"/data/cloudera/parcels/CDAP/master/ext/security/cdap-sentry-binding-0.11.0-SNAPSHOT.jar"}, {"name"=>"security_store_provider", "value"=>"kms"}, {"name"=>"spark2_on_yarn_service", "value"=>"spark2_on_yarn-00053556"}, {"name"=>"yarn_service", "value"=>"yarn-00053556"}, {"name"=>"zookeeper_service", "value"=>"zookeeper-00053556"}]}
20-Mar-2020 18:46:29 2020-03-20 18:46:29 +0000 INFO: ------------ Start CDAP FirstRun --------------
20-Mar-2020 18:46:29 2020-03-20 18:46:29 +0000 DEBUG: Request: clusters/00053556/services/cdap-00053556/commands/firstRun
20-Mar-2020 18:46:29 2020-03-20 18:46:29 +0000 DEBUG: Response: {"id"=>259, "name"=>"First Run", "startTime"=>"2020-03-20T18:46:29.236Z", "active"=>true, "children"=>{"items"=>[]}}
20-Mar-2020 18:46:29 2020-03-20 18:46:29 +0000 INFO: ------------ Wait for CDAP FirstRun --------------
20-Mar-2020 18:46:29 2020-03-20 18:46:29 +0000 INFO: Waiting for command 259 to complete
20-Mar-2020 18:47:20 .........................
20-Mar-2020 18:47:20 2020-03-20 18:47:20 +0000 ERROR: Caught exception: Command 259 not successful: {"id"=>259, "name"=>"First Run", "startTime"=>"2020-03-20T18:46:29.236Z", "endTime"=>"2020-03-20T18:47:19.618Z", "active"=>false, "success"=>false, "resultMessage"=>"Failed to perform First Run of services.", "children"=>{"items"=>[{"id"=>273, "name"=>"Start", "startTime"=>"2020-03-20T18:46:56.019Z", "endTime"=>"2020-03-20T18:47:19.618Z", "active"=>false, "success"=>false, "resultMessage"=>"Failed to start service.", "serviceRef"=>{"clusterName"=>"00053556", "serviceName"=>"cdap-00053556"}}, {"id"=>268, "name"=>"CreateCdapHdfsDirCommand", "startTime"=>"2020-03-20T18:46:46.595Z", "endTime"=>"2020-03-20T18:46:51.201Z", "active"=>false, "success"=>true, "resultMessage"=>"Successfully created HDFS directory.", "serviceRef"=>{"clusterName"=>"00053556", "serviceName"=>"cdap-00053556"}}, {"id"=>260, "name"=>"DeployClusterClientConfig", "startTime"=>"2020-03-20T18:46:29.328Z", "endTime"=>"2020-03-20T18:46:46.535Z", "active"=>false, "success"=>true, "resultMessage"=>"Successfully deployed all client configurations.", "clusterRef"=>{"clusterName"=>"00053556"}}, {"id"=>267, "name"=>"GlobalWait", "startTime"=>"2020-03-20T18:46:46.535Z", "endTime"=>"2020-03-20T18:46:46.552Z", "active"=>false, "success"=>true, "resultMessage"=>"Finished waiting"}, {"id"=>271, "name"=>"CreateCdapUserHdfsDirCommand", "startTime"=>"2020-03-20T18:46:51.227Z", "endTime"=>"2020-03-20T18:46:55.829Z", "active"=>false, "success"=>true, "resultMessage"=>"Successfully created HDFS directory.", "serviceRef"=>{"clusterName"=>"00053556", "serviceName"=>"cdap-00053556"}}]}, "canRetry"=>true} ["/var/bamboo/xml-data/build-dir/IT-CMSPARK2-CMSPK22R2/coopr-internal/data/automatortypes/chef-solo/cookbooks/coopr_scripts/files/default/coopr-internal/bin/coopr_base_to_cm.rb:685:in `wait_for_command_success'", "/var/bamboo/xml-data/build-dir/IT-CMSPARK2-CMSPK22R2/coopr-internal/data/automatortypes/chef-solo/cookbooks/coopr_scripts/files/default/coopr-internal/bin/coopr_base_to_cm.rb:2174:in `<top (required)>'", "coopr_base_to_cm.rb:20:in `load'", "coopr_base_to_cm.rb:20:in `<main>'"]