20/07/01 11:33:02.042 main DEBUG ShutdownHookManager: Adding shutdown hook 20/07/01 11:33:02.069 main DEBUG Shell: Failed to detect a valid hadoop home directory java.io.FileNotFoundException: HADOOP_HOME and hadoop.home.dir are unset. at org.apache.hadoop.util.Shell.checkHadoopHomeInner(Shell.java:468) at org.apache.hadoop.util.Shell.checkHadoopHome(Shell.java:439) at org.apache.hadoop.util.Shell.(Shell.java:516) at org.apache.hadoop.util.StringUtils.(StringUtils.java:78) at org.apache.hadoop.conf.Configuration.getTimeDurationHelper(Configuration.java:1814) at org.apache.hadoop.conf.Configuration.getTimeDuration(Configuration.java:1791) at org.apache.hadoop.util.ShutdownHookManager.getShutdownTimeout(ShutdownHookManager.java:183) at org.apache.hadoop.util.ShutdownHookManager$HookEntry.(ShutdownHookManager.java:207) at org.apache.hadoop.util.ShutdownHookManager.addShutdownHook(ShutdownHookManager.java:302) at org.apache.spark.util.SparkShutdownHookManager.install(ShutdownHookManager.scala:181) at org.apache.spark.util.ShutdownHookManager$.shutdownHooks$lzycompute(ShutdownHookManager.scala:50) at org.apache.spark.util.ShutdownHookManager$.shutdownHooks(ShutdownHookManager.scala:48) at org.apache.spark.util.ShutdownHookManager$.addShutdownHook(ShutdownHookManager.scala:153) at org.apache.spark.util.ShutdownHookManager$.(ShutdownHookManager.scala:58) at org.apache.spark.util.ShutdownHookManager$.(ShutdownHookManager.scala) at org.apache.spark.util.Utils$.createTempDir(Utils.scala:325) at org.apache.spark.deploy.SparkSubmit.prepareSubmitEnvironment(SparkSubmit.scala:343) at org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:877) at org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180) at org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203) at org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90) at org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1013) at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1022) at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala) 20/07/01 11:33:02.090 main DEBUG Shell: setsid exited with exit code 0 20/07/01 11:33:02.176 main DEBUG MutableMetricsFactory: field org.apache.hadoop.metrics2.lib.MutableRate org.apache.hadoop.security.UserGroupInformation$UgiMetrics.loginSuccess with annotation @org.apache.hadoop.metrics2.annotation.Metric(sampleName=Ops, always=false, valueName=Time, about=, interval=10, type=DEFAULT, value=[Rate of successful kerberos logins and latency (milliseconds)]) 20/07/01 11:33:02.178 main DEBUG MutableMetricsFactory: field org.apache.hadoop.metrics2.lib.MutableRate org.apache.hadoop.security.UserGroupInformation$UgiMetrics.loginFailure with annotation @org.apache.hadoop.metrics2.annotation.Metric(sampleName=Ops, always=false, valueName=Time, about=, interval=10, type=DEFAULT, value=[Rate of failed kerberos logins and latency (milliseconds)]) 20/07/01 11:33:02.178 main DEBUG MutableMetricsFactory: field org.apache.hadoop.metrics2.lib.MutableRate org.apache.hadoop.security.UserGroupInformation$UgiMetrics.getGroups with annotation @org.apache.hadoop.metrics2.annotation.Metric(sampleName=Ops, always=false, valueName=Time, about=, interval=10, type=DEFAULT, value=[GetGroups]) 20/07/01 11:33:02.179 main DEBUG MutableMetricsFactory: field private org.apache.hadoop.metrics2.lib.MutableGaugeLong org.apache.hadoop.security.UserGroupInformation$UgiMetrics.renewalFailuresTotal with annotation @org.apache.hadoop.metrics2.annotation.Metric(sampleName=Ops, always=false, valueName=Time, about=, interval=10, type=DEFAULT, value=[Renewal failures since startup]) 20/07/01 11:33:02.179 main DEBUG MutableMetricsFactory: field private org.apache.hadoop.metrics2.lib.MutableGaugeInt org.apache.hadoop.security.UserGroupInformation$UgiMetrics.renewalFailures with annotation @org.apache.hadoop.metrics2.annotation.Metric(sampleName=Ops, always=false, valueName=Time, about=, interval=10, type=DEFAULT, value=[Renewal failures since last successful login]) 20/07/01 11:33:02.180 main DEBUG MetricsSystemImpl: UgiMetrics, User and group related metrics 20/07/01 11:33:02.193 main DEBUG SecurityUtil: Setting hadoop.security.token.service.use_ip to true 20/07/01 11:33:02.206 main DEBUG Groups: Creating new Groups object 20/07/01 11:33:02.209 main DEBUG NativeCodeLoader: Trying to load the custom-built native-hadoop library... 20/07/01 11:33:02.210 main DEBUG NativeCodeLoader: Failed to load native-hadoop with error: java.lang.UnsatisfiedLinkError: no hadoop in java.library.path 20/07/01 11:33:02.210 main DEBUG NativeCodeLoader: java.library.path=/usr/java/packages/lib/amd64:/usr/lib64:/lib64:/lib:/usr/lib 20/07/01 11:33:02.210 main WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 20/07/01 11:33:02.210 main DEBUG PerformanceAdvisory: Falling back to shell based 20/07/01 11:33:02.213 main DEBUG JniBasedUnixGroupsMappingWithFallback: Group mapping impl=org.apache.hadoop.security.ShellBasedUnixGroupsMapping 20/07/01 11:33:02.305 main DEBUG Groups: Group mapping impl=org.apache.hadoop.security.JniBasedUnixGroupsMappingWithFallback; cacheTimeout=300000; warningDeltaMs=5000 20/07/01 11:33:02.311 main DEBUG UserGroupInformation: hadoop login 20/07/01 11:33:02.311 main DEBUG UserGroupInformation: hadoop login commit 20/07/01 11:33:02.314 main DEBUG UserGroupInformation: using local user:UnixPrincipal: jenkins 20/07/01 11:33:02.314 main DEBUG UserGroupInformation: Using user: "UnixPrincipal: jenkins" with name jenkins 20/07/01 11:33:02.314 main DEBUG UserGroupInformation: User entry: "jenkins" 20/07/01 11:33:02.315 main DEBUG UserGroupInformation: UGI loginUser:jenkins (auth:SIMPLE) 20/07/01 11:33:02.317 main INFO SecurityManager: Changing view acls to: jenkins 20/07/01 11:33:02.317 main INFO SecurityManager: Changing modify acls to: jenkins 20/07/01 11:33:02.318 main INFO SecurityManager: Changing view acls groups to: 20/07/01 11:33:02.318 main INFO SecurityManager: Changing modify acls groups to: 20/07/01 11:33:02.319 main INFO SecurityManager: SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(jenkins); groups with view permissions: Set(); users with modify permissions: Set(jenkins); groups with modify permissions: Set()