20/08/08 15:33:38.571 main WARN Utils: Your hostname, spark-jenkins-arm-worker resolves to a loopback address: 127.0.1.1; using 192.168.1.225 instead (on interface eth0) 20/08/08 15:33:38.572 main WARN Utils: Set SPARK_LOCAL_IP if you need to bind to another address 20/08/08 15:33:38.607 main WARN Utils: Your hostname, spark-jenkins-arm-worker resolves to a loopback address: 127.0.1.1; using 192.168.1.225 instead (on interface eth0) 20/08/08 15:33:38.608 main WARN Utils: Set SPARK_LOCAL_IP if you need to bind to another address 20/08/08 15:33:38.628 main WARN Utils: Your hostname, spark-jenkins-arm-worker resolves to a loopback address: 127.0.1.1; using 192.168.1.225 instead (on interface eth0) 20/08/08 15:33:38.630 main WARN Utils: Set SPARK_LOCAL_IP if you need to bind to another address 20/08/08 15:33:38.636 main WARN Utils: Your hostname, spark-jenkins-arm-worker resolves to a loopback address: 127.0.1.1; using 192.168.1.225 instead (on interface eth0) 20/08/08 15:33:38.638 main WARN Utils: Set SPARK_LOCAL_IP if you need to bind to another address 20/08/08 15:33:39.161 main DEBUG ShutdownHookManager: Adding shutdown hook 20/08/08 15:33:39.217 main DEBUG ShutdownHookManager: Adding shutdown hook 20/08/08 15:33:39.224 main DEBUG ShutdownHookManager: Adding shutdown hook 20/08/08 15:33:39.245 main DEBUG ShutdownHookManager: Adding shutdown hook 20/08/08 15:33:39.294 main DEBUG MutableMetricsFactory: field org.apache.hadoop.metrics2.lib.MutableRate org.apache.hadoop.security.UserGroupInformation$UgiMetrics.loginSuccess with annotation @org.apache.hadoop.metrics2.annotation.Metric(about=, sampleName=Ops, always=false, type=DEFAULT, valueName=Time, value=[Rate of successful kerberos logins and latency (milliseconds)]) 20/08/08 15:33:39.306 main DEBUG MutableMetricsFactory: field org.apache.hadoop.metrics2.lib.MutableRate org.apache.hadoop.security.UserGroupInformation$UgiMetrics.loginFailure with annotation @org.apache.hadoop.metrics2.annotation.Metric(about=, sampleName=Ops, always=false, type=DEFAULT, valueName=Time, value=[Rate of failed kerberos logins and latency (milliseconds)]) 20/08/08 15:33:39.306 main DEBUG MutableMetricsFactory: field org.apache.hadoop.metrics2.lib.MutableRate org.apache.hadoop.security.UserGroupInformation$UgiMetrics.getGroups with annotation @org.apache.hadoop.metrics2.annotation.Metric(about=, sampleName=Ops, always=false, type=DEFAULT, valueName=Time, value=[GetGroups]) 20/08/08 15:33:39.308 main DEBUG MetricsSystemImpl: UgiMetrics, User and group related metrics 20/08/08 15:33:39.323 main DEBUG MutableMetricsFactory: field org.apache.hadoop.metrics2.lib.MutableRate org.apache.hadoop.security.UserGroupInformation$UgiMetrics.loginSuccess with annotation @org.apache.hadoop.metrics2.annotation.Metric(about=, sampleName=Ops, always=false, type=DEFAULT, valueName=Time, value=[Rate of successful kerberos logins and latency (milliseconds)]) 20/08/08 15:33:39.323 main DEBUG MutableMetricsFactory: field org.apache.hadoop.metrics2.lib.MutableRate org.apache.hadoop.security.UserGroupInformation$UgiMetrics.loginSuccess with annotation @org.apache.hadoop.metrics2.annotation.Metric(about=, sampleName=Ops, always=false, type=DEFAULT, valueName=Time, value=[Rate of successful kerberos logins and latency (milliseconds)]) 20/08/08 15:33:39.327 main DEBUG MutableMetricsFactory: field org.apache.hadoop.metrics2.lib.MutableRate org.apache.hadoop.security.UserGroupInformation$UgiMetrics.loginSuccess with annotation @org.apache.hadoop.metrics2.annotation.Metric(about=, always=false, sampleName=Ops, type=DEFAULT, valueName=Time, value=[Rate of successful kerberos logins and latency (milliseconds)]) 20/08/08 15:33:39.335 main DEBUG MutableMetricsFactory: field org.apache.hadoop.metrics2.lib.MutableRate org.apache.hadoop.security.UserGroupInformation$UgiMetrics.loginFailure with annotation @org.apache.hadoop.metrics2.annotation.Metric(about=, sampleName=Ops, always=false, type=DEFAULT, valueName=Time, value=[Rate of failed kerberos logins and latency (milliseconds)]) 20/08/08 15:33:39.335 main DEBUG MutableMetricsFactory: field org.apache.hadoop.metrics2.lib.MutableRate org.apache.hadoop.security.UserGroupInformation$UgiMetrics.getGroups with annotation @org.apache.hadoop.metrics2.annotation.Metric(about=, sampleName=Ops, always=false, type=DEFAULT, valueName=Time, value=[GetGroups]) 20/08/08 15:33:39.337 main DEBUG MutableMetricsFactory: field org.apache.hadoop.metrics2.lib.MutableRate org.apache.hadoop.security.UserGroupInformation$UgiMetrics.loginFailure with annotation @org.apache.hadoop.metrics2.annotation.Metric(about=, sampleName=Ops, always=false, type=DEFAULT, valueName=Time, value=[Rate of failed kerberos logins and latency (milliseconds)]) 20/08/08 15:33:39.337 main DEBUG MetricsSystemImpl: UgiMetrics, User and group related metrics 20/08/08 15:33:39.337 main DEBUG MutableMetricsFactory: field org.apache.hadoop.metrics2.lib.MutableRate org.apache.hadoop.security.UserGroupInformation$UgiMetrics.getGroups with annotation @org.apache.hadoop.metrics2.annotation.Metric(about=, sampleName=Ops, always=false, type=DEFAULT, valueName=Time, value=[GetGroups]) 20/08/08 15:33:39.338 main DEBUG MutableMetricsFactory: field org.apache.hadoop.metrics2.lib.MutableRate org.apache.hadoop.security.UserGroupInformation$UgiMetrics.loginFailure with annotation @org.apache.hadoop.metrics2.annotation.Metric(about=, always=false, sampleName=Ops, type=DEFAULT, valueName=Time, value=[Rate of failed kerberos logins and latency (milliseconds)]) 20/08/08 15:33:39.339 main DEBUG MutableMetricsFactory: field org.apache.hadoop.metrics2.lib.MutableRate org.apache.hadoop.security.UserGroupInformation$UgiMetrics.getGroups with annotation @org.apache.hadoop.metrics2.annotation.Metric(about=, always=false, sampleName=Ops, type=DEFAULT, valueName=Time, value=[GetGroups]) 20/08/08 15:33:39.339 main DEBUG MetricsSystemImpl: UgiMetrics, User and group related metrics 20/08/08 15:33:39.340 main DEBUG MetricsSystemImpl: UgiMetrics, User and group related metrics 20/08/08 15:33:39.377 main DEBUG Shell: Failed to detect a valid hadoop home directory java.io.IOException: HADOOP_HOME or hadoop.home.dir are not set. at org.apache.hadoop.util.Shell.checkHadoopHome(Shell.java:329) at org.apache.hadoop.util.Shell.(Shell.java:354) at org.apache.hadoop.util.StringUtils.(StringUtils.java:80) at org.apache.hadoop.security.SecurityUtil.getAuthenticationMethod(SecurityUtil.java:611) at org.apache.hadoop.security.UserGroupInformation.initialize(UserGroupInformation.java:274) at org.apache.hadoop.security.UserGroupInformation.ensureInitialized(UserGroupInformation.java:262) at org.apache.hadoop.security.UserGroupInformation.loginUserFromSubject(UserGroupInformation.java:807) at org.apache.hadoop.security.UserGroupInformation.getLoginUser(UserGroupInformation.java:777) at org.apache.hadoop.security.UserGroupInformation.getCurrentUser(UserGroupInformation.java:650) at org.apache.spark.util.Utils$.$anonfun$getCurrentUserName$1(Utils.scala:2438) at scala.Option.getOrElse(Option.scala:189) at org.apache.spark.util.Utils$.getCurrentUserName(Utils.scala:2438) at org.apache.spark.SecurityManager.(SecurityManager.scala:79) at org.apache.spark.deploy.SparkSubmit.secMgr$lzycompute$1(SparkSubmit.scala:368) at org.apache.spark.deploy.SparkSubmit.secMgr$1(SparkSubmit.scala:368) at org.apache.spark.deploy.SparkSubmit.$anonfun$prepareSubmitEnvironment$8(SparkSubmit.scala:376) at scala.Option.map(Option.scala:230) at org.apache.spark.deploy.SparkSubmit.prepareSubmitEnvironment(SparkSubmit.scala:376) at org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:877) at org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180) at org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203) at org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90) at org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1013) at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1022) at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala) 20/08/08 15:33:39.403 main DEBUG Shell: Failed to detect a valid hadoop home directory java.io.IOException: HADOOP_HOME or hadoop.home.dir are not set. at org.apache.hadoop.util.Shell.checkHadoopHome(Shell.java:329) at org.apache.hadoop.util.Shell.(Shell.java:354) at org.apache.hadoop.util.StringUtils.(StringUtils.java:80) at org.apache.hadoop.security.SecurityUtil.getAuthenticationMethod(SecurityUtil.java:611) at org.apache.hadoop.security.UserGroupInformation.initialize(UserGroupInformation.java:274) at org.apache.hadoop.security.UserGroupInformation.ensureInitialized(UserGroupInformation.java:262) at org.apache.hadoop.security.UserGroupInformation.loginUserFromSubject(UserGroupInformation.java:807) at org.apache.hadoop.security.UserGroupInformation.getLoginUser(UserGroupInformation.java:777) at org.apache.hadoop.security.UserGroupInformation.getCurrentUser(UserGroupInformation.java:650) at org.apache.spark.util.Utils$.$anonfun$getCurrentUserName$1(Utils.scala:2438) at scala.Option.getOrElse(Option.scala:189) at org.apache.spark.util.Utils$.getCurrentUserName(Utils.scala:2438) at org.apache.spark.SecurityManager.(SecurityManager.scala:79) at org.apache.spark.deploy.SparkSubmit.secMgr$lzycompute$1(SparkSubmit.scala:368) at org.apache.spark.deploy.SparkSubmit.secMgr$1(SparkSubmit.scala:368) at org.apache.spark.deploy.SparkSubmit.$anonfun$prepareSubmitEnvironment$8(SparkSubmit.scala:376) at scala.Option.map(Option.scala:230) at org.apache.spark.deploy.SparkSubmit.prepareSubmitEnvironment(SparkSubmit.scala:376) at org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:877) at org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180) at org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203) at org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90) at org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1013) at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1022) at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala) 20/08/08 15:33:39.407 main DEBUG Shell: Failed to detect a valid hadoop home directory java.io.IOException: HADOOP_HOME or hadoop.home.dir are not set. at org.apache.hadoop.util.Shell.checkHadoopHome(Shell.java:329) at org.apache.hadoop.util.Shell.(Shell.java:354) at org.apache.hadoop.util.StringUtils.(StringUtils.java:80) at org.apache.hadoop.security.SecurityUtil.getAuthenticationMethod(SecurityUtil.java:611) at org.apache.hadoop.security.UserGroupInformation.initialize(UserGroupInformation.java:274) at org.apache.hadoop.security.UserGroupInformation.ensureInitialized(UserGroupInformation.java:262) at org.apache.hadoop.security.UserGroupInformation.loginUserFromSubject(UserGroupInformation.java:807) at org.apache.hadoop.security.UserGroupInformation.getLoginUser(UserGroupInformation.java:777) at org.apache.hadoop.security.UserGroupInformation.getCurrentUser(UserGroupInformation.java:650) at org.apache.spark.util.Utils$.$anonfun$getCurrentUserName$1(Utils.scala:2438) at scala.Option.getOrElse(Option.scala:189) at org.apache.spark.util.Utils$.getCurrentUserName(Utils.scala:2438) at org.apache.spark.SecurityManager.(SecurityManager.scala:79) at org.apache.spark.deploy.SparkSubmit.secMgr$lzycompute$1(SparkSubmit.scala:368) at org.apache.spark.deploy.SparkSubmit.secMgr$1(SparkSubmit.scala:368) at org.apache.spark.deploy.SparkSubmit.$anonfun$prepareSubmitEnvironment$8(SparkSubmit.scala:376) at scala.Option.map(Option.scala:230) at org.apache.spark.deploy.SparkSubmit.prepareSubmitEnvironment(SparkSubmit.scala:376) at org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:877) at org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180) at org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203) at org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90) at org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1013) at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1022) at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala) 20/08/08 15:33:39.414 main DEBUG Shell: setsid exited with exit code 0 20/08/08 15:33:39.421 main DEBUG KerberosName: Kerberos krb5 configuration not found, setting default realm to empty 20/08/08 15:33:39.425 main DEBUG Shell: setsid exited with exit code 0 20/08/08 15:33:39.426 main DEBUG Groups: Creating new Groups object 20/08/08 15:33:39.426 main DEBUG Shell: setsid exited with exit code 0 20/08/08 15:33:39.430 main DEBUG NativeCodeLoader: Trying to load the custom-built native-hadoop library... 20/08/08 15:33:39.430 main DEBUG NativeCodeLoader: Failed to load native-hadoop with error: java.lang.UnsatisfiedLinkError: no hadoop in java.library.path 20/08/08 15:33:39.430 main DEBUG NativeCodeLoader: java.library.path=/home/jenkins/deps/arrow/arrow-cpp-bin/lib:/usr/java/packages/lib/aarch64:/usr/lib/aarch64-linux-gnu/jni:/lib/aarch64-linux-gnu:/usr/lib/aarch64-linux-gnu:/usr/lib/jni:/lib:/usr/lib 20/08/08 15:33:39.430 main WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 20/08/08 15:33:39.431 main DEBUG PerformanceAdvisory: Falling back to shell based 20/08/08 15:33:39.431 main DEBUG JniBasedUnixGroupsMappingWithFallback: Group mapping impl=org.apache.hadoop.security.ShellBasedUnixGroupsMapping 20/08/08 15:33:39.431 main DEBUG KerberosName: Kerberos krb5 configuration not found, setting default realm to empty 20/08/08 15:33:39.433 main DEBUG KerberosName: Kerberos krb5 configuration not found, setting default realm to empty 20/08/08 15:33:39.436 main DEBUG Groups: Creating new Groups object 20/08/08 15:33:39.438 main DEBUG Groups: Creating new Groups object 20/08/08 15:33:39.436 main DEBUG Shell: Failed to detect a valid hadoop home directory java.io.IOException: HADOOP_HOME or hadoop.home.dir are not set. at org.apache.hadoop.util.Shell.checkHadoopHome(Shell.java:329) at org.apache.hadoop.util.Shell.(Shell.java:354) at org.apache.hadoop.util.StringUtils.(StringUtils.java:80) at org.apache.hadoop.security.SecurityUtil.getAuthenticationMethod(SecurityUtil.java:611) at org.apache.hadoop.security.UserGroupInformation.initialize(UserGroupInformation.java:274) at org.apache.hadoop.security.UserGroupInformation.ensureInitialized(UserGroupInformation.java:262) at org.apache.hadoop.security.UserGroupInformation.loginUserFromSubject(UserGroupInformation.java:807) at org.apache.hadoop.security.UserGroupInformation.getLoginUser(UserGroupInformation.java:777) at org.apache.hadoop.security.UserGroupInformation.getCurrentUser(UserGroupInformation.java:650) at org.apache.spark.util.Utils$.$anonfun$getCurrentUserName$1(Utils.scala:2438) at scala.Option.getOrElse(Option.scala:189) at org.apache.spark.util.Utils$.getCurrentUserName(Utils.scala:2438) at org.apache.spark.SecurityManager.(SecurityManager.scala:79) at org.apache.spark.deploy.SparkSubmit.secMgr$lzycompute$1(SparkSubmit.scala:368) at org.apache.spark.deploy.SparkSubmit.secMgr$1(SparkSubmit.scala:368) at org.apache.spark.deploy.SparkSubmit.$anonfun$prepareSubmitEnvironment$8(SparkSubmit.scala:376) at scala.Option.map(Option.scala:230) at org.apache.spark.deploy.SparkSubmit.prepareSubmitEnvironment(SparkSubmit.scala:376) at org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:877) at org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180) at org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203) at org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90) at org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1013) at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1022) at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala) 20/08/08 15:33:39.440 main DEBUG NativeCodeLoader: Trying to load the custom-built native-hadoop library... 20/08/08 15:33:39.441 main DEBUG NativeCodeLoader: Failed to load native-hadoop with error: java.lang.UnsatisfiedLinkError: no hadoop in java.library.path 20/08/08 15:33:39.441 main DEBUG NativeCodeLoader: java.library.path=/home/jenkins/deps/arrow/arrow-cpp-bin/lib:/usr/java/packages/lib/aarch64:/usr/lib/aarch64-linux-gnu/jni:/lib/aarch64-linux-gnu:/usr/lib/aarch64-linux-gnu:/usr/lib/jni:/lib:/usr/lib 20/08/08 15:33:39.441 main WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 20/08/08 15:33:39.442 main DEBUG PerformanceAdvisory: Falling back to shell based 20/08/08 15:33:39.442 main DEBUG JniBasedUnixGroupsMappingWithFallback: Group mapping impl=org.apache.hadoop.security.ShellBasedUnixGroupsMapping 20/08/08 15:33:39.442 main DEBUG NativeCodeLoader: Trying to load the custom-built native-hadoop library... 20/08/08 15:33:39.442 main DEBUG NativeCodeLoader: Failed to load native-hadoop with error: java.lang.UnsatisfiedLinkError: no hadoop in java.library.path 20/08/08 15:33:39.442 main DEBUG NativeCodeLoader: java.library.path=/home/jenkins/deps/arrow/arrow-cpp-bin/lib:/usr/java/packages/lib/aarch64:/usr/lib/aarch64-linux-gnu/jni:/lib/aarch64-linux-gnu:/usr/lib/aarch64-linux-gnu:/usr/lib/jni:/lib:/usr/lib 20/08/08 15:33:39.443 main WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 20/08/08 15:33:39.443 main DEBUG PerformanceAdvisory: Falling back to shell based 20/08/08 15:33:39.444 main DEBUG JniBasedUnixGroupsMappingWithFallback: Group mapping impl=org.apache.hadoop.security.ShellBasedUnixGroupsMapping 20/08/08 15:33:39.452 main DEBUG Shell: setsid exited with exit code 0 20/08/08 15:33:39.459 main DEBUG KerberosName: Kerberos krb5 configuration not found, setting default realm to empty 20/08/08 15:33:39.463 main DEBUG Groups: Creating new Groups object 20/08/08 15:33:39.467 main DEBUG NativeCodeLoader: Trying to load the custom-built native-hadoop library... 20/08/08 15:33:39.468 main DEBUG NativeCodeLoader: Failed to load native-hadoop with error: java.lang.UnsatisfiedLinkError: no hadoop in java.library.path 20/08/08 15:33:39.468 main DEBUG NativeCodeLoader: java.library.path=/home/jenkins/deps/arrow/arrow-cpp-bin/lib:/usr/java/packages/lib/aarch64:/usr/lib/aarch64-linux-gnu/jni:/lib/aarch64-linux-gnu:/usr/lib/aarch64-linux-gnu:/usr/lib/jni:/lib:/usr/lib 20/08/08 15:33:39.468 main WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 20/08/08 15:33:39.469 main DEBUG PerformanceAdvisory: Falling back to shell based 20/08/08 15:33:39.469 main DEBUG JniBasedUnixGroupsMappingWithFallback: Group mapping impl=org.apache.hadoop.security.ShellBasedUnixGroupsMapping 20/08/08 15:33:39.568 main DEBUG Groups: Group mapping impl=org.apache.hadoop.security.JniBasedUnixGroupsMappingWithFallback; cacheTimeout=300000; warningDeltaMs=5000 20/08/08 15:33:39.571 main DEBUG UserGroupInformation: hadoop login 20/08/08 15:33:39.572 main DEBUG UserGroupInformation: hadoop login commit 20/08/08 15:33:39.575 main DEBUG UserGroupInformation: using local user:UnixPrincipal: jenkins 20/08/08 15:33:39.575 main DEBUG UserGroupInformation: Using user: "UnixPrincipal: jenkins" with name jenkins 20/08/08 15:33:39.575 main DEBUG UserGroupInformation: User entry: "jenkins" 20/08/08 15:33:39.575 main DEBUG UserGroupInformation: Assuming keytab is managed externally since logged in from subject. 20/08/08 15:33:39.576 main DEBUG UserGroupInformation: UGI loginUser:jenkins (auth:SIMPLE) 20/08/08 15:33:39.578 main INFO SecurityManager: Changing view acls to: jenkins 20/08/08 15:33:39.578 main INFO SecurityManager: Changing modify acls to: jenkins 20/08/08 15:33:39.579 main INFO SecurityManager: Changing view acls groups to: 20/08/08 15:33:39.580 main INFO SecurityManager: Changing modify acls groups to: 20/08/08 15:33:39.580 main INFO SecurityManager: SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(jenkins); groups with view permissions: Set(); users with modify permissions: Set(jenkins); groups with modify permissions: Set() 20/08/08 15:33:39.580 main DEBUG Groups: Group mapping impl=org.apache.hadoop.security.JniBasedUnixGroupsMappingWithFallback; cacheTimeout=300000; warningDeltaMs=5000 20/08/08 15:33:39.583 main DEBUG UserGroupInformation: hadoop login 20/08/08 15:33:39.584 main DEBUG UserGroupInformation: hadoop login commit 20/08/08 15:33:39.586 main DEBUG UserGroupInformation: using local user:UnixPrincipal: jenkins 20/08/08 15:33:39.587 main DEBUG UserGroupInformation: Using user: "UnixPrincipal: jenkins" with name jenkins 20/08/08 15:33:39.587 main DEBUG UserGroupInformation: User entry: "jenkins" 20/08/08 15:33:39.587 main DEBUG UserGroupInformation: Assuming keytab is managed externally since logged in from subject. 20/08/08 15:33:39.587 main DEBUG UserGroupInformation: UGI loginUser:jenkins (auth:SIMPLE) 20/08/08 15:33:39.589 main INFO SecurityManager: Changing view acls to: jenkins 20/08/08 15:33:39.590 main INFO SecurityManager: Changing modify acls to: jenkins 20/08/08 15:33:39.591 main INFO SecurityManager: Changing view acls groups to: 20/08/08 15:33:39.591 main DEBUG Groups: Group mapping impl=org.apache.hadoop.security.JniBasedUnixGroupsMappingWithFallback; cacheTimeout=300000; warningDeltaMs=5000 20/08/08 15:33:39.592 main INFO SecurityManager: Changing modify acls groups to: 20/08/08 15:33:39.592 main INFO SecurityManager: SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(jenkins); groups with view permissions: Set(); users with modify permissions: Set(jenkins); groups with modify permissions: Set() 20/08/08 15:33:39.594 main DEBUG UserGroupInformation: hadoop login 20/08/08 15:33:39.595 main DEBUG UserGroupInformation: hadoop login commit 20/08/08 15:33:39.597 main DEBUG UserGroupInformation: using local user:UnixPrincipal: jenkins 20/08/08 15:33:39.598 main DEBUG UserGroupInformation: Using user: "UnixPrincipal: jenkins" with name jenkins 20/08/08 15:33:39.598 main DEBUG UserGroupInformation: User entry: "jenkins" 20/08/08 15:33:39.598 main DEBUG UserGroupInformation: Assuming keytab is managed externally since logged in from subject. 20/08/08 15:33:39.599 main DEBUG UserGroupInformation: UGI loginUser:jenkins (auth:SIMPLE) 20/08/08 15:33:39.599 main DEBUG Groups: Group mapping impl=org.apache.hadoop.security.JniBasedUnixGroupsMappingWithFallback; cacheTimeout=300000; warningDeltaMs=5000 20/08/08 15:33:39.601 main INFO SecurityManager: Changing view acls to: jenkins 20/08/08 15:33:39.601 main INFO SecurityManager: Changing modify acls to: jenkins 20/08/08 15:33:39.602 main INFO SecurityManager: Changing view acls groups to: 20/08/08 15:33:39.602 main DEBUG UserGroupInformation: hadoop login 20/08/08 15:33:39.602 main INFO SecurityManager: Changing modify acls groups to: 20/08/08 15:33:39.603 main DEBUG UserGroupInformation: hadoop login commit 20/08/08 15:33:39.603 main INFO SecurityManager: SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(jenkins); groups with view permissions: Set(); users with modify permissions: Set(jenkins); groups with modify permissions: Set() 20/08/08 15:33:39.605 main DEBUG UserGroupInformation: using local user:UnixPrincipal: jenkins 20/08/08 15:33:39.605 main DEBUG UserGroupInformation: Using user: "UnixPrincipal: jenkins" with name jenkins 20/08/08 15:33:39.606 main DEBUG UserGroupInformation: User entry: "jenkins" 20/08/08 15:33:39.606 main DEBUG UserGroupInformation: Assuming keytab is managed externally since logged in from subject. 20/08/08 15:33:39.606 main DEBUG UserGroupInformation: UGI loginUser:jenkins (auth:SIMPLE) 20/08/08 15:33:39.608 main INFO SecurityManager: Changing view acls to: jenkins 20/08/08 15:33:39.609 main INFO SecurityManager: Changing modify acls to: jenkins 20/08/08 15:33:39.610 main INFO SecurityManager: Changing view acls groups to: 20/08/08 15:33:39.610 main INFO SecurityManager: Changing modify acls groups to: 20/08/08 15:33:39.611 main INFO SecurityManager: SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(jenkins); groups with view permissions: Set(); users with modify permissions: Set(jenkins); groups with modify permissions: Set() 20/08/08 15:33:39.858 main DEBUG PythonGatewayServer: Started PythonGatewayServer on port 38781 20/08/08 15:33:39.877 main DEBUG PythonGatewayServer: Started PythonGatewayServer on port 46257 20/08/08 15:33:39.889 main DEBUG PythonGatewayServer: Started PythonGatewayServer on port 44103 20/08/08 15:33:39.902 main DEBUG PythonGatewayServer: Started PythonGatewayServer on port 40619 20/08/08 15:33:40.042 Thread-4 INFO SparkContext: Running Spark version 3.1.0-SNAPSHOT 20/08/08 15:33:40.049 Thread-4 INFO SparkContext: Running Spark version 3.1.0-SNAPSHOT 20/08/08 15:33:40.088 Thread-4 INFO ResourceUtils: ============================================================== 20/08/08 15:33:40.088 Thread-4 INFO ResourceUtils: No custom resources configured for spark.driver. 20/08/08 15:33:40.089 Thread-4 INFO ResourceUtils: ============================================================== 20/08/08 15:33:40.089 Thread-4 INFO SparkContext: Submitted application: ReusedPySparkTestCase 20/08/08 15:33:40.098 Thread-4 INFO ResourceUtils: ============================================================== 20/08/08 15:33:40.098 Thread-4 INFO ResourceUtils: No custom resources configured for spark.driver. 20/08/08 15:33:40.099 Thread-4 INFO ResourceUtils: ============================================================== 20/08/08 15:33:40.099 Thread-4 INFO SparkContext: Submitted application: ReusedPySparkTestCase 20/08/08 15:33:40.120 Thread-4 INFO ResourceProfile: Default ResourceProfile created, executor resources: Map(cores -> name: cores, amount: 1, script: , vendor: , memory -> name: memory, amount: 1024, script: , vendor: ), task resources: Map(cpus -> name: cpus, amount: 1.0) 20/08/08 15:33:40.131 Thread-4 INFO ResourceProfile: Default ResourceProfile created, executor resources: Map(cores -> name: cores, amount: 1, script: , vendor: , memory -> name: memory, amount: 1024, script: , vendor: ), task resources: Map(cpus -> name: cpus, amount: 1.0) 20/08/08 15:33:40.144 Thread-4 INFO ResourceProfile: Limiting resource is cpu 20/08/08 15:33:40.145 Thread-4 INFO ResourceProfileManager: Added ResourceProfile id: 0 20/08/08 15:33:40.149 Thread-4 INFO SparkContext: Running Spark version 3.1.0-SNAPSHOT 20/08/08 15:33:40.155 Thread-4 INFO ResourceProfile: Limiting resource is cpu 20/08/08 15:33:40.156 Thread-4 INFO ResourceProfileManager: Added ResourceProfile id: 0 20/08/08 15:33:40.159 Thread-4 INFO SparkContext: Running Spark version 3.1.0-SNAPSHOT 20/08/08 15:33:40.200 Thread-4 INFO ResourceUtils: ============================================================== 20/08/08 15:33:40.201 Thread-4 INFO ResourceUtils: No custom resources configured for spark.driver. 20/08/08 15:33:40.201 Thread-4 INFO ResourceUtils: ============================================================== 20/08/08 15:33:40.202 Thread-4 INFO SparkContext: Submitted application: ReusedPySparkTestCase 20/08/08 15:33:40.207 Thread-4 INFO ResourceUtils: ============================================================== 20/08/08 15:33:40.207 Thread-4 INFO ResourceUtils: No custom resources configured for spark.driver. 20/08/08 15:33:40.208 Thread-4 INFO ResourceUtils: ============================================================== 20/08/08 15:33:40.208 Thread-4 INFO SparkContext: Submitted application: ReusedPySparkTestCase 20/08/08 15:33:40.214 Thread-4 INFO SecurityManager: Changing view acls to: jenkins 20/08/08 15:33:40.214 Thread-4 INFO SecurityManager: Changing modify acls to: jenkins 20/08/08 15:33:40.214 Thread-4 INFO SecurityManager: Changing view acls groups to: 20/08/08 15:33:40.214 Thread-4 INFO SecurityManager: Changing modify acls groups to: 20/08/08 15:33:40.215 Thread-4 INFO SecurityManager: SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(jenkins); groups with view permissions: Set(); users with modify permissions: Set(jenkins); groups with modify permissions: Set() 20/08/08 15:33:40.223 Thread-4 INFO SecurityManager: Changing view acls to: jenkins 20/08/08 15:33:40.223 Thread-4 INFO SecurityManager: Changing modify acls to: jenkins 20/08/08 15:33:40.223 Thread-4 INFO SecurityManager: Changing view acls groups to: 20/08/08 15:33:40.224 Thread-4 INFO SecurityManager: Changing modify acls groups to: 20/08/08 15:33:40.224 Thread-4 INFO SecurityManager: SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(jenkins); groups with view permissions: Set(); users with modify permissions: Set(jenkins); groups with modify permissions: Set() 20/08/08 15:33:40.232 Thread-4 INFO ResourceProfile: Default ResourceProfile created, executor resources: Map(cores -> name: cores, amount: 1, script: , vendor: , memory -> name: memory, amount: 1024, script: , vendor: ), task resources: Map(cpus -> name: cpus, amount: 1.0) 20/08/08 15:33:40.238 Thread-4 INFO ResourceProfile: Default ResourceProfile created, executor resources: Map(cores -> name: cores, amount: 1, script: , vendor: , memory -> name: memory, amount: 1024, script: , vendor: ), task resources: Map(cpus -> name: cpus, amount: 1.0) 20/08/08 15:33:40.261 Thread-4 INFO ResourceProfile: Limiting resource is cpu 20/08/08 15:33:40.262 Thread-4 INFO ResourceProfileManager: Added ResourceProfile id: 0 20/08/08 15:33:40.263 Thread-4 INFO ResourceProfile: Limiting resource is cpu 20/08/08 15:33:40.264 Thread-4 INFO ResourceProfileManager: Added ResourceProfile id: 0 20/08/08 15:33:40.331 Thread-4 INFO SecurityManager: Changing view acls to: jenkins 20/08/08 15:33:40.331 Thread-4 INFO SecurityManager: Changing modify acls to: jenkins 20/08/08 15:33:40.331 Thread-4 INFO SecurityManager: Changing view acls to: jenkins 20/08/08 15:33:40.331 Thread-4 INFO SecurityManager: Changing view acls groups to: 20/08/08 15:33:40.331 Thread-4 INFO SecurityManager: Changing modify acls groups to: 20/08/08 15:33:40.331 Thread-4 INFO SecurityManager: Changing modify acls to: jenkins 20/08/08 15:33:40.331 Thread-4 INFO SecurityManager: SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(jenkins); groups with view permissions: Set(); users with modify permissions: Set(jenkins); groups with modify permissions: Set() 20/08/08 15:33:40.331 Thread-4 INFO SecurityManager: Changing view acls groups to: 20/08/08 15:33:40.332 Thread-4 INFO SecurityManager: Changing modify acls groups to: 20/08/08 15:33:40.332 Thread-4 INFO SecurityManager: SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(jenkins); groups with view permissions: Set(); users with modify permissions: Set(jenkins); groups with modify permissions: Set() 20/08/08 15:33:40.610 Thread-4 DEBUG TransportServer: Shuffle server started on port: 35519 20/08/08 15:33:40.622 Thread-4 INFO Utils: Successfully started service 'sparkDriver' on port 35519. 20/08/08 15:33:40.623 Thread-4 DEBUG SparkEnv: Using serializer: class org.apache.spark.serializer.JavaSerializer 20/08/08 15:33:40.645 Thread-4 DEBUG TransportServer: Shuffle server started on port: 44433 20/08/08 15:33:40.656 Thread-4 INFO Utils: Successfully started service 'sparkDriver' on port 44433. 20/08/08 15:33:40.657 Thread-4 DEBUG SparkEnv: Using serializer: class org.apache.spark.serializer.JavaSerializer 20/08/08 15:33:40.667 Thread-4 INFO SparkEnv: Registering MapOutputTracker 20/08/08 15:33:40.668 Thread-4 DEBUG MapOutputTrackerMasterEndpoint: init 20/08/08 15:33:40.700 Thread-4 INFO SparkEnv: Registering MapOutputTracker 20/08/08 15:33:40.701 Thread-4 DEBUG MapOutputTrackerMasterEndpoint: init 20/08/08 15:33:40.712 Thread-4 INFO SparkEnv: Registering BlockManagerMaster 20/08/08 15:33:40.717 Thread-4 INFO BlockManagerMasterEndpoint: Using org.apache.spark.storage.DefaultTopologyMapper for getting topology information 20/08/08 15:33:40.718 Thread-4 INFO BlockManagerMasterEndpoint: BlockManagerMasterEndpoint up 20/08/08 15:33:40.729 Thread-4 INFO SparkEnv: Registering BlockManagerMasterHeartbeat 20/08/08 15:33:40.743 Thread-4 DEBUG TransportServer: Shuffle server started on port: 37029 20/08/08 15:33:40.745 Thread-4 INFO DiskBlockManager: Created local directory at /home/jenkins/workspace/spark-master-test-python-arm/python/target/3b65e3b8-1878-4dec-9a39-e8f6a364a9e1/blockmgr-10a5a360-d492-4fab-92b0-416b24c86746 20/08/08 15:33:40.746 Thread-4 INFO SparkEnv: Registering BlockManagerMaster 20/08/08 15:33:40.747 Thread-4 DEBUG DiskBlockManager: Adding shutdown hook 20/08/08 15:33:40.748 Thread-4 DEBUG TransportServer: Shuffle server started on port: 44665 20/08/08 15:33:40.755 Thread-4 INFO BlockManagerMasterEndpoint: Using org.apache.spark.storage.DefaultTopologyMapper for getting topology information 20/08/08 15:33:40.756 Thread-4 INFO BlockManagerMasterEndpoint: BlockManagerMasterEndpoint up 20/08/08 15:33:40.760 Thread-4 INFO Utils: Successfully started service 'sparkDriver' on port 44665. 20/08/08 15:33:40.761 Thread-4 INFO SparkEnv: Registering BlockManagerMasterHeartbeat 20/08/08 15:33:40.762 Thread-4 DEBUG SparkEnv: Using serializer: class org.apache.spark.serializer.JavaSerializer 20/08/08 15:33:40.771 Thread-4 INFO Utils: Successfully started service 'sparkDriver' on port 37029. 20/08/08 15:33:40.773 Thread-4 DEBUG SparkEnv: Using serializer: class org.apache.spark.serializer.JavaSerializer 20/08/08 15:33:40.776 Thread-4 INFO DiskBlockManager: Created local directory at /home/jenkins/workspace/spark-master-test-python-arm/python/target/a84b9e1a-ed45-4723-89a2-71ab2bdfb9c6/blockmgr-864bb866-d036-426d-acfd-db39332dae13 20/08/08 15:33:40.778 Thread-4 DEBUG DiskBlockManager: Adding shutdown hook 20/08/08 15:33:40.787 Thread-4 INFO MemoryStore: MemoryStore started with capacity 366.3 MiB 20/08/08 15:33:40.804 Thread-4 INFO SparkEnv: Registering MapOutputTracker 20/08/08 15:33:40.805 Thread-4 DEBUG MapOutputTrackerMasterEndpoint: init 20/08/08 15:33:40.810 Thread-4 INFO SparkEnv: Registering OutputCommitCoordinator 20/08/08 15:33:40.811 Thread-4 DEBUG OutputCommitCoordinator$OutputCommitCoordinatorEndpoint: init 20/08/08 15:33:40.813 Thread-4 INFO SparkEnv: Registering MapOutputTracker 20/08/08 15:33:40.814 Thread-4 DEBUG MapOutputTrackerMasterEndpoint: init 20/08/08 15:33:40.816 Thread-4 INFO MemoryStore: MemoryStore started with capacity 366.3 MiB 20/08/08 15:33:40.838 Thread-4 INFO SparkEnv: Registering OutputCommitCoordinator 20/08/08 15:33:40.838 Thread-4 DEBUG OutputCommitCoordinator$OutputCommitCoordinatorEndpoint: init 20/08/08 15:33:40.845 Thread-4 INFO SparkEnv: Registering BlockManagerMaster 20/08/08 15:33:40.850 Thread-4 INFO BlockManagerMasterEndpoint: Using org.apache.spark.storage.DefaultTopologyMapper for getting topology information 20/08/08 15:33:40.851 Thread-4 INFO BlockManagerMasterEndpoint: BlockManagerMasterEndpoint up 20/08/08 15:33:40.855 Thread-4 INFO SparkEnv: Registering BlockManagerMaster 20/08/08 15:33:40.857 Thread-4 INFO SparkEnv: Registering BlockManagerMasterHeartbeat 20/08/08 15:33:40.860 Thread-4 INFO BlockManagerMasterEndpoint: Using org.apache.spark.storage.DefaultTopologyMapper for getting topology information 20/08/08 15:33:40.861 Thread-4 INFO BlockManagerMasterEndpoint: BlockManagerMasterEndpoint up 20/08/08 15:33:40.867 Thread-4 INFO SparkEnv: Registering BlockManagerMasterHeartbeat 20/08/08 15:33:40.873 Thread-4 INFO DiskBlockManager: Created local directory at /home/jenkins/workspace/spark-master-test-python-arm/python/target/c1079ab9-e95b-42b9-9d66-c20bb67941db/blockmgr-a3782141-8e0d-4480-8122-96b2f88cf39c 20/08/08 15:33:40.875 Thread-4 DEBUG DiskBlockManager: Adding shutdown hook 20/08/08 15:33:40.892 Thread-4 INFO DiskBlockManager: Created local directory at /home/jenkins/workspace/spark-master-test-python-arm/python/target/f4163c16-245d-4431-9bbe-696bc81ea656/blockmgr-ea09a49b-0b29-40f3-80db-84c1762657a0 20/08/08 15:33:40.894 Thread-4 DEBUG DiskBlockManager: Adding shutdown hook 20/08/08 15:33:40.918 Thread-4 INFO MemoryStore: MemoryStore started with capacity 366.3 MiB 20/08/08 15:33:40.941 Thread-4 INFO MemoryStore: MemoryStore started with capacity 366.3 MiB 20/08/08 15:33:40.948 Thread-4 INFO SparkEnv: Registering OutputCommitCoordinator 20/08/08 15:33:40.949 Thread-4 DEBUG OutputCommitCoordinator$OutputCommitCoordinatorEndpoint: init 20/08/08 15:33:40.982 Thread-4 INFO SparkEnv: Registering OutputCommitCoordinator 20/08/08 15:33:40.984 Thread-4 DEBUG OutputCommitCoordinator$OutputCommitCoordinatorEndpoint: init 20/08/08 15:33:41.103 Thread-4 INFO Executor: Starting executor ID driver on host 192.168.1.225 20/08/08 15:33:41.125 Thread-4 INFO Executor: Starting executor ID driver on host 192.168.1.225 20/08/08 15:33:41.135 Thread-4 DEBUG TransportServer: Shuffle server started on port: 36433 20/08/08 15:33:41.135 Thread-4 INFO Utils: Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 36433. 20/08/08 15:33:41.136 Thread-4 INFO NettyBlockTransferService: Server created on 192.168.1.225:36433 20/08/08 15:33:41.138 Thread-4 INFO BlockManager: Using org.apache.spark.storage.RandomBlockReplicationPolicy for block replication policy 20/08/08 15:33:41.148 Thread-4 INFO BlockManagerMaster: Registering BlockManager BlockManagerId(driver, 192.168.1.225, 36433, None) 20/08/08 15:33:41.151 dispatcher-BlockManagerMaster DEBUG DefaultTopologyMapper: Got a request for 192.168.1.225 20/08/08 15:33:41.152 dispatcher-BlockManagerMaster INFO BlockManagerMasterEndpoint: Registering block manager 192.168.1.225:36433 with 366.3 MiB RAM, BlockManagerId(driver, 192.168.1.225, 36433, None) 20/08/08 15:33:41.155 Thread-4 DEBUG TransportServer: Shuffle server started on port: 38159 20/08/08 15:33:41.155 Thread-4 INFO Utils: Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 38159. 20/08/08 15:33:41.156 Thread-4 INFO NettyBlockTransferService: Server created on 192.168.1.225:38159 20/08/08 15:33:41.157 Thread-4 INFO BlockManagerMaster: Registered BlockManager BlockManagerId(driver, 192.168.1.225, 36433, None) 20/08/08 15:33:41.157 Thread-4 INFO BlockManager: Using org.apache.spark.storage.RandomBlockReplicationPolicy for block replication policy 20/08/08 15:33:41.159 Thread-4 INFO BlockManager: Initialized BlockManager: BlockManagerId(driver, 192.168.1.225, 36433, None) 20/08/08 15:33:41.167 Thread-4 INFO BlockManagerMaster: Registering BlockManager BlockManagerId(driver, 192.168.1.225, 38159, None) 20/08/08 15:33:41.169 dispatcher-BlockManagerMaster DEBUG DefaultTopologyMapper: Got a request for 192.168.1.225 20/08/08 15:33:41.171 dispatcher-BlockManagerMaster INFO BlockManagerMasterEndpoint: Registering block manager 192.168.1.225:38159 with 366.3 MiB RAM, BlockManagerId(driver, 192.168.1.225, 38159, None) 20/08/08 15:33:41.175 Thread-4 INFO BlockManagerMaster: Registered BlockManager BlockManagerId(driver, 192.168.1.225, 38159, None) 20/08/08 15:33:41.177 Thread-4 INFO BlockManager: Initialized BlockManager: BlockManagerId(driver, 192.168.1.225, 38159, None) 20/08/08 15:33:41.210 Thread-4 INFO Executor: Starting executor ID driver on host 192.168.1.225 20/08/08 15:33:41.240 Thread-4 DEBUG TransportServer: Shuffle server started on port: 38545 20/08/08 15:33:41.240 Thread-4 INFO Utils: Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 38545. 20/08/08 15:33:41.241 Thread-4 INFO NettyBlockTransferService: Server created on 192.168.1.225:38545 20/08/08 15:33:41.241 Thread-4 INFO Executor: Starting executor ID driver on host 192.168.1.225 20/08/08 15:33:41.243 Thread-4 INFO BlockManager: Using org.apache.spark.storage.RandomBlockReplicationPolicy for block replication policy 20/08/08 15:33:41.252 Thread-4 INFO BlockManagerMaster: Registering BlockManager BlockManagerId(driver, 192.168.1.225, 38545, None) 20/08/08 15:33:41.255 dispatcher-BlockManagerMaster DEBUG DefaultTopologyMapper: Got a request for 192.168.1.225 20/08/08 15:33:41.256 dispatcher-BlockManagerMaster INFO BlockManagerMasterEndpoint: Registering block manager 192.168.1.225:38545 with 366.3 MiB RAM, BlockManagerId(driver, 192.168.1.225, 38545, None) 20/08/08 15:33:41.260 Thread-4 INFO BlockManagerMaster: Registered BlockManager BlockManagerId(driver, 192.168.1.225, 38545, None) 20/08/08 15:33:41.262 Thread-4 INFO BlockManager: Initialized BlockManager: BlockManagerId(driver, 192.168.1.225, 38545, None) 20/08/08 15:33:41.279 Thread-4 DEBUG TransportServer: Shuffle server started on port: 36391 20/08/08 15:33:41.279 Thread-4 INFO Utils: Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 36391. 20/08/08 15:33:41.280 Thread-4 INFO NettyBlockTransferService: Server created on 192.168.1.225:36391 20/08/08 15:33:41.282 Thread-4 INFO BlockManager: Using org.apache.spark.storage.RandomBlockReplicationPolicy for block replication policy 20/08/08 15:33:41.291 Thread-4 INFO BlockManagerMaster: Registering BlockManager BlockManagerId(driver, 192.168.1.225, 36391, None) 20/08/08 15:33:41.294 dispatcher-BlockManagerMaster DEBUG DefaultTopologyMapper: Got a request for 192.168.1.225 20/08/08 15:33:41.295 dispatcher-BlockManagerMaster INFO BlockManagerMasterEndpoint: Registering block manager 192.168.1.225:36391 with 366.3 MiB RAM, BlockManagerId(driver, 192.168.1.225, 36391, None) 20/08/08 15:33:41.305 Thread-4 INFO BlockManagerMaster: Registered BlockManager BlockManagerId(driver, 192.168.1.225, 36391, None) 20/08/08 15:33:41.307 Thread-4 INFO BlockManager: Initialized BlockManager: BlockManagerId(driver, 192.168.1.225, 36391, None) 20/08/08 15:33:41.599 Thread-4 DEBUG log: Logging to org.slf4j.impl.Log4jLoggerAdapter(org.eclipse.jetty.util.log) via org.eclipse.jetty.util.log.Slf4jLog 20/08/08 15:33:41.601 Thread-4 INFO log: Logging initialized @4508ms to org.eclipse.jetty.util.log.Slf4jLog 20/08/08 15:33:41.607 Thread-4 DEBUG log: Logging to org.slf4j.impl.Log4jLoggerAdapter(org.eclipse.jetty.util.log) via org.eclipse.jetty.util.log.Slf4jLog 20/08/08 15:33:41.610 Thread-4 INFO log: Logging initialized @4515ms to org.eclipse.jetty.util.log.Slf4jLog 20/08/08 15:33:41.640 Thread-4 DEBUG DecoratedObjectFactory: Adding Decorator: org.eclipse.jetty.util.DeprecationWarning@10b61f63 20/08/08 15:33:41.650 Thread-4 DEBUG ContainerLifeCycle: o.e.j.s.ServletContextHandler@1616db8f{/,null,UNAVAILABLE} added {ServletHandler@790568c6{STOPPED},MANAGED} 20/08/08 15:33:41.664 Thread-4 DEBUG DecoratedObjectFactory: Adding Decorator: org.eclipse.jetty.util.DeprecationWarning@10b61f63 20/08/08 15:33:41.673 Thread-4 DEBUG ContainerLifeCycle: o.e.j.s.ServletContextHandler@1616db8f{/,null,UNAVAILABLE} added {ServletHandler@790568c6{STOPPED},MANAGED} 20/08/08 15:33:41.700 Thread-4 DEBUG log: Logging to org.slf4j.impl.Log4jLoggerAdapter(org.eclipse.jetty.util.log) via org.eclipse.jetty.util.log.Slf4jLog 20/08/08 15:33:41.702 Thread-4 INFO log: Logging initialized @4602ms to org.eclipse.jetty.util.log.Slf4jLog 20/08/08 15:33:41.754 Thread-4 DEBUG DecoratedObjectFactory: Adding Decorator: org.eclipse.jetty.util.DeprecationWarning@565f78af 20/08/08 15:33:41.764 Thread-4 DEBUG ContainerLifeCycle: o.e.j.s.ServletContextHandler@6c96c688{/,null,UNAVAILABLE} added {ServletHandler@74813a81{STOPPED},MANAGED} 20/08/08 15:33:41.781 Thread-4 DEBUG log: Logging to org.slf4j.impl.Log4jLoggerAdapter(org.eclipse.jetty.util.log) via org.eclipse.jetty.util.log.Slf4jLog 20/08/08 15:33:41.785 Thread-4 DEBUG SparkContext: Adding shutdown hook 20/08/08 15:33:41.787 Thread-4 DEBUG SparkContext: Adding shutdown hook 20/08/08 15:33:41.796 Thread-4 INFO log: Logging initialized @4687ms to org.eclipse.jetty.util.log.Slf4jLog 20/08/08 15:33:41.843 Thread-4 DEBUG DecoratedObjectFactory: Adding Decorator: org.eclipse.jetty.util.DeprecationWarning@390171ec 20/08/08 15:33:41.855 Thread-4 DEBUG ContainerLifeCycle: o.e.j.s.ServletContextHandler@27d20fb0{/,null,UNAVAILABLE} added {ServletHandler@1c262c8b{STOPPED},MANAGED} 20/08/08 15:33:41.954 Thread-4 DEBUG SparkContext: Adding shutdown hook 20/08/08 15:33:42.003 Thread-4 DEBUG SparkContext: Adding shutdown hook 20/08/08 15:33:43.402 Thread-4 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$pythonToJava$1 20/08/08 15:33:43.408 Thread-4 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$pythonToJava$1 20/08/08 15:33:43.426 Thread-4 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$pythonToJava$1) is now cleaned +++ 20/08/08 15:33:43.433 Thread-4 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$pythonToJava$1) is now cleaned +++ 20/08/08 15:33:43.441 Thread-4 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$toJavaArray$1 20/08/08 15:33:43.445 Thread-4 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$toJavaArray$1) is now cleaned +++ 20/08/08 15:33:43.447 Thread-4 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$toJavaArray$1 20/08/08 15:33:43.449 Thread-4 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$toJavaArray$1) is now cleaned +++ 20/08/08 15:33:43.496 Thread-4 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$pythonToJava$1 20/08/08 15:33:43.546 Thread-4 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$applySchemaToPythonRDD$1 20/08/08 15:33:43.548 Thread-4 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$pythonToJava$1) is now cleaned +++ 20/08/08 15:33:43.576 Thread-4 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$toJavaArray$1 20/08/08 15:33:43.577 Thread-4 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$applySchemaToPythonRDD$1) is now cleaned +++ 20/08/08 15:33:43.578 Thread-4 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$toJavaArray$1) is now cleaned +++ 20/08/08 15:33:43.612 Thread-4 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$applySchemaToPythonRDD$1 20/08/08 15:33:43.623 Thread-4 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$applySchemaToPythonRDD$1) is now cleaned +++ 20/08/08 15:33:43.707 Thread-4 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$applySchemaToPythonRDD$1 20/08/08 15:33:43.730 Thread-4 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$applySchemaToPythonRDD$1) is now cleaned +++ 20/08/08 15:33:44.100 Thread-4 INFO SharedState: loading hive config file: file:/home/jenkins/workspace/spark-master-test-python-arm/sql/core/target/scala-2.12/test-classes/hive-site.xml 20/08/08 15:33:44.212 Thread-4 INFO SharedState: Setting hive.metastore.warehouse.dir ('null') to the value of spark.sql.warehouse.dir ('file:/home/jenkins/workspace/spark-master-test-python-arm/spark-warehouse'). 20/08/08 15:33:44.212 Thread-4 INFO SharedState: Warehouse path is 'file:/home/jenkins/workspace/spark-master-test-python-arm/spark-warehouse'. 20/08/08 15:33:44.251 Thread-4 INFO SharedState: loading hive config file: file:/home/jenkins/workspace/spark-master-test-python-arm/sql/core/target/scala-2.12/test-classes/hive-site.xml 20/08/08 15:33:44.326 Thread-4 INFO SharedState: loading hive config file: file:/home/jenkins/workspace/spark-master-test-python-arm/sql/core/target/scala-2.12/test-classes/hive-site.xml 20/08/08 15:33:44.368 Thread-4 INFO SharedState: Setting hive.metastore.warehouse.dir ('null') to the value of spark.sql.warehouse.dir ('file:/home/jenkins/workspace/spark-master-test-python-arm/spark-warehouse'). 20/08/08 15:33:44.368 Thread-4 INFO SharedState: Warehouse path is 'file:/home/jenkins/workspace/spark-master-test-python-arm/spark-warehouse'. 20/08/08 15:33:44.402 Thread-4 INFO SharedState: Setting hive.metastore.warehouse.dir ('null') to the value of spark.sql.warehouse.dir ('file:/home/jenkins/workspace/spark-master-test-python-arm/spark-warehouse'). 20/08/08 15:33:44.409 Thread-4 INFO SharedState: Warehouse path is 'file:/home/jenkins/workspace/spark-master-test-python-arm/spark-warehouse'. 20/08/08 15:33:45.149 Thread-4 INFO SharedState: loading hive config file: file:/home/jenkins/workspace/spark-master-test-python-arm/sql/core/target/scala-2.12/test-classes/hive-site.xml 20/08/08 15:33:45.254 Thread-4 INFO SharedState: Setting hive.metastore.warehouse.dir ('null') to the value of spark.sql.warehouse.dir ('file:/home/jenkins/workspace/spark-master-test-python-arm/spark-warehouse'). 20/08/08 15:33:45.255 Thread-4 INFO SharedState: Warehouse path is 'file:/home/jenkins/workspace/spark-master-test-python-arm/spark-warehouse'. 20/08/08 15:33:46.678 Thread-4 DEBUG CatalystSqlParser: Parsing command: double 20/08/08 15:33:47.060 Thread-4 DEBUG Analyzer$ResolveReferences: Resolving 'input to input#4 20/08/08 15:33:47.061 Thread-4 DEBUG Analyzer$ResolveReferences: Resolving 'output to output#7 20/08/08 15:33:47.284 Thread-4 DEBUG CountVectorizerModel: Input schema: {"type":"struct","fields":[{"name":"id","type":"long","nullable":true,"metadata":{}},{"name":"words","type":{"type":"array","elementType":"string","containsNull":true},"nullable":true,"metadata":{}},{"name":"expected","type":{"type":"udt","class":"org.apache.spark.ml.linalg.VectorUDT","pyClass":"pyspark.ml.linalg.VectorUDT","sqlType":{"type":"struct","fields":[{"name":"type","type":"byte","nullable":false,"metadata":{}},{"name":"size","type":"integer","nullable":true,"metadata":{}},{"name":"indices","type":{"type":"array","elementType":"integer","containsNull":false},"nullable":true,"metadata":{}},{"name":"values","type":{"type":"array","elementType":"double","containsNull":false},"nullable":true,"metadata":{}}]}},"nullable":true,"metadata":{}}]} 20/08/08 15:33:47.302 Thread-4 DEBUG CountVectorizerModel: Expected output schema: {"type":"struct","fields":[{"name":"id","type":"long","nullable":true,"metadata":{}},{"name":"words","type":{"type":"array","elementType":"string","containsNull":true},"nullable":true,"metadata":{}},{"name":"expected","type":{"type":"udt","class":"org.apache.spark.ml.linalg.VectorUDT","pyClass":"pyspark.ml.linalg.VectorUDT","sqlType":{"type":"struct","fields":[{"name":"type","type":"byte","nullable":false,"metadata":{}},{"name":"size","type":"integer","nullable":true,"metadata":{}},{"name":"indices","type":{"type":"array","elementType":"integer","containsNull":false},"nullable":true,"metadata":{}},{"name":"values","type":{"type":"array","elementType":"double","containsNull":false},"nullable":true,"metadata":{}}]}},"nullable":true,"metadata":{}},{"name":"features","type":{"type":"udt","class":"org.apache.spark.ml.linalg.VectorUDT","pyClass":"pyspark.ml.linalg.VectorUDT","sqlType":{"type":"struct","fields":[{"name":"type","type":"byte","nullable":false,"metadata":{}},{"name":"size","type":"integer","nullable":true,"metadata":{}},{"name":"indices","type":{"type":"array","elementType":"integer","containsNull":false},"nullable":true,"metadata":{}},{"name":"values","type":{"type":"array","elementType":"double","containsNull":false},"nullable":true,"metadata":{}}]}},"nullable":false,"metadata":{"ml_attr":{"attrs":{},"num_attrs":3}}}]} 20/08/08 15:33:47.354 Thread-4 INFO MemoryStore: Block broadcast_0 stored as values in memory (estimated size 256.0 B, free 366.3 MiB) 20/08/08 15:33:47.356 Thread-4 DEBUG BlockManager: Put block broadcast_0 locally took 39 ms 20/08/08 15:33:47.358 Thread-4 DEBUG BlockManager: Putting block broadcast_0 without replication took 41 ms 20/08/08 15:33:47.401 Thread-4 INFO MemoryStore: Block broadcast_0_piece0 stored as bytes in memory (estimated size 264.0 B, free 366.3 MiB) 20/08/08 15:33:47.403 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master broadcast_0_piece0 for BlockManagerId(driver, 192.168.1.225, 36433, None) 20/08/08 15:33:47.405 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added broadcast_0_piece0 in memory on 192.168.1.225:36433 (size: 264.0 B, free: 366.3 MiB) 20/08/08 15:33:47.406 Thread-4 DEBUG BlockManagerMaster: Updated info of block broadcast_0_piece0 20/08/08 15:33:47.407 Thread-4 DEBUG BlockManager: Told master about block broadcast_0_piece0 20/08/08 15:33:47.407 Thread-4 DEBUG BlockManager: Put block broadcast_0_piece0 locally took 9 ms 20/08/08 15:33:47.408 Thread-4 DEBUG BlockManager: Putting block broadcast_0_piece0 without replication took 9 ms 20/08/08 15:33:47.409 Thread-4 INFO SparkContext: Created broadcast 0 from broadcast at CountVectorizer.scala:306 20/08/08 15:33:47.795 Thread-4 DEBUG Analyzer$ResolveReferences: Resolving 'prediction to prediction#1 20/08/08 15:33:47.796 Thread-4 DEBUG Analyzer$ResolveReferences: Resolving 'features to features#0 20/08/08 15:33:47.850 Thread-4 DEBUG Analyzer$ResolveReferences: Resolving 'user to user#0L 20/08/08 15:33:47.851 Thread-4 DEBUG Analyzer$ResolveReferences: Resolving 'item to item#1L 20/08/08 15:33:47.852 Thread-4 DEBUG Analyzer$ResolveReferences: Resolving 'rating to rating#2 20/08/08 15:33:47.885 Thread-4 DEBUG Analyzer$ResolveReferences: Resolving 'words to words#1 20/08/08 15:33:47.957 Thread-4 DEBUG Analyzer$ResolveReferences: Resolving 'features to features#4 20/08/08 15:33:48.058 Thread-4 DEBUG Analyzer$ResolveReferences: Resolving 'features to features#9 20/08/08 15:33:48.059 Thread-4 DEBUG Analyzer$ResolveReferences: Resolving 'expected to expected#2 20/08/08 15:33:48.077 Thread-4 DEBUG Analyzer$ResolveReferences: Resolving 'features to features#4 20/08/08 15:33:48.192 Thread-4 DEBUG WholeStageCodegenExec: /* 001 */ public Object generate(Object[] references) { /* 002 */ return new GeneratedIteratorForCodegenStage2(references); /* 003 */ } /* 004 */ /* 005 */ // codegenStageId=2 /* 006 */ final class GeneratedIteratorForCodegenStage2 extends org.apache.spark.sql.execution.BufferedRowIterator { /* 007 */ private Object[] references; /* 008 */ private scala.collection.Iterator[] inputs; /* 009 */ private scala.collection.Iterator inputadapter_input_0; /* 010 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[] project_mutableStateArray_0 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[1]; /* 011 */ /* 012 */ public GeneratedIteratorForCodegenStage2(Object[] references) { /* 013 */ this.references = references; /* 014 */ } /* 015 */ /* 016 */ public void init(int index, scala.collection.Iterator[] inputs) { /* 017 */ partitionIndex = index; /* 018 */ this.inputs = inputs; /* 019 */ inputadapter_input_0 = inputs[0]; /* 020 */ project_mutableStateArray_0[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(2, 0); /* 021 */ /* 022 */ } /* 023 */ /* 024 */ protected void processNext() throws java.io.IOException { /* 025 */ while ( inputadapter_input_0.hasNext()) { /* 026 */ InternalRow inputadapter_row_0 = (InternalRow) inputadapter_input_0.next(); /* 027 */ /* 028 */ long inputadapter_value_0 = inputadapter_row_0.getLong(0); /* 029 */ boolean project_isNull_0 = false; /* 030 */ double project_value_0 = -1.0; /* 031 */ if (!false) { /* 032 */ project_value_0 = (double) inputadapter_value_0; /* 033 */ } /* 034 */ boolean inputadapter_isNull_1 = inputadapter_row_0.isNullAt(1); /* 035 */ double inputadapter_value_1 = inputadapter_isNull_1 ? /* 036 */ -1.0 : (inputadapter_row_0.getDouble(1)); /* 037 */ project_mutableStateArray_0[0].reset(); /* 038 */ /* 039 */ project_mutableStateArray_0[0].zeroOutNullBytes(); /* 040 */ /* 041 */ project_mutableStateArray_0[0].write(0, project_value_0); /* 042 */ /* 043 */ if (inputadapter_isNull_1) { /* 044 */ project_mutableStateArray_0[0].setNullAt(1); /* 045 */ } else { /* 046 */ project_mutableStateArray_0[0].write(1, inputadapter_value_1); /* 047 */ } /* 048 */ append((project_mutableStateArray_0[0].getRow())); /* 049 */ if (shouldStop()) return; /* 050 */ } /* 051 */ } /* 052 */ /* 053 */ } 20/08/08 15:33:48.251 Thread-4 DEBUG CodeGenerator: /* 001 */ public Object generate(Object[] references) { /* 002 */ return new GeneratedIteratorForCodegenStage2(references); /* 003 */ } /* 004 */ /* 005 */ // codegenStageId=2 /* 006 */ final class GeneratedIteratorForCodegenStage2 extends org.apache.spark.sql.execution.BufferedRowIterator { /* 007 */ private Object[] references; /* 008 */ private scala.collection.Iterator[] inputs; /* 009 */ private scala.collection.Iterator inputadapter_input_0; /* 010 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[] project_mutableStateArray_0 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[1]; /* 011 */ /* 012 */ public GeneratedIteratorForCodegenStage2(Object[] references) { /* 013 */ this.references = references; /* 014 */ } /* 015 */ /* 016 */ public void init(int index, scala.collection.Iterator[] inputs) { /* 017 */ partitionIndex = index; /* 018 */ this.inputs = inputs; /* 019 */ inputadapter_input_0 = inputs[0]; /* 020 */ project_mutableStateArray_0[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(2, 0); /* 021 */ /* 022 */ } /* 023 */ /* 024 */ protected void processNext() throws java.io.IOException { /* 025 */ while ( inputadapter_input_0.hasNext()) { /* 026 */ InternalRow inputadapter_row_0 = (InternalRow) inputadapter_input_0.next(); /* 027 */ /* 028 */ long inputadapter_value_0 = inputadapter_row_0.getLong(0); /* 029 */ boolean project_isNull_0 = false; /* 030 */ double project_value_0 = -1.0; /* 031 */ if (!false) { /* 032 */ project_value_0 = (double) inputadapter_value_0; /* 033 */ } /* 034 */ boolean inputadapter_isNull_1 = inputadapter_row_0.isNullAt(1); /* 035 */ double inputadapter_value_1 = inputadapter_isNull_1 ? /* 036 */ -1.0 : (inputadapter_row_0.getDouble(1)); /* 037 */ project_mutableStateArray_0[0].reset(); /* 038 */ /* 039 */ project_mutableStateArray_0[0].zeroOutNullBytes(); /* 040 */ /* 041 */ project_mutableStateArray_0[0].write(0, project_value_0); /* 042 */ /* 043 */ if (inputadapter_isNull_1) { /* 044 */ project_mutableStateArray_0[0].setNullAt(1); /* 045 */ } else { /* 046 */ project_mutableStateArray_0[0].write(1, inputadapter_value_1); /* 047 */ } /* 048 */ append((project_mutableStateArray_0[0].getRow())); /* 049 */ if (shouldStop()) return; /* 050 */ } /* 051 */ } /* 052 */ /* 053 */ } 20/08/08 15:33:48.578 Thread-4 DEBUG WholeStageCodegenExec: /* 001 */ public Object generate(Object[] references) { /* 002 */ return new GeneratedIteratorForCodegenStage1(references); /* 003 */ } /* 004 */ /* 005 */ // codegenStageId=1 /* 006 */ final class GeneratedIteratorForCodegenStage1 extends org.apache.spark.sql.execution.BufferedRowIterator { /* 007 */ private Object[] references; /* 008 */ private scala.collection.Iterator[] inputs; /* 009 */ private scala.collection.Iterator rdd_input_0; /* 010 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[] rdd_mutableStateArray_0 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[2]; /* 011 */ /* 012 */ public GeneratedIteratorForCodegenStage1(Object[] references) { /* 013 */ this.references = references; /* 014 */ } /* 015 */ /* 016 */ public void init(int index, scala.collection.Iterator[] inputs) { /* 017 */ partitionIndex = index; /* 018 */ this.inputs = inputs; /* 019 */ rdd_input_0 = inputs[0]; /* 020 */ rdd_mutableStateArray_0[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(3, 0); /* 021 */ rdd_mutableStateArray_0[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(3, 0); /* 022 */ /* 023 */ } /* 024 */ /* 025 */ protected void processNext() throws java.io.IOException { /* 026 */ while ( rdd_input_0.hasNext()) { /* 027 */ InternalRow rdd_row_0 = (InternalRow) rdd_input_0.next(); /* 028 */ ((org.apache.spark.sql.execution.metric.SQLMetric) references[0] /* numOutputRows */).add(1); /* 029 */ boolean rdd_isNull_0 = rdd_row_0.isNullAt(0); /* 030 */ long rdd_value_0 = rdd_isNull_0 ? /* 031 */ -1L : (rdd_row_0.getLong(0)); /* 032 */ Object project_arg_0 = rdd_isNull_0 ? null : ((scala.Function1[]) references[1] /* converters */)[0].apply(rdd_value_0); /* 033 */ /* 034 */ Integer project_result_0 = null; /* 035 */ try { /* 036 */ project_result_0 = (Integer)((scala.Function1) references[3] /* udf */).apply(project_arg_0); /* 037 */ } catch (Exception e) { /* 038 */ throw new org.apache.spark.SparkException(((java.lang.String) references[2] /* errMsg */), e); /* 039 */ } /* 040 */ /* 041 */ boolean project_isNull_0 = project_result_0 == null; /* 042 */ int project_value_0 = -1; /* 043 */ if (!project_isNull_0) { /* 044 */ project_value_0 = project_result_0; /* 045 */ } /* 046 */ boolean rdd_isNull_1 = rdd_row_0.isNullAt(1); /* 047 */ long rdd_value_1 = rdd_isNull_1 ? /* 048 */ -1L : (rdd_row_0.getLong(1)); /* 049 */ Object project_arg_1 = rdd_isNull_1 ? null : ((scala.Function1[]) references[4] /* converters */)[0].apply(rdd_value_1); /* 050 */ /* 051 */ Integer project_result_1 = null; /* 052 */ try { /* 053 */ project_result_1 = (Integer)((scala.Function1) references[6] /* udf */).apply(project_arg_1); /* 054 */ } catch (Exception e) { /* 055 */ throw new org.apache.spark.SparkException(((java.lang.String) references[5] /* errMsg */), e); /* 056 */ } /* 057 */ /* 058 */ boolean project_isNull_2 = project_result_1 == null; /* 059 */ int project_value_2 = -1; /* 060 */ if (!project_isNull_2) { /* 061 */ project_value_2 = project_result_1; /* 062 */ } /* 063 */ boolean rdd_isNull_2 = rdd_row_0.isNullAt(2); /* 064 */ double rdd_value_2 = rdd_isNull_2 ? /* 065 */ -1.0 : (rdd_row_0.getDouble(2)); /* 066 */ boolean project_isNull_4 = rdd_isNull_2; /* 067 */ float project_value_4 = -1.0f; /* 068 */ if (!rdd_isNull_2) { /* 069 */ project_value_4 = (float) rdd_value_2; /* 070 */ } /* 071 */ rdd_mutableStateArray_0[1].reset(); /* 072 */ /* 073 */ rdd_mutableStateArray_0[1].zeroOutNullBytes(); /* 074 */ /* 075 */ rdd_mutableStateArray_0[1].write(0, project_value_0); /* 076 */ /* 077 */ rdd_mutableStateArray_0[1].write(1, project_value_2); /* 078 */ /* 079 */ if (project_isNull_4) { /* 080 */ rdd_mutableStateArray_0[1].setNullAt(2); /* 081 */ } else { /* 082 */ rdd_mutableStateArray_0[1].write(2, project_value_4); /* 083 */ } /* 084 */ append((rdd_mutableStateArray_0[1].getRow())); /* 085 */ if (shouldStop()) return; /* 086 */ } /* 087 */ } /* 088 */ /* 089 */ } 20/08/08 15:33:48.590 Thread-4 INFO CodeGenerator: Code generated in 396.897911 ms 20/08/08 15:33:48.613 Thread-4 DEBUG WholeStageCodegenExec: /* 001 */ public Object generate(Object[] references) { /* 002 */ return new GeneratedIteratorForCodegenStage1(references); /* 003 */ } /* 004 */ /* 005 */ // codegenStageId=1 /* 006 */ final class GeneratedIteratorForCodegenStage1 extends org.apache.spark.sql.execution.BufferedRowIterator { /* 007 */ private Object[] references; /* 008 */ private scala.collection.Iterator[] inputs; /* 009 */ private boolean range_initRange_0; /* 010 */ private long range_nextIndex_0; /* 011 */ private TaskContext range_taskContext_0; /* 012 */ private InputMetrics range_inputMetrics_0; /* 013 */ private long range_batchEnd_0; /* 014 */ private long range_numElementsTodo_0; /* 015 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[] range_mutableStateArray_0 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[1]; /* 016 */ /* 017 */ public GeneratedIteratorForCodegenStage1(Object[] references) { /* 018 */ this.references = references; /* 019 */ } /* 020 */ /* 021 */ public void init(int index, scala.collection.Iterator[] inputs) { /* 022 */ partitionIndex = index; /* 023 */ this.inputs = inputs; /* 024 */ /* 025 */ range_taskContext_0 = TaskContext.get(); /* 026 */ range_inputMetrics_0 = range_taskContext_0.taskMetrics().inputMetrics(); /* 027 */ range_mutableStateArray_0[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(1, 0); /* 028 */ /* 029 */ } /* 030 */ /* 031 */ private void initRange(int idx) { /* 032 */ java.math.BigInteger index = java.math.BigInteger.valueOf(idx); /* 033 */ java.math.BigInteger numSlice = java.math.BigInteger.valueOf(4L); /* 034 */ java.math.BigInteger numElement = java.math.BigInteger.valueOf(10L); /* 035 */ java.math.BigInteger step = java.math.BigInteger.valueOf(1L); /* 036 */ java.math.BigInteger start = java.math.BigInteger.valueOf(0L); /* 037 */ long partitionEnd; /* 038 */ /* 039 */ java.math.BigInteger st = index.multiply(numElement).divide(numSlice).multiply(step).add(start); /* 040 */ if (st.compareTo(java.math.BigInteger.valueOf(Long.MAX_VALUE)) > 0) { /* 041 */ range_nextIndex_0 = Long.MAX_VALUE; /* 042 */ } else if (st.compareTo(java.math.BigInteger.valueOf(Long.MIN_VALUE)) < 0) { /* 043 */ range_nextIndex_0 = Long.MIN_VALUE; /* 044 */ } else { /* 045 */ range_nextIndex_0 = st.longValue(); /* 046 */ } /* 047 */ range_batchEnd_0 = range_nextIndex_0; /* 048 */ /* 049 */ java.math.BigInteger end = index.add(java.math.BigInteger.ONE).multiply(numElement).divide(numSlice) /* 050 */ .multiply(step).add(start); /* 051 */ if (end.compareTo(java.math.BigInteger.valueOf(Long.MAX_VALUE)) > 0) { /* 052 */ partitionEnd = Long.MAX_VALUE; /* 053 */ } else if (end.compareTo(java.math.BigInteger.valueOf(Long.MIN_VALUE)) < 0) { /* 054 */ partitionEnd = Long.MIN_VALUE; /* 055 */ } else { /* 056 */ partitionEnd = end.longValue(); /* 057 */ } /* 058 */ /* 059 */ java.math.BigInteger startToEnd = java.math.BigInteger.valueOf(partitionEnd).subtract( /* 060 */ java.math.BigInteger.valueOf(range_nextIndex_0)); /* 061 */ range_numElementsTodo_0 = startToEnd.divide(step).longValue(); /* 062 */ if (range_numElementsTodo_0 < 0) { /* 063 */ range_numElementsTodo_0 = 0; /* 064 */ } else if (startToEnd.remainder(step).compareTo(java.math.BigInteger.valueOf(0L)) != 0) { /* 065 */ range_numElementsTodo_0++; /* 066 */ } /* 067 */ } /* 068 */ /* 069 */ protected void processNext() throws java.io.IOException { /* 070 */ // initialize Range /* 071 */ if (!range_initRange_0) { /* 072 */ range_initRange_0 = true; /* 073 */ initRange(partitionIndex); /* 074 */ } /* 075 */ /* 076 */ while (true) { /* 077 */ if (range_nextIndex_0 == range_batchEnd_0) { /* 078 */ long range_nextBatchTodo_0; /* 079 */ if (range_numElementsTodo_0 > 1000L) { /* 080 */ range_nextBatchTodo_0 = 1000L; /* 081 */ range_numElementsTodo_0 -= 1000L; /* 082 */ } else { /* 083 */ range_nextBatchTodo_0 = range_numElementsTodo_0; /* 084 */ range_numElementsTodo_0 = 0; /* 085 */ if (range_nextBatchTodo_0 == 0) break; /* 086 */ } /* 087 */ range_batchEnd_0 += range_nextBatchTodo_0 * 1L; /* 088 */ } /* 089 */ /* 090 */ int range_localEnd_0 = (int)((range_batchEnd_0 - range_nextIndex_0) / 1L); /* 091 */ for (int range_localIdx_0 = 0; range_localIdx_0 < range_localEnd_0; range_localIdx_0++) { /* 092 */ long range_value_0 = ((long)range_localIdx_0 * 1L) + range_nextIndex_0; /* 093 */ /* 094 */ range_mutableStateArray_0[0].reset(); /* 095 */ /* 096 */ range_mutableStateArray_0[0].write(0, range_value_0); /* 097 */ append((range_mutableStateArray_0[0].getRow())); /* 098 */ /* 099 */ if (shouldStop()) { /* 100 */ range_nextIndex_0 = range_value_0 + 1L; /* 101 */ ((org.apache.spark.sql.execution.metric.SQLMetric) references[0] /* numOutputRows */).add(range_localIdx_0 + 1); /* 102 */ range_inputMetrics_0.incRecordsRead(range_localIdx_0 + 1); /* 103 */ return; /* 104 */ } /* 105 */ /* 106 */ } /* 107 */ range_nextIndex_0 = range_batchEnd_0; /* 108 */ ((org.apache.spark.sql.execution.metric.SQLMetric) references[0] /* numOutputRows */).add(range_localEnd_0); /* 109 */ range_inputMetrics_0.incRecordsRead(range_localEnd_0); /* 110 */ range_taskContext_0.killTaskIfInterrupted(); /* 111 */ } /* 112 */ } /* 113 */ /* 114 */ } 20/08/08 15:33:48.621 Thread-4 DEBUG CodeGenerator: /* 001 */ public Object generate(Object[] references) { /* 002 */ return new GeneratedIteratorForCodegenStage1(references); /* 003 */ } /* 004 */ /* 005 */ // codegenStageId=1 /* 006 */ final class GeneratedIteratorForCodegenStage1 extends org.apache.spark.sql.execution.BufferedRowIterator { /* 007 */ private Object[] references; /* 008 */ private scala.collection.Iterator[] inputs; /* 009 */ private boolean range_initRange_0; /* 010 */ private long range_nextIndex_0; /* 011 */ private TaskContext range_taskContext_0; /* 012 */ private InputMetrics range_inputMetrics_0; /* 013 */ private long range_batchEnd_0; /* 014 */ private long range_numElementsTodo_0; /* 015 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[] range_mutableStateArray_0 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[1]; /* 016 */ /* 017 */ public GeneratedIteratorForCodegenStage1(Object[] references) { /* 018 */ this.references = references; /* 019 */ } /* 020 */ /* 021 */ public void init(int index, scala.collection.Iterator[] inputs) { /* 022 */ partitionIndex = index; /* 023 */ this.inputs = inputs; /* 024 */ /* 025 */ range_taskContext_0 = TaskContext.get(); /* 026 */ range_inputMetrics_0 = range_taskContext_0.taskMetrics().inputMetrics(); /* 027 */ range_mutableStateArray_0[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(1, 0); /* 028 */ /* 029 */ } /* 030 */ /* 031 */ private void initRange(int idx) { /* 032 */ java.math.BigInteger index = java.math.BigInteger.valueOf(idx); /* 033 */ java.math.BigInteger numSlice = java.math.BigInteger.valueOf(4L); /* 034 */ java.math.BigInteger numElement = java.math.BigInteger.valueOf(10L); /* 035 */ java.math.BigInteger step = java.math.BigInteger.valueOf(1L); /* 036 */ java.math.BigInteger start = java.math.BigInteger.valueOf(0L); /* 037 */ long partitionEnd; /* 038 */ /* 039 */ java.math.BigInteger st = index.multiply(numElement).divide(numSlice).multiply(step).add(start); /* 040 */ if (st.compareTo(java.math.BigInteger.valueOf(Long.MAX_VALUE)) > 0) { /* 041 */ range_nextIndex_0 = Long.MAX_VALUE; /* 042 */ } else if (st.compareTo(java.math.BigInteger.valueOf(Long.MIN_VALUE)) < 0) { /* 043 */ range_nextIndex_0 = Long.MIN_VALUE; /* 044 */ } else { /* 045 */ range_nextIndex_0 = st.longValue(); /* 046 */ } /* 047 */ range_batchEnd_0 = range_nextIndex_0; /* 048 */ /* 049 */ java.math.BigInteger end = index.add(java.math.BigInteger.ONE).multiply(numElement).divide(numSlice) /* 050 */ .multiply(step).add(start); /* 051 */ if (end.compareTo(java.math.BigInteger.valueOf(Long.MAX_VALUE)) > 0) { /* 052 */ partitionEnd = Long.MAX_VALUE; /* 053 */ } else if (end.compareTo(java.math.BigInteger.valueOf(Long.MIN_VALUE)) < 0) { /* 054 */ partitionEnd = Long.MIN_VALUE; /* 055 */ } else { /* 056 */ partitionEnd = end.longValue(); /* 057 */ } /* 058 */ /* 059 */ java.math.BigInteger startToEnd = java.math.BigInteger.valueOf(partitionEnd).subtract( /* 060 */ java.math.BigInteger.valueOf(range_nextIndex_0)); /* 061 */ range_numElementsTodo_0 = startToEnd.divide(step).longValue(); /* 062 */ if (range_numElementsTodo_0 < 0) { /* 063 */ range_numElementsTodo_0 = 0; /* 064 */ } else if (startToEnd.remainder(step).compareTo(java.math.BigInteger.valueOf(0L)) != 0) { /* 065 */ range_numElementsTodo_0++; /* 066 */ } /* 067 */ } /* 068 */ /* 069 */ protected void processNext() throws java.io.IOException { /* 070 */ // initialize Range /* 071 */ if (!range_initRange_0) { /* 072 */ range_initRange_0 = true; /* 073 */ initRange(partitionIndex); /* 074 */ } /* 075 */ /* 076 */ while (true) { /* 077 */ if (range_nextIndex_0 == range_batchEnd_0) { /* 078 */ long range_nextBatchTodo_0; /* 079 */ if (range_numElementsTodo_0 > 1000L) { /* 080 */ range_nextBatchTodo_0 = 1000L; /* 081 */ range_numElementsTodo_0 -= 1000L; /* 082 */ } else { /* 083 */ range_nextBatchTodo_0 = range_numElementsTodo_0; /* 084 */ range_numElementsTodo_0 = 0; /* 085 */ if (range_nextBatchTodo_0 == 0) break; /* 086 */ } /* 087 */ range_batchEnd_0 += range_nextBatchTodo_0 * 1L; /* 088 */ } /* 089 */ /* 090 */ int range_localEnd_0 = (int)((range_batchEnd_0 - range_nextIndex_0) / 1L); /* 091 */ for (int range_localIdx_0 = 0; range_localIdx_0 < range_localEnd_0; range_localIdx_0++) { /* 092 */ long range_value_0 = ((long)range_localIdx_0 * 1L) + range_nextIndex_0; /* 093 */ /* 094 */ range_mutableStateArray_0[0].reset(); /* 095 */ /* 096 */ range_mutableStateArray_0[0].write(0, range_value_0); /* 097 */ append((range_mutableStateArray_0[0].getRow())); /* 098 */ /* 099 */ if (shouldStop()) { /* 100 */ range_nextIndex_0 = range_value_0 + 1L; /* 101 */ ((org.apache.spark.sql.execution.metric.SQLMetric) references[0] /* numOutputRows */).add(range_localIdx_0 + 1); /* 102 */ range_inputMetrics_0.incRecordsRead(range_localIdx_0 + 1); /* 103 */ return; /* 104 */ } /* 105 */ /* 106 */ } /* 107 */ range_nextIndex_0 = range_batchEnd_0; /* 108 */ ((org.apache.spark.sql.execution.metric.SQLMetric) references[0] /* numOutputRows */).add(range_localEnd_0); /* 109 */ range_inputMetrics_0.incRecordsRead(range_localEnd_0); /* 110 */ range_taskContext_0.killTaskIfInterrupted(); /* 111 */ } /* 112 */ } /* 113 */ /* 114 */ } 20/08/08 15:33:48.634 Thread-4 DEBUG CodeGenerator: /* 001 */ public Object generate(Object[] references) { /* 002 */ return new GeneratedIteratorForCodegenStage1(references); /* 003 */ } /* 004 */ /* 005 */ // codegenStageId=1 /* 006 */ final class GeneratedIteratorForCodegenStage1 extends org.apache.spark.sql.execution.BufferedRowIterator { /* 007 */ private Object[] references; /* 008 */ private scala.collection.Iterator[] inputs; /* 009 */ private scala.collection.Iterator rdd_input_0; /* 010 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[] rdd_mutableStateArray_0 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[2]; /* 011 */ /* 012 */ public GeneratedIteratorForCodegenStage1(Object[] references) { /* 013 */ this.references = references; /* 014 */ } /* 015 */ /* 016 */ public void init(int index, scala.collection.Iterator[] inputs) { /* 017 */ partitionIndex = index; /* 018 */ this.inputs = inputs; /* 019 */ rdd_input_0 = inputs[0]; /* 020 */ rdd_mutableStateArray_0[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(3, 0); /* 021 */ rdd_mutableStateArray_0[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(3, 0); /* 022 */ /* 023 */ } /* 024 */ /* 025 */ protected void processNext() throws java.io.IOException { /* 026 */ while ( rdd_input_0.hasNext()) { /* 027 */ InternalRow rdd_row_0 = (InternalRow) rdd_input_0.next(); /* 028 */ ((org.apache.spark.sql.execution.metric.SQLMetric) references[0] /* numOutputRows */).add(1); /* 029 */ boolean rdd_isNull_0 = rdd_row_0.isNullAt(0); /* 030 */ long rdd_value_0 = rdd_isNull_0 ? /* 031 */ -1L : (rdd_row_0.getLong(0)); /* 032 */ Object project_arg_0 = rdd_isNull_0 ? null : ((scala.Function1[]) references[1] /* converters */)[0].apply(rdd_value_0); /* 033 */ /* 034 */ Integer project_result_0 = null; /* 035 */ try { /* 036 */ project_result_0 = (Integer)((scala.Function1) references[3] /* udf */).apply(project_arg_0); /* 037 */ } catch (Exception e) { /* 038 */ throw new org.apache.spark.SparkException(((java.lang.String) references[2] /* errMsg */), e); /* 039 */ } /* 040 */ /* 041 */ boolean project_isNull_0 = project_result_0 == null; /* 042 */ int project_value_0 = -1; /* 043 */ if (!project_isNull_0) { /* 044 */ project_value_0 = project_result_0; /* 045 */ } /* 046 */ boolean rdd_isNull_1 = rdd_row_0.isNullAt(1); /* 047 */ long rdd_value_1 = rdd_isNull_1 ? /* 048 */ -1L : (rdd_row_0.getLong(1)); /* 049 */ Object project_arg_1 = rdd_isNull_1 ? null : ((scala.Function1[]) references[4] /* converters */)[0].apply(rdd_value_1); /* 050 */ /* 051 */ Integer project_result_1 = null; /* 052 */ try { /* 053 */ project_result_1 = (Integer)((scala.Function1) references[6] /* udf */).apply(project_arg_1); /* 054 */ } catch (Exception e) { /* 055 */ throw new org.apache.spark.SparkException(((java.lang.String) references[5] /* errMsg */), e); /* 056 */ } /* 057 */ /* 058 */ boolean project_isNull_2 = project_result_1 == null; /* 059 */ int project_value_2 = -1; /* 060 */ if (!project_isNull_2) { /* 061 */ project_value_2 = project_result_1; /* 062 */ } /* 063 */ boolean rdd_isNull_2 = rdd_row_0.isNullAt(2); /* 064 */ double rdd_value_2 = rdd_isNull_2 ? /* 065 */ -1.0 : (rdd_row_0.getDouble(2)); /* 066 */ boolean project_isNull_4 = rdd_isNull_2; /* 067 */ float project_value_4 = -1.0f; /* 068 */ if (!rdd_isNull_2) { /* 069 */ project_value_4 = (float) rdd_value_2; /* 070 */ } /* 071 */ rdd_mutableStateArray_0[1].reset(); /* 072 */ /* 073 */ rdd_mutableStateArray_0[1].zeroOutNullBytes(); /* 074 */ /* 075 */ rdd_mutableStateArray_0[1].write(0, project_value_0); /* 076 */ /* 077 */ rdd_mutableStateArray_0[1].write(1, project_value_2); /* 078 */ /* 079 */ if (project_isNull_4) { /* 080 */ rdd_mutableStateArray_0[1].setNullAt(2); /* 081 */ } else { /* 082 */ rdd_mutableStateArray_0[1].write(2, project_value_4); /* 083 */ } /* 084 */ append((rdd_mutableStateArray_0[1].getRow())); /* 085 */ if (shouldStop()) return; /* 086 */ } /* 087 */ } /* 088 */ /* 089 */ } 20/08/08 15:33:48.691 Thread-4 INFO CodeGenerator: Code generated in 78.325348 ms 20/08/08 15:33:48.724 Thread-4 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$inputRDDs$1$adapted 20/08/08 15:33:48.759 Thread-4 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$inputRDDs$1$adapted) is now cleaned +++ 20/08/08 15:33:48.776 Thread-4 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$doExecute$4$adapted 20/08/08 15:33:48.781 Thread-4 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$doExecute$4$adapted) is now cleaned +++ 20/08/08 15:33:48.847 Thread-4 DEBUG WholeStageCodegenExec: /* 001 */ public Object generate(Object[] references) { /* 002 */ return new GeneratedIteratorForCodegenStage1(references); /* 003 */ } /* 004 */ /* 005 */ // codegenStageId=1 /* 006 */ final class GeneratedIteratorForCodegenStage1 extends org.apache.spark.sql.execution.BufferedRowIterator { /* 007 */ private Object[] references; /* 008 */ private scala.collection.Iterator[] inputs; /* 009 */ private scala.collection.Iterator rdd_input_0; /* 010 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[] rdd_mutableStateArray_0 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[4]; /* 011 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[] rdd_mutableStateArray_1 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[4]; /* 012 */ /* 013 */ public GeneratedIteratorForCodegenStage1(Object[] references) { /* 014 */ this.references = references; /* 015 */ } /* 016 */ /* 017 */ public void init(int index, scala.collection.Iterator[] inputs) { /* 018 */ partitionIndex = index; /* 019 */ this.inputs = inputs; /* 020 */ rdd_input_0 = inputs[0]; /* 021 */ rdd_mutableStateArray_0[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(2, 32); /* 022 */ rdd_mutableStateArray_0[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(rdd_mutableStateArray_0[0], 4); /* 023 */ rdd_mutableStateArray_1[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[1], 4); /* 024 */ rdd_mutableStateArray_1[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[1], 8); /* 025 */ rdd_mutableStateArray_0[2] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(1, 32); /* 026 */ rdd_mutableStateArray_0[3] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(rdd_mutableStateArray_0[2], 4); /* 027 */ rdd_mutableStateArray_1[2] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[3], 4); /* 028 */ rdd_mutableStateArray_1[3] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[3], 8); /* 029 */ /* 030 */ } /* 031 */ /* 032 */ private void project_writeFields_0_1(InternalRow project_tmpInput_0) { /* 033 */ if ((project_tmpInput_0.isNullAt(3))) { /* 034 */ rdd_mutableStateArray_0[3].setNullAt(3); /* 035 */ } else { /* 036 */ // Remember the current cursor so that we can calculate how many bytes are /* 037 */ // written later. /* 038 */ final int project_previousCursor_2 = rdd_mutableStateArray_0[3].cursor(); /* 039 */ /* 040 */ final ArrayData project_tmpInput_2 = (project_tmpInput_0.getArray(3)); /* 041 */ if (project_tmpInput_2 instanceof UnsafeArrayData) { /* 042 */ rdd_mutableStateArray_0[3].write((UnsafeArrayData) project_tmpInput_2); /* 043 */ } else { /* 044 */ final int project_numElements_1 = project_tmpInput_2.numElements(); /* 045 */ rdd_mutableStateArray_1[3].initialize(project_numElements_1); /* 046 */ /* 047 */ for (int project_index_1 = 0; project_index_1 < project_numElements_1; project_index_1++) { /* 048 */ rdd_mutableStateArray_1[3].write(project_index_1, project_tmpInput_2.getDouble(project_index_1)); /* 049 */ } /* 050 */ } /* 051 */ /* 052 */ rdd_mutableStateArray_0[3].setOffsetAndSizeFromPreviousCursor(3, project_previousCursor_2); /* 053 */ } /* 054 */ /* 055 */ } /* 056 */ /* 057 */ private void project_writeFields_0_0(InternalRow project_tmpInput_0) { /* 058 */ rdd_mutableStateArray_0[3].write(0, (project_tmpInput_0.getByte(0))); /* 059 */ /* 060 */ if ((project_tmpInput_0.isNullAt(1))) { /* 061 */ rdd_mutableStateArray_0[3].setNullAt(1); /* 062 */ } else { /* 063 */ rdd_mutableStateArray_0[3].write(1, (project_tmpInput_0.getInt(1))); /* 064 */ } /* 065 */ /* 066 */ if ((project_tmpInput_0.isNullAt(2))) { /* 067 */ rdd_mutableStateArray_0[3].setNullAt(2); /* 068 */ } else { /* 069 */ // Remember the current cursor so that we can calculate how many bytes are /* 070 */ // written later. /* 071 */ final int project_previousCursor_1 = rdd_mutableStateArray_0[3].cursor(); /* 072 */ /* 073 */ final ArrayData project_tmpInput_1 = (project_tmpInput_0.getArray(2)); /* 074 */ if (project_tmpInput_1 instanceof UnsafeArrayData) { /* 075 */ rdd_mutableStateArray_0[3].write((UnsafeArrayData) project_tmpInput_1); /* 076 */ } else { /* 077 */ final int project_numElements_0 = project_tmpInput_1.numElements(); /* 078 */ rdd_mutableStateArray_1[2].initialize(project_numElements_0); /* 079 */ /* 080 */ for (int project_index_0 = 0; project_index_0 < project_numElements_0; project_index_0++) { /* 081 */ rdd_mutableStateArray_1[2].write(project_index_0, project_tmpInput_1.getInt(project_index_0)); /* 082 */ } /* 083 */ } /* 084 */ /* 085 */ rdd_mutableStateArray_0[3].setOffsetAndSizeFromPreviousCursor(2, project_previousCursor_1); /* 086 */ } /* 087 */ /* 088 */ } /* 089 */ /* 090 */ protected void processNext() throws java.io.IOException { /* 091 */ while ( rdd_input_0.hasNext()) { /* 092 */ InternalRow rdd_row_0 = (InternalRow) rdd_input_0.next(); /* 093 */ ((org.apache.spark.sql.execution.metric.SQLMetric) references[0] /* numOutputRows */).add(1); /* 094 */ boolean rdd_isNull_0 = rdd_row_0.isNullAt(0); /* 095 */ InternalRow rdd_value_0 = rdd_isNull_0 ? /* 096 */ null : (rdd_row_0.getStruct(0, 4)); /* 097 */ rdd_mutableStateArray_0[2].reset(); /* 098 */ /* 099 */ rdd_mutableStateArray_0[2].zeroOutNullBytes(); /* 100 */ /* 101 */ if (rdd_isNull_0) { /* 102 */ rdd_mutableStateArray_0[2].setNullAt(0); /* 103 */ } else { /* 104 */ final InternalRow project_tmpInput_0 = rdd_value_0; /* 105 */ if (project_tmpInput_0 instanceof UnsafeRow) { /* 106 */ rdd_mutableStateArray_0[2].write(0, (UnsafeRow) project_tmpInput_0); /* 107 */ } else { /* 108 */ // Remember the current cursor so that we can calculate how many bytes are /* 109 */ // written later. /* 110 */ final int project_previousCursor_0 = rdd_mutableStateArray_0[2].cursor(); /* 111 */ /* 112 */ rdd_mutableStateArray_0[3].resetRowWriter(); /* 113 */ project_writeFields_0_0(project_tmpInput_0); /* 114 */ project_writeFields_0_1(project_tmpInput_0); /* 115 */ /* 116 */ rdd_mutableStateArray_0[2].setOffsetAndSizeFromPreviousCursor(0, project_previousCursor_0); /* 117 */ } /* 118 */ } /* 119 */ append((rdd_mutableStateArray_0[2].getRow())); /* 120 */ if (shouldStop()) return; /* 121 */ } /* 122 */ } /* 123 */ /* 124 */ } 20/08/08 15:33:48.849 Thread-4 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$doExecute$1 20/08/08 15:33:48.851 Thread-4 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$doExecute$1) is now cleaned +++ 20/08/08 15:33:48.859 Thread-4 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$doExecute$2 20/08/08 15:33:48.860 Thread-4 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$doExecute$2) is now cleaned +++ 20/08/08 15:33:48.929 Thread-4 DEBUG WholeStageCodegenExec: /* 001 */ public Object generate(Object[] references) { /* 002 */ return new GeneratedIteratorForCodegenStage1(references); /* 003 */ } /* 004 */ /* 005 */ // codegenStageId=1 /* 006 */ final class GeneratedIteratorForCodegenStage1 extends org.apache.spark.sql.execution.BufferedRowIterator { /* 007 */ private Object[] references; /* 008 */ private scala.collection.Iterator[] inputs; /* 009 */ private scala.collection.Iterator rdd_input_0; /* 010 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[] rdd_mutableStateArray_0 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[5]; /* 011 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[] rdd_mutableStateArray_1 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[7]; /* 012 */ /* 013 */ public GeneratedIteratorForCodegenStage1(Object[] references) { /* 014 */ this.references = references; /* 015 */ } /* 016 */ /* 017 */ public void init(int index, scala.collection.Iterator[] inputs) { /* 018 */ partitionIndex = index; /* 019 */ this.inputs = inputs; /* 020 */ wholestagecodegen_init_0_0(); /* 021 */ wholestagecodegen_init_0_1(); /* 022 */ /* 023 */ } /* 024 */ /* 025 */ private void project_writeFields_1_1(InternalRow project_tmpInput_3) { /* 026 */ if ((project_tmpInput_3.isNullAt(3))) { /* 027 */ rdd_mutableStateArray_0[4].setNullAt(3); /* 028 */ } else { /* 029 */ // Remember the current cursor so that we can calculate how many bytes are /* 030 */ // written later. /* 031 */ final int project_previousCursor_5 = rdd_mutableStateArray_0[4].cursor(); /* 032 */ /* 033 */ final ArrayData project_tmpInput_5 = (project_tmpInput_3.getArray(3)); /* 034 */ if (project_tmpInput_5 instanceof UnsafeArrayData) { /* 035 */ rdd_mutableStateArray_0[4].write((UnsafeArrayData) project_tmpInput_5); /* 036 */ } else { /* 037 */ final int project_numElements_3 = project_tmpInput_5.numElements(); /* 038 */ rdd_mutableStateArray_1[6].initialize(project_numElements_3); /* 039 */ /* 040 */ for (int project_index_3 = 0; project_index_3 < project_numElements_3; project_index_3++) { /* 041 */ rdd_mutableStateArray_1[6].write(project_index_3, project_tmpInput_5.getDouble(project_index_3)); /* 042 */ } /* 043 */ } /* 044 */ /* 045 */ rdd_mutableStateArray_0[4].setOffsetAndSizeFromPreviousCursor(3, project_previousCursor_5); /* 046 */ } /* 047 */ /* 048 */ } /* 049 */ /* 050 */ private void project_writeFields_1_0(InternalRow project_tmpInput_3) { /* 051 */ rdd_mutableStateArray_0[4].write(0, (project_tmpInput_3.getByte(0))); /* 052 */ /* 053 */ if ((project_tmpInput_3.isNullAt(1))) { /* 054 */ rdd_mutableStateArray_0[4].setNullAt(1); /* 055 */ } else { /* 056 */ rdd_mutableStateArray_0[4].write(1, (project_tmpInput_3.getInt(1))); /* 057 */ } /* 058 */ /* 059 */ if ((project_tmpInput_3.isNullAt(2))) { /* 060 */ rdd_mutableStateArray_0[4].setNullAt(2); /* 061 */ } else { /* 062 */ // Remember the current cursor so that we can calculate how many bytes are /* 063 */ // written later. /* 064 */ final int project_previousCursor_4 = rdd_mutableStateArray_0[4].cursor(); /* 065 */ /* 066 */ final ArrayData project_tmpInput_4 = (project_tmpInput_3.getArray(2)); /* 067 */ if (project_tmpInput_4 instanceof UnsafeArrayData) { /* 068 */ rdd_mutableStateArray_0[4].write((UnsafeArrayData) project_tmpInput_4); /* 069 */ } else { /* 070 */ final int project_numElements_2 = project_tmpInput_4.numElements(); /* 071 */ rdd_mutableStateArray_1[5].initialize(project_numElements_2); /* 072 */ /* 073 */ for (int project_index_2 = 0; project_index_2 < project_numElements_2; project_index_2++) { /* 074 */ rdd_mutableStateArray_1[5].write(project_index_2, project_tmpInput_4.getInt(project_index_2)); /* 075 */ } /* 076 */ } /* 077 */ /* 078 */ rdd_mutableStateArray_0[4].setOffsetAndSizeFromPreviousCursor(2, project_previousCursor_4); /* 079 */ } /* 080 */ /* 081 */ } /* 082 */ /* 083 */ private void project_writeFields_0_1(InternalRow project_tmpInput_0) { /* 084 */ if ((project_tmpInput_0.isNullAt(3))) { /* 085 */ rdd_mutableStateArray_0[3].setNullAt(3); /* 086 */ } else { /* 087 */ // Remember the current cursor so that we can calculate how many bytes are /* 088 */ // written later. /* 089 */ final int project_previousCursor_2 = rdd_mutableStateArray_0[3].cursor(); /* 090 */ /* 091 */ final ArrayData project_tmpInput_2 = (project_tmpInput_0.getArray(3)); /* 092 */ if (project_tmpInput_2 instanceof UnsafeArrayData) { /* 093 */ rdd_mutableStateArray_0[3].write((UnsafeArrayData) project_tmpInput_2); /* 094 */ } else { /* 095 */ final int project_numElements_1 = project_tmpInput_2.numElements(); /* 096 */ rdd_mutableStateArray_1[4].initialize(project_numElements_1); /* 097 */ /* 098 */ for (int project_index_1 = 0; project_index_1 < project_numElements_1; project_index_1++) { /* 099 */ rdd_mutableStateArray_1[4].write(project_index_1, project_tmpInput_2.getDouble(project_index_1)); /* 100 */ } /* 101 */ } /* 102 */ /* 103 */ rdd_mutableStateArray_0[3].setOffsetAndSizeFromPreviousCursor(3, project_previousCursor_2); /* 104 */ } /* 105 */ /* 106 */ } /* 107 */ /* 108 */ private void wholestagecodegen_init_0_1() { /* 109 */ rdd_mutableStateArray_1[4] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[3], 8); /* 110 */ rdd_mutableStateArray_0[4] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(rdd_mutableStateArray_0[2], 4); /* 111 */ rdd_mutableStateArray_1[5] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[4], 4); /* 112 */ rdd_mutableStateArray_1[6] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[4], 8); /* 113 */ /* 114 */ } /* 115 */ /* 116 */ private void project_writeFields_0_0(InternalRow project_tmpInput_0) { /* 117 */ rdd_mutableStateArray_0[3].write(0, (project_tmpInput_0.getByte(0))); /* 118 */ /* 119 */ if ((project_tmpInput_0.isNullAt(1))) { /* 120 */ rdd_mutableStateArray_0[3].setNullAt(1); /* 121 */ } else { /* 122 */ rdd_mutableStateArray_0[3].write(1, (project_tmpInput_0.getInt(1))); /* 123 */ } /* 124 */ /* 125 */ if ((project_tmpInput_0.isNullAt(2))) { /* 126 */ rdd_mutableStateArray_0[3].setNullAt(2); /* 127 */ } else { /* 128 */ // Remember the current cursor so that we can calculate how many bytes are /* 129 */ // written later. /* 130 */ final int project_previousCursor_1 = rdd_mutableStateArray_0[3].cursor(); /* 131 */ /* 132 */ final ArrayData project_tmpInput_1 = (project_tmpInput_0.getArray(2)); /* 133 */ if (project_tmpInput_1 instanceof UnsafeArrayData) { /* 134 */ rdd_mutableStateArray_0[3].write((UnsafeArrayData) project_tmpInput_1); /* 135 */ } else { /* 136 */ final int project_numElements_0 = project_tmpInput_1.numElements(); /* 137 */ rdd_mutableStateArray_1[3].initialize(project_numElements_0); /* 138 */ /* 139 */ for (int project_index_0 = 0; project_index_0 < project_numElements_0; project_index_0++) { /* 140 */ rdd_mutableStateArray_1[3].write(project_index_0, project_tmpInput_1.getInt(project_index_0)); /* 141 */ } /* 142 */ } /* 143 */ /* 144 */ rdd_mutableStateArray_0[3].setOffsetAndSizeFromPreviousCursor(2, project_previousCursor_1); /* 145 */ } /* 146 */ /* 147 */ } /* 148 */ /* 149 */ protected void processNext() throws java.io.IOException { /* 150 */ while ( rdd_input_0.hasNext()) { /* 151 */ InternalRow rdd_row_0 = (InternalRow) rdd_input_0.next(); /* 152 */ ((org.apache.spark.sql.execution.metric.SQLMetric) references[0] /* numOutputRows */).add(1); /* 153 */ boolean rdd_isNull_1 = rdd_row_0.isNullAt(1); /* 154 */ ArrayData rdd_value_1 = rdd_isNull_1 ? /* 155 */ null : (rdd_row_0.getArray(1)); /* 156 */ /* 157 */ Object project_arg_0 = null; /* 158 */ if (rdd_isNull_1) { /* 159 */ project_arg_0 = ((scala.Function1[]) references[1] /* converters */)[0].apply(null); /* 160 */ } else { /* 161 */ project_arg_0 = ((scala.Function1[]) references[1] /* converters */)[0].apply(rdd_value_1); /* 162 */ } /* 163 */ /* 164 */ InternalRow project_result_0 = null; /* 165 */ try { /* 166 */ project_result_0 = (InternalRow)((scala.Function1[]) references[1] /* converters */)[1].apply(((scala.Function1) references[3] /* udf */).apply(project_arg_0)); /* 167 */ } catch (Exception e) { /* 168 */ throw new org.apache.spark.SparkException(((java.lang.String) references[2] /* errMsg */), e); /* 169 */ } /* 170 */ /* 171 */ boolean project_isNull_0 = project_result_0 == null; /* 172 */ InternalRow project_value_0 = null; /* 173 */ if (!project_isNull_0) { /* 174 */ project_value_0 = project_result_0; /* 175 */ } /* 176 */ boolean rdd_isNull_2 = rdd_row_0.isNullAt(2); /* 177 */ InternalRow rdd_value_2 = rdd_isNull_2 ? /* 178 */ null : (rdd_row_0.getStruct(2, 4)); /* 179 */ rdd_mutableStateArray_0[2].reset(); /* 180 */ /* 181 */ rdd_mutableStateArray_0[2].zeroOutNullBytes(); /* 182 */ /* 183 */ if (project_isNull_0) { /* 184 */ rdd_mutableStateArray_0[2].setNullAt(0); /* 185 */ } else { /* 186 */ final InternalRow project_tmpInput_0 = project_value_0; /* 187 */ if (project_tmpInput_0 instanceof UnsafeRow) { /* 188 */ rdd_mutableStateArray_0[2].write(0, (UnsafeRow) project_tmpInput_0); /* 189 */ } else { /* 190 */ // Remember the current cursor so that we can calculate how many bytes are /* 191 */ // written later. /* 192 */ final int project_previousCursor_0 = rdd_mutableStateArray_0[2].cursor(); /* 193 */ /* 194 */ rdd_mutableStateArray_0[3].resetRowWriter(); /* 195 */ project_writeFields_0_0(project_tmpInput_0); /* 196 */ project_writeFields_0_1(project_tmpInput_0); /* 197 */ /* 198 */ rdd_mutableStateArray_0[2].setOffsetAndSizeFromPreviousCursor(0, project_previousCursor_0); /* 199 */ } /* 200 */ } /* 201 */ /* 202 */ if (rdd_isNull_2) { /* 203 */ rdd_mutableStateArray_0[2].setNullAt(1); /* 204 */ } else { /* 205 */ final InternalRow project_tmpInput_3 = rdd_value_2; /* 206 */ if (project_tmpInput_3 instanceof UnsafeRow) { /* 207 */ rdd_mutableStateArray_0[2].write(1, (UnsafeRow) project_tmpInput_3); /* 208 */ } else { /* 209 */ // Remember the current cursor so that we can calculate how many bytes are /* 210 */ // written later. /* 211 */ final int project_previousCursor_3 = rdd_mutableStateArray_0[2].cursor(); /* 212 */ /* 213 */ rdd_mutableStateArray_0[4].resetRowWriter(); /* 214 */ project_writeFields_1_0(project_tmpInput_3); /* 215 */ project_writeFields_1_1(project_tmpInput_3); /* 216 */ /* 217 */ rdd_mutableStateArray_0[2].setOffsetAndSizeFromPreviousCursor(1, project_previousCursor_3); /* 218 */ } /* 219 */ } /* 220 */ append((rdd_mutableStateArray_0[2].getRow())); /* 221 */ if (shouldStop()) return; /* 222 */ } /* 223 */ } /* 224 */ /* 225 */ private void wholestagecodegen_init_0_0() { /* 226 */ rdd_input_0 = inputs[0]; /* 227 */ rdd_mutableStateArray_0[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(3, 64); /* 228 */ rdd_mutableStateArray_1[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[0], 8); /* 229 */ rdd_mutableStateArray_0[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(rdd_mutableStateArray_0[0], 4); /* 230 */ rdd_mutableStateArray_1[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[1], 4); /* 231 */ rdd_mutableStateArray_1[2] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[1], 8); /* 232 */ rdd_mutableStateArray_0[2] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(2, 64); /* 233 */ rdd_mutableStateArray_0[3] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(rdd_mutableStateArray_0[2], 4); /* 234 */ rdd_mutableStateArray_1[3] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[3], 4); /* 235 */ /* 236 */ } /* 237 */ /* 238 */ } 20/08/08 15:33:48.930 Thread-4 DEBUG CodeGenerator: /* 001 */ public Object generate(Object[] references) { /* 002 */ return new GeneratedIteratorForCodegenStage1(references); /* 003 */ } /* 004 */ /* 005 */ // codegenStageId=1 /* 006 */ final class GeneratedIteratorForCodegenStage1 extends org.apache.spark.sql.execution.BufferedRowIterator { /* 007 */ private Object[] references; /* 008 */ private scala.collection.Iterator[] inputs; /* 009 */ private scala.collection.Iterator rdd_input_0; /* 010 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[] rdd_mutableStateArray_0 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[4]; /* 011 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[] rdd_mutableStateArray_1 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[4]; /* 012 */ /* 013 */ public GeneratedIteratorForCodegenStage1(Object[] references) { /* 014 */ this.references = references; /* 015 */ } /* 016 */ /* 017 */ public void init(int index, scala.collection.Iterator[] inputs) { /* 018 */ partitionIndex = index; /* 019 */ this.inputs = inputs; /* 020 */ rdd_input_0 = inputs[0]; /* 021 */ rdd_mutableStateArray_0[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(2, 32); /* 022 */ rdd_mutableStateArray_0[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(rdd_mutableStateArray_0[0], 4); /* 023 */ rdd_mutableStateArray_1[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[1], 4); /* 024 */ rdd_mutableStateArray_1[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[1], 8); /* 025 */ rdd_mutableStateArray_0[2] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(1, 32); /* 026 */ rdd_mutableStateArray_0[3] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(rdd_mutableStateArray_0[2], 4); /* 027 */ rdd_mutableStateArray_1[2] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[3], 4); /* 028 */ rdd_mutableStateArray_1[3] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[3], 8); /* 029 */ /* 030 */ } /* 031 */ /* 032 */ private void project_writeFields_0_1(InternalRow project_tmpInput_0) { /* 033 */ if ((project_tmpInput_0.isNullAt(3))) { /* 034 */ rdd_mutableStateArray_0[3].setNullAt(3); /* 035 */ } else { /* 036 */ // Remember the current cursor so that we can calculate how many bytes are /* 037 */ // written later. /* 038 */ final int project_previousCursor_2 = rdd_mutableStateArray_0[3].cursor(); /* 039 */ /* 040 */ final ArrayData project_tmpInput_2 = (project_tmpInput_0.getArray(3)); /* 041 */ if (project_tmpInput_2 instanceof UnsafeArrayData) { /* 042 */ rdd_mutableStateArray_0[3].write((UnsafeArrayData) project_tmpInput_2); /* 043 */ } else { /* 044 */ final int project_numElements_1 = project_tmpInput_2.numElements(); /* 045 */ rdd_mutableStateArray_1[3].initialize(project_numElements_1); /* 046 */ /* 047 */ for (int project_index_1 = 0; project_index_1 < project_numElements_1; project_index_1++) { /* 048 */ rdd_mutableStateArray_1[3].write(project_index_1, project_tmpInput_2.getDouble(project_index_1)); /* 049 */ } /* 050 */ } /* 051 */ /* 052 */ rdd_mutableStateArray_0[3].setOffsetAndSizeFromPreviousCursor(3, project_previousCursor_2); /* 053 */ } /* 054 */ /* 055 */ } /* 056 */ /* 057 */ private void project_writeFields_0_0(InternalRow project_tmpInput_0) { /* 058 */ rdd_mutableStateArray_0[3].write(0, (project_tmpInput_0.getByte(0))); /* 059 */ /* 060 */ if ((project_tmpInput_0.isNullAt(1))) { /* 061 */ rdd_mutableStateArray_0[3].setNullAt(1); /* 062 */ } else { /* 063 */ rdd_mutableStateArray_0[3].write(1, (project_tmpInput_0.getInt(1))); /* 064 */ } /* 065 */ /* 066 */ if ((project_tmpInput_0.isNullAt(2))) { /* 067 */ rdd_mutableStateArray_0[3].setNullAt(2); /* 068 */ } else { /* 069 */ // Remember the current cursor so that we can calculate how many bytes are /* 070 */ // written later. /* 071 */ final int project_previousCursor_1 = rdd_mutableStateArray_0[3].cursor(); /* 072 */ /* 073 */ final ArrayData project_tmpInput_1 = (project_tmpInput_0.getArray(2)); /* 074 */ if (project_tmpInput_1 instanceof UnsafeArrayData) { /* 075 */ rdd_mutableStateArray_0[3].write((UnsafeArrayData) project_tmpInput_1); /* 076 */ } else { /* 077 */ final int project_numElements_0 = project_tmpInput_1.numElements(); /* 078 */ rdd_mutableStateArray_1[2].initialize(project_numElements_0); /* 079 */ /* 080 */ for (int project_index_0 = 0; project_index_0 < project_numElements_0; project_index_0++) { /* 081 */ rdd_mutableStateArray_1[2].write(project_index_0, project_tmpInput_1.getInt(project_index_0)); /* 082 */ } /* 083 */ } /* 084 */ /* 085 */ rdd_mutableStateArray_0[3].setOffsetAndSizeFromPreviousCursor(2, project_previousCursor_1); /* 086 */ } /* 087 */ /* 088 */ } /* 089 */ /* 090 */ protected void processNext() throws java.io.IOException { /* 091 */ while ( rdd_input_0.hasNext()) { /* 092 */ InternalRow rdd_row_0 = (InternalRow) rdd_input_0.next(); /* 093 */ ((org.apache.spark.sql.execution.metric.SQLMetric) references[0] /* numOutputRows */).add(1); /* 094 */ boolean rdd_isNull_0 = rdd_row_0.isNullAt(0); /* 095 */ InternalRow rdd_value_0 = rdd_isNull_0 ? /* 096 */ null : (rdd_row_0.getStruct(0, 4)); /* 097 */ rdd_mutableStateArray_0[2].reset(); /* 098 */ /* 099 */ rdd_mutableStateArray_0[2].zeroOutNullBytes(); /* 100 */ /* 101 */ if (rdd_isNull_0) { /* 102 */ rdd_mutableStateArray_0[2].setNullAt(0); /* 103 */ } else { /* 104 */ final InternalRow project_tmpInput_0 = rdd_value_0; /* 105 */ if (project_tmpInput_0 instanceof UnsafeRow) { /* 106 */ rdd_mutableStateArray_0[2].write(0, (UnsafeRow) project_tmpInput_0); /* 107 */ } else { /* 108 */ // Remember the current cursor so that we can calculate how many bytes are /* 109 */ // written later. /* 110 */ final int project_previousCursor_0 = rdd_mutableStateArray_0[2].cursor(); /* 111 */ /* 112 */ rdd_mutableStateArray_0[3].resetRowWriter(); /* 113 */ project_writeFields_0_0(project_tmpInput_0); /* 114 */ project_writeFields_0_1(project_tmpInput_0); /* 115 */ /* 116 */ rdd_mutableStateArray_0[2].setOffsetAndSizeFromPreviousCursor(0, project_previousCursor_0); /* 117 */ } /* 118 */ } /* 119 */ append((rdd_mutableStateArray_0[2].getRow())); /* 120 */ if (shouldStop()) return; /* 121 */ } /* 122 */ } /* 123 */ /* 124 */ } 20/08/08 15:33:48.937 Thread-4 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$doExecute$4$adapted 20/08/08 15:33:48.940 Thread-4 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$doExecute$4$adapted) is now cleaned +++ 20/08/08 15:33:48.955 Thread-4 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$collect$2 20/08/08 15:33:48.965 Thread-4 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$collect$2) is now cleaned +++ 20/08/08 15:33:48.988 Thread-4 DEBUG CodeGenerator: /* 001 */ public Object generate(Object[] references) { /* 002 */ return new GeneratedIteratorForCodegenStage1(references); /* 003 */ } /* 004 */ /* 005 */ // codegenStageId=1 /* 006 */ final class GeneratedIteratorForCodegenStage1 extends org.apache.spark.sql.execution.BufferedRowIterator { /* 007 */ private Object[] references; /* 008 */ private scala.collection.Iterator[] inputs; /* 009 */ private scala.collection.Iterator rdd_input_0; /* 010 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[] rdd_mutableStateArray_0 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[5]; /* 011 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[] rdd_mutableStateArray_1 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[7]; /* 012 */ /* 013 */ public GeneratedIteratorForCodegenStage1(Object[] references) { /* 014 */ this.references = references; /* 015 */ } /* 016 */ /* 017 */ public void init(int index, scala.collection.Iterator[] inputs) { /* 018 */ partitionIndex = index; /* 019 */ this.inputs = inputs; /* 020 */ wholestagecodegen_init_0_0(); /* 021 */ wholestagecodegen_init_0_1(); /* 022 */ /* 023 */ } /* 024 */ /* 025 */ private void project_writeFields_1_1(InternalRow project_tmpInput_3) { /* 026 */ if ((project_tmpInput_3.isNullAt(3))) { /* 027 */ rdd_mutableStateArray_0[4].setNullAt(3); /* 028 */ } else { /* 029 */ // Remember the current cursor so that we can calculate how many bytes are /* 030 */ // written later. /* 031 */ final int project_previousCursor_5 = rdd_mutableStateArray_0[4].cursor(); /* 032 */ /* 033 */ final ArrayData project_tmpInput_5 = (project_tmpInput_3.getArray(3)); /* 034 */ if (project_tmpInput_5 instanceof UnsafeArrayData) { /* 035 */ rdd_mutableStateArray_0[4].write((UnsafeArrayData) project_tmpInput_5); /* 036 */ } else { /* 037 */ final int project_numElements_3 = project_tmpInput_5.numElements(); /* 038 */ rdd_mutableStateArray_1[6].initialize(project_numElements_3); /* 039 */ /* 040 */ for (int project_index_3 = 0; project_index_3 < project_numElements_3; project_index_3++) { /* 041 */ rdd_mutableStateArray_1[6].write(project_index_3, project_tmpInput_5.getDouble(project_index_3)); /* 042 */ } /* 043 */ } /* 044 */ /* 045 */ rdd_mutableStateArray_0[4].setOffsetAndSizeFromPreviousCursor(3, project_previousCursor_5); /* 046 */ } /* 047 */ /* 048 */ } /* 049 */ /* 050 */ private void project_writeFields_1_0(InternalRow project_tmpInput_3) { /* 051 */ rdd_mutableStateArray_0[4].write(0, (project_tmpInput_3.getByte(0))); /* 052 */ /* 053 */ if ((project_tmpInput_3.isNullAt(1))) { /* 054 */ rdd_mutableStateArray_0[4].setNullAt(1); /* 055 */ } else { /* 056 */ rdd_mutableStateArray_0[4].write(1, (project_tmpInput_3.getInt(1))); /* 057 */ } /* 058 */ /* 059 */ if ((project_tmpInput_3.isNullAt(2))) { /* 060 */ rdd_mutableStateArray_0[4].setNullAt(2); /* 061 */ } else { /* 062 */ // Remember the current cursor so that we can calculate how many bytes are /* 063 */ // written later. /* 064 */ final int project_previousCursor_4 = rdd_mutableStateArray_0[4].cursor(); /* 065 */ /* 066 */ final ArrayData project_tmpInput_4 = (project_tmpInput_3.getArray(2)); /* 067 */ if (project_tmpInput_4 instanceof UnsafeArrayData) { /* 068 */ rdd_mutableStateArray_0[4].write((UnsafeArrayData) project_tmpInput_4); /* 069 */ } else { /* 070 */ final int project_numElements_2 = project_tmpInput_4.numElements(); /* 071 */ rdd_mutableStateArray_1[5].initialize(project_numElements_2); /* 072 */ /* 073 */ for (int project_index_2 = 0; project_index_2 < project_numElements_2; project_index_2++) { /* 074 */ rdd_mutableStateArray_1[5].write(project_index_2, project_tmpInput_4.getInt(project_index_2)); /* 075 */ } /* 076 */ } /* 077 */ /* 078 */ rdd_mutableStateArray_0[4].setOffsetAndSizeFromPreviousCursor(2, project_previousCursor_4); /* 079 */ } /* 080 */ /* 081 */ } /* 082 */ /* 083 */ private void project_writeFields_0_1(InternalRow project_tmpInput_0) { /* 084 */ if ((project_tmpInput_0.isNullAt(3))) { /* 085 */ rdd_mutableStateArray_0[3].setNullAt(3); /* 086 */ } else { /* 087 */ // Remember the current cursor so that we can calculate how many bytes are /* 088 */ // written later. /* 089 */ final int project_previousCursor_2 = rdd_mutableStateArray_0[3].cursor(); /* 090 */ /* 091 */ final ArrayData project_tmpInput_2 = (project_tmpInput_0.getArray(3)); /* 092 */ if (project_tmpInput_2 instanceof UnsafeArrayData) { /* 093 */ rdd_mutableStateArray_0[3].write((UnsafeArrayData) project_tmpInput_2); /* 094 */ } else { /* 095 */ final int project_numElements_1 = project_tmpInput_2.numElements(); /* 096 */ rdd_mutableStateArray_1[4].initialize(project_numElements_1); /* 097 */ /* 098 */ for (int project_index_1 = 0; project_index_1 < project_numElements_1; project_index_1++) { /* 099 */ rdd_mutableStateArray_1[4].write(project_index_1, project_tmpInput_2.getDouble(project_index_1)); /* 100 */ } /* 101 */ } /* 102 */ /* 103 */ rdd_mutableStateArray_0[3].setOffsetAndSizeFromPreviousCursor(3, project_previousCursor_2); /* 104 */ } /* 105 */ /* 106 */ } /* 107 */ /* 108 */ private void wholestagecodegen_init_0_1() { /* 109 */ rdd_mutableStateArray_1[4] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[3], 8); /* 110 */ rdd_mutableStateArray_0[4] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(rdd_mutableStateArray_0[2], 4); /* 111 */ rdd_mutableStateArray_1[5] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[4], 4); /* 112 */ rdd_mutableStateArray_1[6] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[4], 8); /* 113 */ /* 114 */ } /* 115 */ /* 116 */ private void project_writeFields_0_0(InternalRow project_tmpInput_0) { /* 117 */ rdd_mutableStateArray_0[3].write(0, (project_tmpInput_0.getByte(0))); /* 118 */ /* 119 */ if ((project_tmpInput_0.isNullAt(1))) { /* 120 */ rdd_mutableStateArray_0[3].setNullAt(1); /* 121 */ } else { /* 122 */ rdd_mutableStateArray_0[3].write(1, (project_tmpInput_0.getInt(1))); /* 123 */ } /* 124 */ /* 125 */ if ((project_tmpInput_0.isNullAt(2))) { /* 126 */ rdd_mutableStateArray_0[3].setNullAt(2); /* 127 */ } else { /* 128 */ // Remember the current cursor so that we can calculate how many bytes are /* 129 */ // written later. /* 130 */ final int project_previousCursor_1 = rdd_mutableStateArray_0[3].cursor(); /* 131 */ /* 132 */ final ArrayData project_tmpInput_1 = (project_tmpInput_0.getArray(2)); /* 133 */ if (project_tmpInput_1 instanceof UnsafeArrayData) { /* 134 */ rdd_mutableStateArray_0[3].write((UnsafeArrayData) project_tmpInput_1); /* 135 */ } else { /* 136 */ final int project_numElements_0 = project_tmpInput_1.numElements(); /* 137 */ rdd_mutableStateArray_1[3].initialize(project_numElements_0); /* 138 */ /* 139 */ for (int project_index_0 = 0; project_index_0 < project_numElements_0; project_index_0++) { /* 140 */ rdd_mutableStateArray_1[3].write(project_index_0, project_tmpInput_1.getInt(project_index_0)); /* 141 */ } /* 142 */ } /* 143 */ /* 144 */ rdd_mutableStateArray_0[3].setOffsetAndSizeFromPreviousCursor(2, project_previousCursor_1); /* 145 */ } /* 146 */ /* 147 */ } /* 148 */ /* 149 */ protected void processNext() throws java.io.IOException { /* 150 */ while ( rdd_input_0.hasNext()) { /* 151 */ InternalRow rdd_row_0 = (InternalRow) rdd_input_0.next(); /* 152 */ ((org.apache.spark.sql.execution.metric.SQLMetric) references[0] /* numOutputRows */).add(1); /* 153 */ boolean rdd_isNull_1 = rdd_row_0.isNullAt(1); /* 154 */ ArrayData rdd_value_1 = rdd_isNull_1 ? /* 155 */ null : (rdd_row_0.getArray(1)); /* 156 */ /* 157 */ Object project_arg_0 = null; /* 158 */ if (rdd_isNull_1) { /* 159 */ project_arg_0 = ((scala.Function1[]) references[1] /* converters */)[0].apply(null); /* 160 */ } else { /* 161 */ project_arg_0 = ((scala.Function1[]) references[1] /* converters */)[0].apply(rdd_value_1); /* 162 */ } /* 163 */ /* 164 */ InternalRow project_result_0 = null; /* 165 */ try { /* 166 */ project_result_0 = (InternalRow)((scala.Function1[]) references[1] /* converters */)[1].apply(((scala.Function1) references[3] /* udf */).apply(project_arg_0)); /* 167 */ } catch (Exception e) { /* 168 */ throw new org.apache.spark.SparkException(((java.lang.String) references[2] /* errMsg */), e); /* 169 */ } /* 170 */ /* 171 */ boolean project_isNull_0 = project_result_0 == null; /* 172 */ InternalRow project_value_0 = null; /* 173 */ if (!project_isNull_0) { /* 174 */ project_value_0 = project_result_0; /* 175 */ } /* 176 */ boolean rdd_isNull_2 = rdd_row_0.isNullAt(2); /* 177 */ InternalRow rdd_value_2 = rdd_isNull_2 ? /* 178 */ null : (rdd_row_0.getStruct(2, 4)); /* 179 */ rdd_mutableStateArray_0[2].reset(); /* 180 */ /* 181 */ rdd_mutableStateArray_0[2].zeroOutNullBytes(); /* 182 */ /* 183 */ if (project_isNull_0) { /* 184 */ rdd_mutableStateArray_0[2].setNullAt(0); /* 185 */ } else { /* 186 */ final InternalRow project_tmpInput_0 = project_value_0; /* 187 */ if (project_tmpInput_0 instanceof UnsafeRow) { /* 188 */ rdd_mutableStateArray_0[2].write(0, (UnsafeRow) project_tmpInput_0); /* 189 */ } else { /* 190 */ // Remember the current cursor so that we can calculate how many bytes are /* 191 */ // written later. /* 192 */ final int project_previousCursor_0 = rdd_mutableStateArray_0[2].cursor(); /* 193 */ /* 194 */ rdd_mutableStateArray_0[3].resetRowWriter(); /* 195 */ project_writeFields_0_0(project_tmpInput_0); /* 196 */ project_writeFields_0_1(project_tmpInput_0); /* 197 */ /* 198 */ rdd_mutableStateArray_0[2].setOffsetAndSizeFromPreviousCursor(0, project_previousCursor_0); /* 199 */ } /* 200 */ } /* 201 */ /* 202 */ if (rdd_isNull_2) { /* 203 */ rdd_mutableStateArray_0[2].setNullAt(1); /* 204 */ } else { /* 205 */ final InternalRow project_tmpInput_3 = rdd_value_2; /* 206 */ if (project_tmpInput_3 instanceof UnsafeRow) { /* 207 */ rdd_mutableStateArray_0[2].write(1, (UnsafeRow) project_tmpInput_3); /* 208 */ } else { /* 209 */ // Remember the current cursor so that we can calculate how many bytes are /* 210 */ // written later. /* 211 */ final int project_previousCursor_3 = rdd_mutableStateArray_0[2].cursor(); /* 212 */ /* 213 */ rdd_mutableStateArray_0[4].resetRowWriter(); /* 214 */ project_writeFields_1_0(project_tmpInput_3); /* 215 */ project_writeFields_1_1(project_tmpInput_3); /* 216 */ /* 217 */ rdd_mutableStateArray_0[2].setOffsetAndSizeFromPreviousCursor(1, project_previousCursor_3); /* 218 */ } /* 219 */ } /* 220 */ append((rdd_mutableStateArray_0[2].getRow())); /* 221 */ if (shouldStop()) return; /* 222 */ } /* 223 */ } /* 224 */ /* 225 */ private void wholestagecodegen_init_0_0() { /* 226 */ rdd_input_0 = inputs[0]; /* 227 */ rdd_mutableStateArray_0[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(3, 64); /* 228 */ rdd_mutableStateArray_1[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[0], 8); /* 229 */ rdd_mutableStateArray_0[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(rdd_mutableStateArray_0[0], 4); /* 230 */ rdd_mutableStateArray_1[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[1], 4); /* 231 */ rdd_mutableStateArray_1[2] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[1], 8); /* 232 */ rdd_mutableStateArray_0[2] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(2, 64); /* 233 */ rdd_mutableStateArray_0[3] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(rdd_mutableStateArray_0[2], 4); /* 234 */ rdd_mutableStateArray_1[3] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[3], 4); /* 235 */ /* 236 */ } /* 237 */ /* 238 */ } 20/08/08 15:33:49.015 Thread-4 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$runJob$5 20/08/08 15:33:49.021 Thread-4 INFO CodeGenerator: Code generated in 442.092143 ms 20/08/08 15:33:49.021 Thread-4 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$runJob$5) is now cleaned +++ 20/08/08 15:33:49.023 Thread-4 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$doExecute$4$adapted 20/08/08 15:33:49.025 Thread-4 INFO SparkContext: Starting job: at /usr/lib/python3.6/runpy.py:85 20/08/08 15:33:49.026 Thread-4 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$doExecute$4$adapted) is now cleaned +++ 20/08/08 15:33:49.034 dag-scheduler-event-loop DEBUG DAGScheduler: Merging stage rdd profiles: Set() 20/08/08 15:33:49.061 dag-scheduler-event-loop INFO DAGScheduler: Got job 0 ( at /usr/lib/python3.6/runpy.py:85) with 4 output partitions 20/08/08 15:33:49.062 dag-scheduler-event-loop INFO DAGScheduler: Final stage: ResultStage 0 ( at /usr/lib/python3.6/runpy.py:85) 20/08/08 15:33:49.062 dag-scheduler-event-loop INFO DAGScheduler: Parents of final stage: List() 20/08/08 15:33:49.064 dag-scheduler-event-loop INFO DAGScheduler: Missing parents: List() 20/08/08 15:33:49.067 dag-scheduler-event-loop DEBUG DAGScheduler: submitStage(ResultStage 0 (name= at /usr/lib/python3.6/runpy.py:85;jobs=0)) 20/08/08 15:33:49.071 dag-scheduler-event-loop DEBUG DAGScheduler: missing: List() 20/08/08 15:33:49.073 dag-scheduler-event-loop INFO DAGScheduler: Submitting ResultStage 0 (MapPartitionsRDD[6] at at /usr/lib/python3.6/runpy.py:85), which has no missing parents 20/08/08 15:33:49.073 dag-scheduler-event-loop DEBUG DAGScheduler: submitMissingTasks(ResultStage 0) 20/08/08 15:33:49.165 Thread-4 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$rdd$1 20/08/08 15:33:49.167 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_0 stored as values in memory (estimated size 16.6 KiB, free 366.3 MiB) 20/08/08 15:33:49.169 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_0 locally took 45 ms 20/08/08 15:33:49.172 Thread-4 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$rdd$1) is now cleaned +++ 20/08/08 15:33:49.173 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_0 without replication took 49 ms 20/08/08 15:33:49.199 Thread-4 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$fit$2 20/08/08 15:33:49.201 Thread-4 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$fit$2) is now cleaned +++ 20/08/08 15:33:49.203 Thread-4 INFO Instrumentation: [6aa586b7] Stage class: ALS 20/08/08 15:33:49.204 Thread-4 INFO Instrumentation: [6aa586b7] Stage uid: ALS_381a035d1235 20/08/08 15:33:49.217 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_0_piece0 stored as bytes in memory (estimated size 7.8 KiB, free 366.3 MiB) 20/08/08 15:33:49.220 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master broadcast_0_piece0 for BlockManagerId(driver, 192.168.1.225, 38545, None) 20/08/08 15:33:49.222 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added broadcast_0_piece0 in memory on 192.168.1.225:38545 (size: 7.8 KiB, free: 366.3 MiB) 20/08/08 15:33:49.227 dag-scheduler-event-loop DEBUG BlockManagerMaster: Updated info of block broadcast_0_piece0 20/08/08 15:33:49.233 dag-scheduler-event-loop DEBUG BlockManager: Told master about block broadcast_0_piece0 20/08/08 15:33:49.234 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_0_piece0 locally took 20 ms 20/08/08 15:33:49.234 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_0_piece0 without replication took 20 ms 20/08/08 15:33:49.235 dag-scheduler-event-loop INFO SparkContext: Created broadcast 0 from broadcast at DAGScheduler.scala:1348 20/08/08 15:33:49.249 dag-scheduler-event-loop INFO DAGScheduler: Submitting 4 missing tasks from ResultStage 0 (MapPartitionsRDD[6] at at /usr/lib/python3.6/runpy.py:85) (first 15 tasks are for partitions Vector(0, 1, 2, 3)) 20/08/08 15:33:49.251 dag-scheduler-event-loop INFO TaskSchedulerImpl: Adding task set 0.0 with 4 tasks resource profile 0 20/08/08 15:33:49.266 Thread-4 DEBUG WholeStageCodegenExec: /* 001 */ public Object generate(Object[] references) { /* 002 */ return new GeneratedIteratorForCodegenStage1(references); /* 003 */ } /* 004 */ /* 005 */ // codegenStageId=1 /* 006 */ final class GeneratedIteratorForCodegenStage1 extends org.apache.spark.sql.execution.BufferedRowIterator { /* 007 */ private Object[] references; /* 008 */ private scala.collection.Iterator[] inputs; /* 009 */ private scala.collection.Iterator rdd_input_0; /* 010 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[] rdd_mutableStateArray_0 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[1]; /* 011 */ /* 012 */ public GeneratedIteratorForCodegenStage1(Object[] references) { /* 013 */ this.references = references; /* 014 */ } /* 015 */ /* 016 */ public void init(int index, scala.collection.Iterator[] inputs) { /* 017 */ partitionIndex = index; /* 018 */ this.inputs = inputs; /* 019 */ rdd_input_0 = inputs[0]; /* 020 */ rdd_mutableStateArray_0[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(3, 0); /* 021 */ /* 022 */ } /* 023 */ /* 024 */ protected void processNext() throws java.io.IOException { /* 025 */ while ( rdd_input_0.hasNext()) { /* 026 */ InternalRow rdd_row_0 = (InternalRow) rdd_input_0.next(); /* 027 */ ((org.apache.spark.sql.execution.metric.SQLMetric) references[0] /* numOutputRows */).add(1); /* 028 */ boolean rdd_isNull_0 = rdd_row_0.isNullAt(0); /* 029 */ long rdd_value_0 = rdd_isNull_0 ? /* 030 */ -1L : (rdd_row_0.getLong(0)); /* 031 */ boolean rdd_isNull_1 = rdd_row_0.isNullAt(1); /* 032 */ long rdd_value_1 = rdd_isNull_1 ? /* 033 */ -1L : (rdd_row_0.getLong(1)); /* 034 */ boolean rdd_isNull_2 = rdd_row_0.isNullAt(2); /* 035 */ double rdd_value_2 = rdd_isNull_2 ? /* 036 */ -1.0 : (rdd_row_0.getDouble(2)); /* 037 */ rdd_mutableStateArray_0[0].reset(); /* 038 */ /* 039 */ rdd_mutableStateArray_0[0].zeroOutNullBytes(); /* 040 */ /* 041 */ if (rdd_isNull_0) { /* 042 */ rdd_mutableStateArray_0[0].setNullAt(0); /* 043 */ } else { /* 044 */ rdd_mutableStateArray_0[0].write(0, rdd_value_0); /* 045 */ } /* 046 */ /* 047 */ if (rdd_isNull_1) { /* 048 */ rdd_mutableStateArray_0[0].setNullAt(1); /* 049 */ } else { /* 050 */ rdd_mutableStateArray_0[0].write(1, rdd_value_1); /* 051 */ } /* 052 */ /* 053 */ if (rdd_isNull_2) { /* 054 */ rdd_mutableStateArray_0[0].setNullAt(2); /* 055 */ } else { /* 056 */ rdd_mutableStateArray_0[0].write(2, rdd_value_2); /* 057 */ } /* 058 */ append((rdd_mutableStateArray_0[0].getRow())); /* 059 */ if (shouldStop()) return; /* 060 */ } /* 061 */ } /* 062 */ /* 063 */ } 20/08/08 15:33:49.271 Thread-4 DEBUG CodeGenerator: /* 001 */ public Object generate(Object[] references) { /* 002 */ return new GeneratedIteratorForCodegenStage1(references); /* 003 */ } /* 004 */ /* 005 */ // codegenStageId=1 /* 006 */ final class GeneratedIteratorForCodegenStage1 extends org.apache.spark.sql.execution.BufferedRowIterator { /* 007 */ private Object[] references; /* 008 */ private scala.collection.Iterator[] inputs; /* 009 */ private scala.collection.Iterator rdd_input_0; /* 010 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[] rdd_mutableStateArray_0 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[1]; /* 011 */ /* 012 */ public GeneratedIteratorForCodegenStage1(Object[] references) { /* 013 */ this.references = references; /* 014 */ } /* 015 */ /* 016 */ public void init(int index, scala.collection.Iterator[] inputs) { /* 017 */ partitionIndex = index; /* 018 */ this.inputs = inputs; /* 019 */ rdd_input_0 = inputs[0]; /* 020 */ rdd_mutableStateArray_0[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(3, 0); /* 021 */ /* 022 */ } /* 023 */ /* 024 */ protected void processNext() throws java.io.IOException { /* 025 */ while ( rdd_input_0.hasNext()) { /* 026 */ InternalRow rdd_row_0 = (InternalRow) rdd_input_0.next(); /* 027 */ ((org.apache.spark.sql.execution.metric.SQLMetric) references[0] /* numOutputRows */).add(1); /* 028 */ boolean rdd_isNull_0 = rdd_row_0.isNullAt(0); /* 029 */ long rdd_value_0 = rdd_isNull_0 ? /* 030 */ -1L : (rdd_row_0.getLong(0)); /* 031 */ boolean rdd_isNull_1 = rdd_row_0.isNullAt(1); /* 032 */ long rdd_value_1 = rdd_isNull_1 ? /* 033 */ -1L : (rdd_row_0.getLong(1)); /* 034 */ boolean rdd_isNull_2 = rdd_row_0.isNullAt(2); /* 035 */ double rdd_value_2 = rdd_isNull_2 ? /* 036 */ -1.0 : (rdd_row_0.getDouble(2)); /* 037 */ rdd_mutableStateArray_0[0].reset(); /* 038 */ /* 039 */ rdd_mutableStateArray_0[0].zeroOutNullBytes(); /* 040 */ /* 041 */ if (rdd_isNull_0) { /* 042 */ rdd_mutableStateArray_0[0].setNullAt(0); /* 043 */ } else { /* 044 */ rdd_mutableStateArray_0[0].write(0, rdd_value_0); /* 045 */ } /* 046 */ /* 047 */ if (rdd_isNull_1) { /* 048 */ rdd_mutableStateArray_0[0].setNullAt(1); /* 049 */ } else { /* 050 */ rdd_mutableStateArray_0[0].write(1, rdd_value_1); /* 051 */ } /* 052 */ /* 053 */ if (rdd_isNull_2) { /* 054 */ rdd_mutableStateArray_0[0].setNullAt(2); /* 055 */ } else { /* 056 */ rdd_mutableStateArray_0[0].write(2, rdd_value_2); /* 057 */ } /* 058 */ append((rdd_mutableStateArray_0[0].getRow())); /* 059 */ if (shouldStop()) return; /* 060 */ } /* 061 */ } /* 062 */ /* 063 */ } 20/08/08 15:33:49.277 dag-scheduler-event-loop DEBUG TaskSetManager: Epoch for TaskSet 0.0: 0 20/08/08 15:33:49.281 dag-scheduler-event-loop DEBUG TaskSetManager: Adding pending tasks took 3 ms 20/08/08 15:33:49.283 dag-scheduler-event-loop DEBUG TaskSetManager: Valid locality levels for TaskSet 0.0: NO_PREF, ANY 20/08/08 15:33:49.297 Thread-4 INFO CodeGenerator: Code generated in 30.144258 ms 20/08/08 15:33:49.298 Thread-4 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$doExecute$4$adapted 20/08/08 15:33:49.300 Thread-4 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$doExecute$4$adapted) is now cleaned +++ 20/08/08 15:33:49.302 Thread-4 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$rdd$1 20/08/08 15:33:49.306 Thread-4 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$rdd$1) is now cleaned +++ 20/08/08 15:33:49.305 dispatcher-event-loop-2 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_0.0, runningTasks: 0 20/08/08 15:33:49.319 Thread-4 INFO Instrumentation: [6aa586b7] training: numPartitions=4 storageLevel=StorageLevel(1 replicas) 20/08/08 15:33:49.322 dispatcher-event-loop-2 DEBUG TaskSetManager: Valid locality levels for TaskSet 0.0: NO_PREF, ANY 20/08/08 15:33:49.342 Thread-4 INFO CodeGenerator: Code generated in 493.211181 ms 20/08/08 15:33:49.343 Thread-4 INFO Instrumentation: [6aa586b7] {"rank":1,"maxIter":1} 20/08/08 15:33:49.344 Thread-4 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$doExecute$4$adapted 20/08/08 15:33:49.346 Thread-4 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$doExecute$4$adapted) is now cleaned +++ 20/08/08 15:33:49.352 Thread-4 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$take$2 20/08/08 15:33:49.361 Thread-4 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$take$2) is now cleaned +++ 20/08/08 15:33:49.375 dispatcher-event-loop-2 INFO TaskSetManager: Starting task 0.0 in stage 0.0 (TID 0, 192.168.1.225, executor driver, partition 0, PROCESS_LOCAL, 7450 bytes) taskResourceAssignments Map() 20/08/08 15:33:49.391 dispatcher-event-loop-2 INFO TaskSetManager: Starting task 1.0 in stage 0.0 (TID 1, 192.168.1.225, executor driver, partition 1, PROCESS_LOCAL, 7450 bytes) taskResourceAssignments Map() 20/08/08 15:33:49.392 dispatcher-event-loop-2 INFO TaskSetManager: Starting task 2.0 in stage 0.0 (TID 2, 192.168.1.225, executor driver, partition 2, PROCESS_LOCAL, 7450 bytes) taskResourceAssignments Map() 20/08/08 15:33:49.393 dispatcher-event-loop-2 INFO TaskSetManager: Starting task 3.0 in stage 0.0 (TID 3, 192.168.1.225, executor driver, partition 3, PROCESS_LOCAL, 7450 bytes) taskResourceAssignments Map() 20/08/08 15:33:49.420 Thread-4 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$executeTake$2 20/08/08 15:33:49.421 Thread-4 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$executeTake$2) is now cleaned +++ 20/08/08 15:33:49.424 Thread-4 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$runJob$5 20/08/08 15:33:49.428 Executor task launch worker for task 0 INFO Executor: Running task 0.0 in stage 0.0 (TID 0) 20/08/08 15:33:49.430 Thread-4 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$runJob$5) is now cleaned +++ 20/08/08 15:33:49.432 Thread-4 INFO SparkContext: Starting job: head at MetadataUtils.scala:51 20/08/08 15:33:49.432 Executor task launch worker for task 2 INFO Executor: Running task 2.0 in stage 0.0 (TID 2) 20/08/08 15:33:49.432 Executor task launch worker for task 1 INFO Executor: Running task 1.0 in stage 0.0 (TID 1) 20/08/08 15:33:49.437 Thread-4 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$runJob$5 20/08/08 15:33:49.437 Executor task launch worker for task 3 INFO Executor: Running task 3.0 in stage 0.0 (TID 3) 20/08/08 15:33:49.439 dag-scheduler-event-loop DEBUG DAGScheduler: Merging stage rdd profiles: Set() 20/08/08 15:33:49.441 Thread-4 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$runJob$5) is now cleaned +++ 20/08/08 15:33:49.446 Executor task launch worker for task 3 DEBUG ExecutorMetricsPoller: stageTCMP: (0, 0) -> 2 20/08/08 15:33:49.448 Thread-4 INFO SparkContext: Starting job: isEmpty at ALS.scala:947 20/08/08 15:33:49.454 Executor task launch worker for task 2 DEBUG ExecutorMetricsPoller: stageTCMP: (0, 0) -> 1 20/08/08 15:33:49.455 dag-scheduler-event-loop DEBUG DAGScheduler: Merging stage rdd profiles: Set() 20/08/08 15:33:49.448 Executor task launch worker for task 1 DEBUG ExecutorMetricsPoller: stageTCMP: (0, 0) -> 3 20/08/08 15:33:49.448 Executor task launch worker for task 0 DEBUG ExecutorMetricsPoller: stageTCMP: (0, 0) -> 4 20/08/08 15:33:49.460 dag-scheduler-event-loop INFO DAGScheduler: Got job 0 (head at MetadataUtils.scala:51) with 1 output partitions 20/08/08 15:33:49.461 dag-scheduler-event-loop INFO DAGScheduler: Final stage: ResultStage 0 (head at MetadataUtils.scala:51) 20/08/08 15:33:49.462 dag-scheduler-event-loop INFO DAGScheduler: Parents of final stage: List() 20/08/08 15:33:49.465 dag-scheduler-event-loop INFO DAGScheduler: Missing parents: List() 20/08/08 15:33:49.481 dag-scheduler-event-loop INFO DAGScheduler: Got job 0 (isEmpty at ALS.scala:947) with 1 output partitions 20/08/08 15:33:49.482 dag-scheduler-event-loop INFO DAGScheduler: Final stage: ResultStage 0 (isEmpty at ALS.scala:947) 20/08/08 15:33:49.483 dag-scheduler-event-loop INFO DAGScheduler: Parents of final stage: List() 20/08/08 15:33:49.490 dag-scheduler-event-loop INFO DAGScheduler: Missing parents: List() 20/08/08 15:33:49.495 dag-scheduler-event-loop DEBUG DAGScheduler: submitStage(ResultStage 0 (name=isEmpty at ALS.scala:947;jobs=0)) 20/08/08 15:33:49.510 dag-scheduler-event-loop DEBUG DAGScheduler: submitStage(ResultStage 0 (name=head at MetadataUtils.scala:51;jobs=0)) 20/08/08 15:33:49.502 dag-scheduler-event-loop DEBUG DAGScheduler: missing: List() 20/08/08 15:33:49.514 dag-scheduler-event-loop DEBUG DAGScheduler: missing: List() 20/08/08 15:33:49.517 dag-scheduler-event-loop INFO DAGScheduler: Submitting ResultStage 0 (MapPartitionsRDD[6] at head at MetadataUtils.scala:51), which has no missing parents 20/08/08 15:33:49.517 dag-scheduler-event-loop DEBUG DAGScheduler: submitMissingTasks(ResultStage 0) 20/08/08 15:33:49.530 dag-scheduler-event-loop INFO DAGScheduler: Submitting ResultStage 0 (MapPartitionsRDD[9] at map at ALS.scala:699), which has no missing parents 20/08/08 15:33:49.531 dag-scheduler-event-loop DEBUG DAGScheduler: submitMissingTasks(ResultStage 0) 20/08/08 15:33:49.542 Executor task launch worker for task 0 DEBUG BlockManager: Getting local block broadcast_0 20/08/08 15:33:49.543 Executor task launch worker for task 0 DEBUG BlockManager: Level for block broadcast_0 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:49.547 Thread-4 INFO CodeGenerator: Code generated in 616.140126 ms 20/08/08 15:33:49.549 Thread-4 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$doExecute$4$adapted 20/08/08 15:33:49.551 Thread-4 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$doExecute$4$adapted) is now cleaned +++ 20/08/08 15:33:49.661 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_0 stored as values in memory (estimated size 27.9 KiB, free 366.3 MiB) 20/08/08 15:33:49.664 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_0 locally took 54 ms 20/08/08 15:33:49.667 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_0 without replication took 57 ms 20/08/08 15:33:49.713 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.2 KiB, free 366.3 MiB) 20/08/08 15:33:49.715 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master broadcast_0_piece0 for BlockManagerId(driver, 192.168.1.225, 38159, None) 20/08/08 15:33:49.717 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added broadcast_0_piece0 in memory on 192.168.1.225:38159 (size: 13.2 KiB, free: 366.3 MiB) 20/08/08 15:33:49.720 dag-scheduler-event-loop DEBUG BlockManagerMaster: Updated info of block broadcast_0_piece0 20/08/08 15:33:49.720 dag-scheduler-event-loop DEBUG BlockManager: Told master about block broadcast_0_piece0 20/08/08 15:33:49.721 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_0_piece0 locally took 12 ms 20/08/08 15:33:49.721 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_0_piece0 without replication took 12 ms 20/08/08 15:33:49.722 dag-scheduler-event-loop INFO SparkContext: Created broadcast 0 from broadcast at DAGScheduler.scala:1348 20/08/08 15:33:49.723 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_0 stored as values in memory (estimated size 15.1 KiB, free 366.3 MiB) 20/08/08 15:33:49.725 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_0 locally took 58 ms 20/08/08 15:33:49.728 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_0 without replication took 61 ms 20/08/08 15:33:49.749 dag-scheduler-event-loop INFO DAGScheduler: Submitting 1 missing tasks from ResultStage 0 (MapPartitionsRDD[9] at map at ALS.scala:699) (first 15 tasks are for partitions Vector(0)) 20/08/08 15:33:49.750 dag-scheduler-event-loop INFO TaskSchedulerImpl: Adding task set 0.0 with 1 tasks resource profile 0 20/08/08 15:33:49.781 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_0_piece0 stored as bytes in memory (estimated size 7.0 KiB, free 366.3 MiB) 20/08/08 15:33:49.783 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master broadcast_0_piece0 for BlockManagerId(driver, 192.168.1.225, 36391, None) 20/08/08 15:33:49.784 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added broadcast_0_piece0 in memory on 192.168.1.225:36391 (size: 7.0 KiB, free: 366.3 MiB) 20/08/08 15:33:49.787 dag-scheduler-event-loop DEBUG BlockManagerMaster: Updated info of block broadcast_0_piece0 20/08/08 15:33:49.788 dag-scheduler-event-loop DEBUG BlockManager: Told master about block broadcast_0_piece0 20/08/08 15:33:49.793 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_0_piece0 locally took 14 ms 20/08/08 15:33:49.793 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_0_piece0 without replication took 15 ms 20/08/08 15:33:49.794 dag-scheduler-event-loop INFO SparkContext: Created broadcast 0 from broadcast at DAGScheduler.scala:1348 20/08/08 15:33:49.807 dag-scheduler-event-loop DEBUG TaskSetManager: Epoch for TaskSet 0.0: 0 20/08/08 15:33:49.812 dag-scheduler-event-loop DEBUG TaskSetManager: Adding pending tasks took 3 ms 20/08/08 15:33:49.813 dag-scheduler-event-loop DEBUG TaskSetManager: Valid locality levels for TaskSet 0.0: NO_PREF, ANY 20/08/08 15:33:49.815 dag-scheduler-event-loop INFO DAGScheduler: Submitting 1 missing tasks from ResultStage 0 (MapPartitionsRDD[6] at head at MetadataUtils.scala:51) (first 15 tasks are for partitions Vector(0)) 20/08/08 15:33:49.817 Thread-4 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$collect$2 20/08/08 15:33:49.817 dag-scheduler-event-loop INFO TaskSchedulerImpl: Adding task set 0.0 with 1 tasks resource profile 0 20/08/08 15:33:49.829 Thread-4 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$collect$2) is now cleaned +++ 20/08/08 15:33:49.840 dispatcher-event-loop-2 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_0.0, runningTasks: 0 20/08/08 15:33:49.841 dispatcher-event-loop-2 DEBUG TaskSetManager: Valid locality levels for TaskSet 0.0: NO_PREF, ANY 20/08/08 15:33:49.857 dag-scheduler-event-loop DEBUG TaskSetManager: Epoch for TaskSet 0.0: 0 20/08/08 15:33:49.862 dag-scheduler-event-loop DEBUG TaskSetManager: Adding pending tasks took 3 ms 20/08/08 15:33:49.864 Thread-4 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$runJob$5 20/08/08 15:33:49.864 dag-scheduler-event-loop DEBUG TaskSetManager: Valid locality levels for TaskSet 0.0: NO_PREF, ANY 20/08/08 15:33:49.868 Thread-4 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$runJob$5) is now cleaned +++ 20/08/08 15:33:49.881 Thread-4 INFO SparkContext: Starting job: at /usr/lib/python3.6/runpy.py:85 20/08/08 15:33:49.888 dag-scheduler-event-loop DEBUG DAGScheduler: Merging stage rdd profiles: Set() 20/08/08 15:33:49.888 dispatcher-event-loop-2 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_0.0, runningTasks: 0 20/08/08 15:33:49.894 dispatcher-event-loop-2 DEBUG TaskSetManager: Valid locality levels for TaskSet 0.0: NO_PREF, ANY 20/08/08 15:33:49.911 dispatcher-event-loop-2 INFO TaskSetManager: Starting task 0.0 in stage 0.0 (TID 0, 192.168.1.225, executor driver, partition 0, PROCESS_LOCAL, 7373 bytes) taskResourceAssignments Map() 20/08/08 15:33:49.918 dispatcher-event-loop-2 DEBUG TaskSetManager: No tasks for locality level NO_PREF, so moving to locality level ANY 20/08/08 15:33:49.932 dag-scheduler-event-loop INFO DAGScheduler: Got job 0 ( at /usr/lib/python3.6/runpy.py:85) with 4 output partitions 20/08/08 15:33:49.933 dag-scheduler-event-loop INFO DAGScheduler: Final stage: ResultStage 0 ( at /usr/lib/python3.6/runpy.py:85) 20/08/08 15:33:49.933 dag-scheduler-event-loop INFO DAGScheduler: Parents of final stage: List() 20/08/08 15:33:49.936 dag-scheduler-event-loop INFO DAGScheduler: Missing parents: List() 20/08/08 15:33:49.950 dispatcher-event-loop-2 INFO TaskSetManager: Starting task 0.0 in stage 0.0 (TID 0, 192.168.1.225, executor driver, partition 0, PROCESS_LOCAL, 7398 bytes) taskResourceAssignments Map() 20/08/08 15:33:49.959 Executor task launch worker for task 0 INFO Executor: Running task 0.0 in stage 0.0 (TID 0) 20/08/08 15:33:49.960 dispatcher-event-loop-2 DEBUG TaskSetManager: No tasks for locality level NO_PREF, so moving to locality level ANY 20/08/08 15:33:49.962 dag-scheduler-event-loop DEBUG DAGScheduler: submitStage(ResultStage 0 (name= at /usr/lib/python3.6/runpy.py:85;jobs=0)) 20/08/08 15:33:49.964 dag-scheduler-event-loop DEBUG DAGScheduler: missing: List() 20/08/08 15:33:49.989 Executor task launch worker for task 0 DEBUG ExecutorMetricsPoller: stageTCMP: (0, 0) -> 1 20/08/08 15:33:50.005 dag-scheduler-event-loop INFO DAGScheduler: Submitting ResultStage 0 (MapPartitionsRDD[6] at at /usr/lib/python3.6/runpy.py:85), which has no missing parents 20/08/08 15:33:50.005 dag-scheduler-event-loop DEBUG DAGScheduler: submitMissingTasks(ResultStage 0) 20/08/08 15:33:50.010 Executor task launch worker for task 0 INFO Executor: Running task 0.0 in stage 0.0 (TID 0) 20/08/08 15:33:50.031 Executor task launch worker for task 0 DEBUG ExecutorMetricsPoller: stageTCMP: (0, 0) -> 1 20/08/08 15:33:50.080 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_1 stored as values in memory (estimated size 31.1 KiB, free 366.3 MiB) 20/08/08 15:33:50.081 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_1 locally took 14 ms 20/08/08 15:33:50.081 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_1 without replication took 15 ms 20/08/08 15:33:50.088 Executor task launch worker for task 0 DEBUG BlockManager: Getting local block broadcast_0 20/08/08 15:33:50.090 Executor task launch worker for task 0 DEBUG BlockManager: Level for block broadcast_0 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:50.097 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_1_piece0 stored as bytes in memory (estimated size 13.1 KiB, free 366.3 MiB) 20/08/08 15:33:50.097 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master broadcast_1_piece0 for BlockManagerId(driver, 192.168.1.225, 36433, None) 20/08/08 15:33:50.098 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added broadcast_1_piece0 in memory on 192.168.1.225:36433 (size: 13.1 KiB, free: 366.3 MiB) 20/08/08 15:33:50.099 dag-scheduler-event-loop DEBUG BlockManagerMaster: Updated info of block broadcast_1_piece0 20/08/08 15:33:50.099 dag-scheduler-event-loop DEBUG BlockManager: Told master about block broadcast_1_piece0 20/08/08 15:33:50.099 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_1_piece0 locally took 2 ms 20/08/08 15:33:50.099 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_1_piece0 without replication took 2 ms 20/08/08 15:33:50.099 dag-scheduler-event-loop INFO SparkContext: Created broadcast 1 from broadcast at DAGScheduler.scala:1348 20/08/08 15:33:50.121 dag-scheduler-event-loop INFO DAGScheduler: Submitting 4 missing tasks from ResultStage 0 (MapPartitionsRDD[6] at at /usr/lib/python3.6/runpy.py:85) (first 15 tasks are for partitions Vector(0, 1, 2, 3)) 20/08/08 15:33:50.123 dag-scheduler-event-loop INFO TaskSchedulerImpl: Adding task set 0.0 with 4 tasks resource profile 0 20/08/08 15:33:50.125 Executor task launch worker for task 3 DEBUG GenerateMutableProjection: code for cast(input[0, bigint, false] as double): /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificMutableProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificMutableProjection extends org.apache.spark.sql.catalyst.expressions.codegen.BaseMutableProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private InternalRow mutableRow; /* 009 */ private double value_2; /* 010 */ /* 011 */ public SpecificMutableProjection(Object[] references) { /* 012 */ this.references = references; /* 013 */ mutableRow = new org.apache.spark.sql.catalyst.expressions.GenericInternalRow(1); /* 014 */ /* 015 */ /* 016 */ } /* 017 */ /* 018 */ public void initialize(int partitionIndex) { /* 019 */ /* 020 */ } /* 021 */ /* 022 */ public org.apache.spark.sql.catalyst.expressions.codegen.BaseMutableProjection target(InternalRow row) { /* 023 */ mutableRow = row; /* 024 */ return this; /* 025 */ } /* 026 */ /* 027 */ /* Provide immutable access to the last projected row. */ /* 028 */ public InternalRow currentValue() { /* 029 */ return (InternalRow) mutableRow; /* 030 */ } /* 031 */ /* 032 */ public java.lang.Object apply(java.lang.Object _i) { /* 033 */ InternalRow i = (InternalRow) _i; /* 034 */ /* 035 */ /* 036 */ long value_1 = i.getLong(0); /* 037 */ boolean isNull_0 = false; /* 038 */ double value_0 = -1.0; /* 039 */ if (!false) { /* 040 */ value_0 = (double) value_1; /* 041 */ } /* 042 */ value_2 = value_0; /* 043 */ /* 044 */ // copy all the results into MutableRow /* 045 */ mutableRow.setDouble(0, value_2); /* 046 */ return mutableRow; /* 047 */ } /* 048 */ /* 049 */ /* 050 */ } 20/08/08 15:33:50.133 Executor task launch worker for task 0 DEBUG GenerateMutableProjection: code for cast(input[0, bigint, false] as double): /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificMutableProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificMutableProjection extends org.apache.spark.sql.catalyst.expressions.codegen.BaseMutableProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private InternalRow mutableRow; /* 009 */ private double value_2; /* 010 */ /* 011 */ public SpecificMutableProjection(Object[] references) { /* 012 */ this.references = references; /* 013 */ mutableRow = new org.apache.spark.sql.catalyst.expressions.GenericInternalRow(1); /* 014 */ /* 015 */ /* 016 */ } /* 017 */ /* 018 */ public void initialize(int partitionIndex) { /* 019 */ /* 020 */ } /* 021 */ /* 022 */ public org.apache.spark.sql.catalyst.expressions.codegen.BaseMutableProjection target(InternalRow row) { /* 023 */ mutableRow = row; /* 024 */ return this; /* 025 */ } /* 026 */ /* 027 */ /* Provide immutable access to the last projected row. */ /* 028 */ public InternalRow currentValue() { /* 029 */ return (InternalRow) mutableRow; /* 030 */ } /* 031 */ /* 032 */ public java.lang.Object apply(java.lang.Object _i) { /* 033 */ InternalRow i = (InternalRow) _i; /* 034 */ /* 035 */ /* 036 */ long value_1 = i.getLong(0); /* 037 */ boolean isNull_0 = false; /* 038 */ double value_0 = -1.0; /* 039 */ if (!false) { /* 040 */ value_0 = (double) value_1; /* 041 */ } /* 042 */ value_2 = value_0; /* 043 */ /* 044 */ // copy all the results into MutableRow /* 045 */ mutableRow.setDouble(0, value_2); /* 046 */ return mutableRow; /* 047 */ } /* 048 */ /* 049 */ /* 050 */ } 20/08/08 15:33:50.131 Executor task launch worker for task 1 DEBUG GenerateMutableProjection: code for cast(input[0, bigint, false] as double): /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificMutableProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificMutableProjection extends org.apache.spark.sql.catalyst.expressions.codegen.BaseMutableProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private InternalRow mutableRow; /* 009 */ private double value_2; /* 010 */ /* 011 */ public SpecificMutableProjection(Object[] references) { /* 012 */ this.references = references; /* 013 */ mutableRow = new org.apache.spark.sql.catalyst.expressions.GenericInternalRow(1); /* 014 */ /* 015 */ /* 016 */ } /* 017 */ /* 018 */ public void initialize(int partitionIndex) { /* 019 */ /* 020 */ } /* 021 */ /* 022 */ public org.apache.spark.sql.catalyst.expressions.codegen.BaseMutableProjection target(InternalRow row) { /* 023 */ mutableRow = row; /* 024 */ return this; /* 025 */ } /* 026 */ /* 027 */ /* Provide immutable access to the last projected row. */ /* 028 */ public InternalRow currentValue() { /* 029 */ return (InternalRow) mutableRow; /* 030 */ } /* 031 */ /* 032 */ public java.lang.Object apply(java.lang.Object _i) { /* 033 */ InternalRow i = (InternalRow) _i; /* 034 */ /* 035 */ /* 036 */ long value_1 = i.getLong(0); /* 037 */ boolean isNull_0 = false; /* 038 */ double value_0 = -1.0; /* 039 */ if (!false) { /* 040 */ value_0 = (double) value_1; /* 041 */ } /* 042 */ value_2 = value_0; /* 043 */ /* 044 */ // copy all the results into MutableRow /* 045 */ mutableRow.setDouble(0, value_2); /* 046 */ return mutableRow; /* 047 */ } /* 048 */ /* 049 */ /* 050 */ } 20/08/08 15:33:50.146 Executor task launch worker for task 0 DEBUG BlockManager: Getting local block broadcast_0 20/08/08 15:33:50.148 Executor task launch worker for task 0 DEBUG BlockManager: Level for block broadcast_0 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:50.141 Executor task launch worker for task 2 DEBUG GenerateMutableProjection: code for cast(input[0, bigint, false] as double): /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificMutableProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificMutableProjection extends org.apache.spark.sql.catalyst.expressions.codegen.BaseMutableProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private InternalRow mutableRow; /* 009 */ private double value_2; /* 010 */ /* 011 */ public SpecificMutableProjection(Object[] references) { /* 012 */ this.references = references; /* 013 */ mutableRow = new org.apache.spark.sql.catalyst.expressions.GenericInternalRow(1); /* 014 */ /* 015 */ /* 016 */ } /* 017 */ /* 018 */ public void initialize(int partitionIndex) { /* 019 */ /* 020 */ } /* 021 */ /* 022 */ public org.apache.spark.sql.catalyst.expressions.codegen.BaseMutableProjection target(InternalRow row) { /* 023 */ mutableRow = row; /* 024 */ return this; /* 025 */ } /* 026 */ /* 027 */ /* Provide immutable access to the last projected row. */ /* 028 */ public InternalRow currentValue() { /* 029 */ return (InternalRow) mutableRow; /* 030 */ } /* 031 */ /* 032 */ public java.lang.Object apply(java.lang.Object _i) { /* 033 */ InternalRow i = (InternalRow) _i; /* 034 */ /* 035 */ /* 036 */ long value_1 = i.getLong(0); /* 037 */ boolean isNull_0 = false; /* 038 */ double value_0 = -1.0; /* 039 */ if (!false) { /* 040 */ value_0 = (double) value_1; /* 041 */ } /* 042 */ value_2 = value_0; /* 043 */ /* 044 */ // copy all the results into MutableRow /* 045 */ mutableRow.setDouble(0, value_2); /* 046 */ return mutableRow; /* 047 */ } /* 048 */ /* 049 */ /* 050 */ } 20/08/08 15:33:50.140 Executor task launch worker for task 3 DEBUG CodeGenerator: /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificMutableProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificMutableProjection extends org.apache.spark.sql.catalyst.expressions.codegen.BaseMutableProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private InternalRow mutableRow; /* 009 */ private double value_2; /* 010 */ /* 011 */ public SpecificMutableProjection(Object[] references) { /* 012 */ this.references = references; /* 013 */ mutableRow = new org.apache.spark.sql.catalyst.expressions.GenericInternalRow(1); /* 014 */ /* 015 */ /* 016 */ } /* 017 */ /* 018 */ public void initialize(int partitionIndex) { /* 019 */ /* 020 */ } /* 021 */ /* 022 */ public org.apache.spark.sql.catalyst.expressions.codegen.BaseMutableProjection target(InternalRow row) { /* 023 */ mutableRow = row; /* 024 */ return this; /* 025 */ } /* 026 */ /* 027 */ /* Provide immutable access to the last projected row. */ /* 028 */ public InternalRow currentValue() { /* 029 */ return (InternalRow) mutableRow; /* 030 */ } /* 031 */ /* 032 */ public java.lang.Object apply(java.lang.Object _i) { /* 033 */ InternalRow i = (InternalRow) _i; /* 034 */ /* 035 */ /* 036 */ long value_1 = i.getLong(0); /* 037 */ boolean isNull_0 = false; /* 038 */ double value_0 = -1.0; /* 039 */ if (!false) { /* 040 */ value_0 = (double) value_1; /* 041 */ } /* 042 */ value_2 = value_0; /* 043 */ /* 044 */ // copy all the results into MutableRow /* 045 */ mutableRow.setDouble(0, value_2); /* 046 */ return mutableRow; /* 047 */ } /* 048 */ /* 049 */ /* 050 */ } 20/08/08 15:33:50.149 dag-scheduler-event-loop DEBUG TaskSetManager: Epoch for TaskSet 0.0: 0 20/08/08 15:33:50.154 dag-scheduler-event-loop DEBUG TaskSetManager: Adding pending tasks took 3 ms 20/08/08 15:33:50.155 dag-scheduler-event-loop DEBUG TaskSetManager: Valid locality levels for TaskSet 0.0: NO_PREF, ANY 20/08/08 15:33:50.162 Executor task launch worker for task 3 INFO CodeGenerator: Code generated in 36.092535 ms 20/08/08 15:33:50.174 dispatcher-event-loop-2 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_0.0, runningTasks: 0 20/08/08 15:33:50.177 dispatcher-event-loop-2 DEBUG TaskSetManager: Valid locality levels for TaskSet 0.0: NO_PREF, ANY 20/08/08 15:33:50.230 dispatcher-event-loop-2 INFO TaskSetManager: Starting task 0.0 in stage 0.0 (TID 0, 192.168.1.225, executor driver, partition 0, PROCESS_LOCAL, 7333 bytes) taskResourceAssignments Map() 20/08/08 15:33:50.236 dispatcher-event-loop-2 INFO TaskSetManager: Starting task 1.0 in stage 0.0 (TID 1, 192.168.1.225, executor driver, partition 1, PROCESS_LOCAL, 7421 bytes) taskResourceAssignments Map() 20/08/08 15:33:50.237 dispatcher-event-loop-2 INFO TaskSetManager: Starting task 2.0 in stage 0.0 (TID 2, 192.168.1.225, executor driver, partition 2, PROCESS_LOCAL, 7396 bytes) taskResourceAssignments Map() 20/08/08 15:33:50.237 dispatcher-event-loop-2 INFO TaskSetManager: Starting task 3.0 in stage 0.0 (TID 3, 192.168.1.225, executor driver, partition 3, PROCESS_LOCAL, 7385 bytes) taskResourceAssignments Map() 20/08/08 15:33:50.268 Executor task launch worker for task 1 INFO Executor: Running task 1.0 in stage 0.0 (TID 1) 20/08/08 15:33:50.268 Executor task launch worker for task 0 INFO Executor: Running task 0.0 in stage 0.0 (TID 0) 20/08/08 15:33:50.268 Executor task launch worker for task 2 INFO Executor: Running task 2.0 in stage 0.0 (TID 2) 20/08/08 15:33:50.280 Executor task launch worker for task 1 DEBUG ExecutorMetricsPoller: stageTCMP: (0, 0) -> 1 20/08/08 15:33:50.293 Executor task launch worker for task 2 DEBUG ExecutorMetricsPoller: stageTCMP: (0, 0) -> 2 20/08/08 15:33:50.294 Executor task launch worker for task 0 DEBUG ExecutorMetricsPoller: stageTCMP: (0, 0) -> 3 20/08/08 15:33:50.337 Executor task launch worker for task 3 INFO Executor: Running task 3.0 in stage 0.0 (TID 3) 20/08/08 15:33:50.338 Executor task launch worker for task 3 DEBUG ExecutorMetricsPoller: stageTCMP: (0, 0) -> 4 20/08/08 15:33:50.376 Executor task launch worker for task 3 DEBUG BlockManager: Getting local block broadcast_1 20/08/08 15:33:50.378 Executor task launch worker for task 3 DEBUG BlockManager: Level for block broadcast_1 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:50.863 stdout writer for /home/jenkins/venv/py36/bin/python3.6 DEBUG TaskMemoryManager: Task 1 acquired 4.0 MiB for HybridRowQueue(org.apache.spark.memory.TaskMemoryManager@360b538e,/home/jenkins/workspace/spark-master-test-python-arm/python/target/c1079ab9-e95b-42b9-9d66-c20bb67941db/spark-3c380b3e-bad9-4179-b58d-fb5610c338dc,1,org.apache.spark.serializer.SerializerManager@18d448be) 20/08/08 15:33:50.878 Executor task launch worker for task 1 DEBUG GenerateUnsafeProjection: code for input[0, bigint, false],input[1, double, true]: /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificUnsafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificUnsafeProjection extends org.apache.spark.sql.catalyst.expressions.UnsafeProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[] mutableStateArray_0 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[1]; /* 009 */ /* 010 */ public SpecificUnsafeProjection(Object[] references) { /* 011 */ this.references = references; /* 012 */ mutableStateArray_0[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(2, 0); /* 013 */ /* 014 */ } /* 015 */ /* 016 */ public void initialize(int partitionIndex) { /* 017 */ /* 018 */ } /* 019 */ /* 020 */ // Scala.Function1 need this /* 021 */ public java.lang.Object apply(java.lang.Object row) { /* 022 */ return apply((InternalRow) row); /* 023 */ } /* 024 */ /* 025 */ public UnsafeRow apply(InternalRow i) { /* 026 */ mutableStateArray_0[0].reset(); /* 027 */ /* 028 */ /* 029 */ mutableStateArray_0[0].zeroOutNullBytes(); /* 030 */ /* 031 */ long value_0 = i.getLong(0); /* 032 */ mutableStateArray_0[0].write(0, value_0); /* 033 */ /* 034 */ boolean isNull_1 = i.isNullAt(1); /* 035 */ double value_1 = isNull_1 ? /* 036 */ -1.0 : (i.getDouble(1)); /* 037 */ if (isNull_1) { /* 038 */ mutableStateArray_0[0].setNullAt(1); /* 039 */ } else { /* 040 */ mutableStateArray_0[0].write(1, value_1); /* 041 */ } /* 042 */ return (mutableStateArray_0[0].getRow()); /* 043 */ } /* 044 */ /* 045 */ /* 046 */ } 20/08/08 15:33:50.880 Executor task launch worker for task 1 DEBUG CodeGenerator: /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificUnsafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificUnsafeProjection extends org.apache.spark.sql.catalyst.expressions.UnsafeProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[] mutableStateArray_0 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[1]; /* 009 */ /* 010 */ public SpecificUnsafeProjection(Object[] references) { /* 011 */ this.references = references; /* 012 */ mutableStateArray_0[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(2, 0); /* 013 */ /* 014 */ } /* 015 */ /* 016 */ public void initialize(int partitionIndex) { /* 017 */ /* 018 */ } /* 019 */ /* 020 */ // Scala.Function1 need this /* 021 */ public java.lang.Object apply(java.lang.Object row) { /* 022 */ return apply((InternalRow) row); /* 023 */ } /* 024 */ /* 025 */ public UnsafeRow apply(InternalRow i) { /* 026 */ mutableStateArray_0[0].reset(); /* 027 */ /* 028 */ /* 029 */ mutableStateArray_0[0].zeroOutNullBytes(); /* 030 */ /* 031 */ long value_0 = i.getLong(0); /* 032 */ mutableStateArray_0[0].write(0, value_0); /* 033 */ /* 034 */ boolean isNull_1 = i.isNullAt(1); /* 035 */ double value_1 = isNull_1 ? /* 036 */ -1.0 : (i.getDouble(1)); /* 037 */ if (isNull_1) { /* 038 */ mutableStateArray_0[0].setNullAt(1); /* 039 */ } else { /* 040 */ mutableStateArray_0[0].write(1, value_1); /* 041 */ } /* 042 */ return (mutableStateArray_0[0].getRow()); /* 043 */ } /* 044 */ /* 045 */ /* 046 */ } 20/08/08 15:33:50.902 stdout writer for /home/jenkins/venv/py36/bin/python3.6 DEBUG TaskMemoryManager: Task 3 acquired 4.0 MiB for HybridRowQueue(org.apache.spark.memory.TaskMemoryManager@18a83dda,/home/jenkins/workspace/spark-master-test-python-arm/python/target/c1079ab9-e95b-42b9-9d66-c20bb67941db/spark-3c380b3e-bad9-4179-b58d-fb5610c338dc,1,org.apache.spark.serializer.SerializerManager@18d448be) 20/08/08 15:33:50.929 Executor task launch worker for task 0 DEBUG GenerateUnsafeProjection: code for input[0, bigint, false],input[1, double, true]: /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificUnsafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificUnsafeProjection extends org.apache.spark.sql.catalyst.expressions.UnsafeProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[] mutableStateArray_0 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[1]; /* 009 */ /* 010 */ public SpecificUnsafeProjection(Object[] references) { /* 011 */ this.references = references; /* 012 */ mutableStateArray_0[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(2, 0); /* 013 */ /* 014 */ } /* 015 */ /* 016 */ public void initialize(int partitionIndex) { /* 017 */ /* 018 */ } /* 019 */ /* 020 */ // Scala.Function1 need this /* 021 */ public java.lang.Object apply(java.lang.Object row) { /* 022 */ return apply((InternalRow) row); /* 023 */ } /* 024 */ /* 025 */ public UnsafeRow apply(InternalRow i) { /* 026 */ mutableStateArray_0[0].reset(); /* 027 */ /* 028 */ /* 029 */ mutableStateArray_0[0].zeroOutNullBytes(); /* 030 */ /* 031 */ long value_0 = i.getLong(0); /* 032 */ mutableStateArray_0[0].write(0, value_0); /* 033 */ /* 034 */ boolean isNull_1 = i.isNullAt(1); /* 035 */ double value_1 = isNull_1 ? /* 036 */ -1.0 : (i.getDouble(1)); /* 037 */ if (isNull_1) { /* 038 */ mutableStateArray_0[0].setNullAt(1); /* 039 */ } else { /* 040 */ mutableStateArray_0[0].write(1, value_1); /* 041 */ } /* 042 */ return (mutableStateArray_0[0].getRow()); /* 043 */ } /* 044 */ /* 045 */ /* 046 */ } 20/08/08 15:33:50.929 Executor task launch worker for task 1 INFO CodeGenerator: Code generated in 50.819217 ms 20/08/08 15:33:50.923 stdout writer for /home/jenkins/venv/py36/bin/python3.6 DEBUG TaskMemoryManager: Task 2 acquired 4.0 MiB for HybridRowQueue(org.apache.spark.memory.TaskMemoryManager@1f0ae42f,/home/jenkins/workspace/spark-master-test-python-arm/python/target/c1079ab9-e95b-42b9-9d66-c20bb67941db/spark-3c380b3e-bad9-4179-b58d-fb5610c338dc,1,org.apache.spark.serializer.SerializerManager@18d448be) 20/08/08 15:33:50.905 Executor task launch worker for task 2 DEBUG GenerateUnsafeProjection: code for input[0, bigint, false],input[1, double, true]: /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificUnsafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificUnsafeProjection extends org.apache.spark.sql.catalyst.expressions.UnsafeProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[] mutableStateArray_0 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[1]; /* 009 */ /* 010 */ public SpecificUnsafeProjection(Object[] references) { /* 011 */ this.references = references; /* 012 */ mutableStateArray_0[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(2, 0); /* 013 */ /* 014 */ } /* 015 */ /* 016 */ public void initialize(int partitionIndex) { /* 017 */ /* 018 */ } /* 019 */ /* 020 */ // Scala.Function1 need this /* 021 */ public java.lang.Object apply(java.lang.Object row) { /* 022 */ return apply((InternalRow) row); /* 023 */ } /* 024 */ /* 025 */ public UnsafeRow apply(InternalRow i) { /* 026 */ mutableStateArray_0[0].reset(); /* 027 */ /* 028 */ /* 029 */ mutableStateArray_0[0].zeroOutNullBytes(); /* 030 */ /* 031 */ long value_0 = i.getLong(0); /* 032 */ mutableStateArray_0[0].write(0, value_0); /* 033 */ /* 034 */ boolean isNull_1 = i.isNullAt(1); /* 035 */ double value_1 = isNull_1 ? /* 036 */ -1.0 : (i.getDouble(1)); /* 037 */ if (isNull_1) { /* 038 */ mutableStateArray_0[0].setNullAt(1); /* 039 */ } else { /* 040 */ mutableStateArray_0[0].write(1, value_1); /* 041 */ } /* 042 */ return (mutableStateArray_0[0].getRow()); /* 043 */ } /* 044 */ /* 045 */ /* 046 */ } 20/08/08 15:33:50.893 Executor task launch worker for task 3 DEBUG GenerateUnsafeProjection: code for input[0, bigint, false],input[1, double, true]: /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificUnsafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificUnsafeProjection extends org.apache.spark.sql.catalyst.expressions.UnsafeProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[] mutableStateArray_0 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[1]; /* 009 */ /* 010 */ public SpecificUnsafeProjection(Object[] references) { /* 011 */ this.references = references; /* 012 */ mutableStateArray_0[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(2, 0); /* 013 */ /* 014 */ } /* 015 */ /* 016 */ public void initialize(int partitionIndex) { /* 017 */ /* 018 */ } /* 019 */ /* 020 */ // Scala.Function1 need this /* 021 */ public java.lang.Object apply(java.lang.Object row) { /* 022 */ return apply((InternalRow) row); /* 023 */ } /* 024 */ /* 025 */ public UnsafeRow apply(InternalRow i) { /* 026 */ mutableStateArray_0[0].reset(); /* 027 */ /* 028 */ /* 029 */ mutableStateArray_0[0].zeroOutNullBytes(); /* 030 */ /* 031 */ long value_0 = i.getLong(0); /* 032 */ mutableStateArray_0[0].write(0, value_0); /* 033 */ /* 034 */ boolean isNull_1 = i.isNullAt(1); /* 035 */ double value_1 = isNull_1 ? /* 036 */ -1.0 : (i.getDouble(1)); /* 037 */ if (isNull_1) { /* 038 */ mutableStateArray_0[0].setNullAt(1); /* 039 */ } else { /* 040 */ mutableStateArray_0[0].write(1, value_1); /* 041 */ } /* 042 */ return (mutableStateArray_0[0].getRow()); /* 043 */ } /* 044 */ /* 045 */ /* 046 */ } 20/08/08 15:33:50.955 stdout writer for /home/jenkins/venv/py36/bin/python3.6 DEBUG TaskMemoryManager: Task 0 acquired 4.0 MiB for HybridRowQueue(org.apache.spark.memory.TaskMemoryManager@11d56d53,/home/jenkins/workspace/spark-master-test-python-arm/python/target/c1079ab9-e95b-42b9-9d66-c20bb67941db/spark-3c380b3e-bad9-4179-b58d-fb5610c338dc,1,org.apache.spark.serializer.SerializerManager@18d448be) 20/08/08 15:33:50.977 Executor task launch worker for task 2 INFO PythonUDFRunner: Times: total = 762, boot = 665, init = 97, finish = 0 20/08/08 15:33:50.979 Executor task launch worker for task 0 INFO PythonUDFRunner: Times: total = 775, boot = 707, init = 68, finish = 0 20/08/08 15:33:50.980 Executor task launch worker for task 3 INFO PythonUDFRunner: Times: total = 752, boot = 674, init = 78, finish = 0 20/08/08 15:33:50.979 Executor task launch worker for task 1 INFO PythonUDFRunner: Times: total = 745, boot = 661, init = 83, finish = 1 20/08/08 15:33:50.984 Executor task launch worker for task 2 DEBUG TaskMemoryManager: Task 2 release 4.0 MiB from HybridRowQueue(org.apache.spark.memory.TaskMemoryManager@1f0ae42f,/home/jenkins/workspace/spark-master-test-python-arm/python/target/c1079ab9-e95b-42b9-9d66-c20bb67941db/spark-3c380b3e-bad9-4179-b58d-fb5610c338dc,1,org.apache.spark.serializer.SerializerManager@18d448be) 20/08/08 15:33:50.984 Executor task launch worker for task 3 DEBUG TaskMemoryManager: Task 3 release 4.0 MiB from HybridRowQueue(org.apache.spark.memory.TaskMemoryManager@18a83dda,/home/jenkins/workspace/spark-master-test-python-arm/python/target/c1079ab9-e95b-42b9-9d66-c20bb67941db/spark-3c380b3e-bad9-4179-b58d-fb5610c338dc,1,org.apache.spark.serializer.SerializerManager@18d448be) 20/08/08 15:33:50.993 Executor task launch worker for task 1 DEBUG TaskMemoryManager: Task 1 release 4.0 MiB from HybridRowQueue(org.apache.spark.memory.TaskMemoryManager@360b538e,/home/jenkins/workspace/spark-master-test-python-arm/python/target/c1079ab9-e95b-42b9-9d66-c20bb67941db/spark-3c380b3e-bad9-4179-b58d-fb5610c338dc,1,org.apache.spark.serializer.SerializerManager@18d448be) 20/08/08 15:33:51.001 Executor task launch worker for task 0 DEBUG TaskMemoryManager: Task 0 release 4.0 MiB from HybridRowQueue(org.apache.spark.memory.TaskMemoryManager@11d56d53,/home/jenkins/workspace/spark-master-test-python-arm/python/target/c1079ab9-e95b-42b9-9d66-c20bb67941db/spark-3c380b3e-bad9-4179-b58d-fb5610c338dc,1,org.apache.spark.serializer.SerializerManager@18d448be) 20/08/08 15:33:51.004 Executor task launch worker for task 1 INFO Executor: Finished task 1.0 in stage 0.0 (TID 1). 2189 bytes result sent to driver 20/08/08 15:33:51.004 Executor task launch worker for task 2 INFO Executor: Finished task 2.0 in stage 0.0 (TID 2). 2177 bytes result sent to driver 20/08/08 15:33:51.005 Executor task launch worker for task 0 INFO Executor: Finished task 0.0 in stage 0.0 (TID 0). 2169 bytes result sent to driver 20/08/08 15:33:51.005 Executor task launch worker for task 3 INFO Executor: Finished task 3.0 in stage 0.0 (TID 3). 2191 bytes result sent to driver 20/08/08 15:33:51.006 Executor task launch worker for task 0 DEBUG ExecutorMetricsPoller: stageTCMP: (0, 0) -> 3 20/08/08 15:33:51.006 Executor task launch worker for task 3 DEBUG ExecutorMetricsPoller: stageTCMP: (0, 0) -> 2 20/08/08 15:33:51.007 Executor task launch worker for task 1 DEBUG ExecutorMetricsPoller: stageTCMP: (0, 0) -> 1 20/08/08 15:33:51.008 Executor task launch worker for task 2 DEBUG ExecutorMetricsPoller: removing (0, 0) from stageTCMP 20/08/08 15:33:51.009 dispatcher-event-loop-3 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_0.0, runningTasks: 3 20/08/08 15:33:51.010 dispatcher-event-loop-3 DEBUG TaskSetManager: No tasks for locality level NO_PREF, so moving to locality level ANY 20/08/08 15:33:51.014 dispatcher-event-loop-3 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_0.0, runningTasks: 2 20/08/08 15:33:51.016 task-result-getter-0 INFO TaskSetManager: Finished task 1.0 in stage 0.0 (TID 1) in 1624 ms on 192.168.1.225 (executor driver) (1/4) 20/08/08 15:33:51.018 dispatcher-event-loop-3 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_0.0, runningTasks: 1 20/08/08 15:33:51.018 task-result-getter-1 INFO TaskSetManager: Finished task 0.0 in stage 0.0 (TID 0) in 1670 ms on 192.168.1.225 (executor driver) (2/4) 20/08/08 15:33:51.021 task-result-getter-2 INFO TaskSetManager: Finished task 2.0 in stage 0.0 (TID 2) in 1629 ms on 192.168.1.225 (executor driver) (3/4) 20/08/08 15:33:51.021 dispatcher-event-loop-3 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_0.0, runningTasks: 0 20/08/08 15:33:51.022 task-result-getter-3 INFO TaskSetManager: Finished task 3.0 in stage 0.0 (TID 3) in 1630 ms on 192.168.1.225 (executor driver) (4/4) 20/08/08 15:33:51.022 dag-scheduler-event-loop INFO PythonAccumulatorV2: Connected to AccumulatorServer at host: 127.0.0.1 port: 45717 20/08/08 15:33:51.023 task-result-getter-3 INFO TaskSchedulerImpl: Removed TaskSet 0.0, whose tasks have all completed, from pool 20/08/08 15:33:51.030 dag-scheduler-event-loop INFO DAGScheduler: ResultStage 0 ( at /usr/lib/python3.6/runpy.py:85) finished in 1.944 s 20/08/08 15:33:51.035 dag-scheduler-event-loop DEBUG DAGScheduler: After removal of stage 0, remaining stages = 0 20/08/08 15:33:51.036 dag-scheduler-event-loop INFO DAGScheduler: Job 0 is finished. Cancelling potential speculative or zombie tasks for this job 20/08/08 15:33:51.036 dag-scheduler-event-loop INFO TaskSchedulerImpl: Killing all running tasks in stage 0: Stage finished 20/08/08 15:33:51.039 Thread-4 INFO DAGScheduler: Job 0 finished: at /usr/lib/python3.6/runpy.py:85, took 2.013347 s 20/08/08 15:33:51.213 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanBroadcast(0) 20/08/08 15:33:51.216 Spark Context Cleaner DEBUG ContextCleaner: Cleaning broadcast 0 20/08/08 15:33:51.216 Spark Context Cleaner DEBUG TorrentBroadcast: Unpersisting TorrentBroadcast 0 20/08/08 15:33:51.238 block-manager-storage-async-thread-pool-0 DEBUG BlockManagerStorageEndpoint: removing broadcast 0 20/08/08 15:33:51.238 block-manager-storage-async-thread-pool-0 DEBUG BlockManager: Removing broadcast 0 20/08/08 15:33:51.241 block-manager-storage-async-thread-pool-0 DEBUG BlockManager: Removing block broadcast_0_piece0 20/08/08 15:33:51.243 block-manager-storage-async-thread-pool-0 DEBUG MemoryStore: Block broadcast_0_piece0 of size 7948 dropped from memory (free 384076404) 20/08/08 15:33:51.244 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master broadcast_0_piece0 for BlockManagerId(driver, 192.168.1.225, 38545, None) 20/08/08 15:33:51.250 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Removed broadcast_0_piece0 on 192.168.1.225:38545 in memory (size: 7.8 KiB, free: 366.3 MiB) 20/08/08 15:33:51.250 block-manager-storage-async-thread-pool-0 DEBUG BlockManagerMaster: Updated info of block broadcast_0_piece0 20/08/08 15:33:51.253 block-manager-storage-async-thread-pool-0 DEBUG BlockManager: Told master about block broadcast_0_piece0 20/08/08 15:33:51.254 block-manager-storage-async-thread-pool-0 DEBUG BlockManager: Removing block broadcast_0 20/08/08 15:33:51.254 block-manager-storage-async-thread-pool-0 DEBUG MemoryStore: Block broadcast_0 of size 16984 dropped from memory (free 384093388) 20/08/08 15:33:51.261 block-manager-storage-async-thread-pool-2 DEBUG BlockManagerStorageEndpoint: Done removing broadcast 0, response is 0 20/08/08 15:33:51.265 block-manager-storage-async-thread-pool-2 DEBUG BlockManagerStorageEndpoint: Sent response: 0 to 192.168.1.225:44665 20/08/08 15:33:51.266 Spark Context Cleaner DEBUG ContextCleaner: Cleaned broadcast 0 20/08/08 15:33:51.288 dispatcher-event-loop-2 INFO MapOutputTrackerMasterEndpoint: MapOutputTrackerMasterEndpoint stopped! 20/08/08 15:33:51.303 Thread-4 INFO MemoryStore: MemoryStore cleared 20/08/08 15:33:51.303 Thread-4 INFO BlockManager: BlockManager stopped 20/08/08 15:33:51.312 Thread-4 INFO BlockManagerMaster: BlockManagerMaster stopped 20/08/08 15:33:51.316 dispatcher-event-loop-3 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint: OutputCommitCoordinator stopped! 20/08/08 15:33:51.329 Thread-4 INFO SparkContext: Successfully stopped SparkContext 20/08/08 15:33:51.336 Executor task launch worker for task 0 INFO Executor: Finished task 0.0 in stage 0.0 (TID 0). 2014 bytes result sent to driver 20/08/08 15:33:51.337 Executor task launch worker for task 0 DEBUG ExecutorMetricsPoller: removing (0, 0) from stageTCMP 20/08/08 15:33:51.339 dispatcher-event-loop-0 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_0.0, runningTasks: 0 20/08/08 15:33:51.339 dispatcher-event-loop-0 DEBUG TaskSetManager: No tasks for locality level NO_PREF, so moving to locality level ANY 20/08/08 15:33:51.344 task-result-getter-0 INFO TaskSetManager: Finished task 0.0 in stage 0.0 (TID 0) in 1434 ms on 192.168.1.225 (executor driver) (1/1) 20/08/08 15:33:51.348 task-result-getter-0 INFO TaskSchedulerImpl: Removed TaskSet 0.0, whose tasks have all completed, from pool 20/08/08 15:33:51.351 dag-scheduler-event-loop INFO PythonAccumulatorV2: Connected to AccumulatorServer at host: 127.0.0.1 port: 45985 20/08/08 15:33:51.356 dag-scheduler-event-loop INFO DAGScheduler: ResultStage 0 (head at MetadataUtils.scala:51) finished in 1.795 s 20/08/08 15:33:51.360 dag-scheduler-event-loop DEBUG DAGScheduler: After removal of stage 0, remaining stages = 0 20/08/08 15:33:51.361 dag-scheduler-event-loop INFO DAGScheduler: Job 0 is finished. Cancelling potential speculative or zombie tasks for this job 20/08/08 15:33:51.361 dag-scheduler-event-loop INFO TaskSchedulerImpl: Killing all running tasks in stage 0: Stage finished 20/08/08 15:33:51.363 Thread-4 INFO DAGScheduler: Job 0 finished: head at MetadataUtils.scala:51, took 1.930979 s 20/08/08 15:33:51.392 Thread-4 DEBUG GenerateSafeProjection: code for createexternalrow(newInstance(class org.apache.spark.ml.linalg.VectorUDT).deserialize, StructField(features,org.apache.spark.ml.linalg.VectorUDT@3bfc3ba7,true)): /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificSafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificSafeProjection extends org.apache.spark.sql.catalyst.expressions.codegen.BaseProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private InternalRow mutableRow; /* 009 */ private boolean resultIsNull_0; /* 010 */ private InternalRow[] mutableStateArray_0 = new InternalRow[1]; /* 011 */ /* 012 */ public SpecificSafeProjection(Object[] references) { /* 013 */ this.references = references; /* 014 */ mutableRow = (InternalRow) references[references.length - 1]; /* 015 */ /* 016 */ /* 017 */ } /* 018 */ /* 019 */ public void initialize(int partitionIndex) { /* 020 */ /* 021 */ } /* 022 */ /* 023 */ public java.lang.Object apply(java.lang.Object _i) { /* 024 */ InternalRow i = (InternalRow) _i; /* 025 */ org.apache.spark.sql.Row value_4 = CreateExternalRow_0(i); /* 026 */ if (false) { /* 027 */ mutableRow.setNullAt(0); /* 028 */ } else { /* 029 */ /* 030 */ mutableRow.update(0, value_4); /* 031 */ } /* 032 */ /* 033 */ return mutableRow; /* 034 */ } /* 035 */ /* 036 */ /* 037 */ private org.apache.spark.sql.Row CreateExternalRow_0(InternalRow i) { /* 038 */ Object[] values_0 = new Object[1]; /* 039 */ /* 040 */ final org.apache.spark.ml.linalg.VectorUDT value_2 = false ? /* 041 */ null : new org.apache.spark.ml.linalg.VectorUDT(); /* 042 */ boolean isNull_1 = true; /* 043 */ org.apache.spark.ml.linalg.Vector value_1 = null; /* 044 */ if (!false) { /* 045 */ resultIsNull_0 = false; /* 046 */ if (!resultIsNull_0) { /* 047 */ boolean isNull_3 = i.isNullAt(0); /* 048 */ InternalRow value_3 = isNull_3 ? /* 049 */ null : (i.getStruct(0, 4)); /* 050 */ resultIsNull_0 = isNull_3; /* 051 */ mutableStateArray_0[0] = value_3; /* 052 */ } /* 053 */ /* 054 */ isNull_1 = resultIsNull_0; /* 055 */ if (!isNull_1) { /* 056 */ /* 057 */ Object funcResult_0 = null; /* 058 */ funcResult_0 = value_2.deserialize(mutableStateArray_0[0]); /* 059 */ /* 060 */ if (funcResult_0 != null) { /* 061 */ value_1 = (org.apache.spark.ml.linalg.Vector) funcResult_0; /* 062 */ } else { /* 063 */ isNull_1 = true; /* 064 */ } /* 065 */ /* 066 */ /* 067 */ } /* 068 */ } /* 069 */ if (isNull_1) { /* 070 */ values_0[0] = null; /* 071 */ } else { /* 072 */ values_0[0] = value_1; /* 073 */ } /* 074 */ /* 075 */ final org.apache.spark.sql.Row value_0 = new org.apache.spark.sql.catalyst.expressions.GenericRowWithSchema(values_0, ((org.apache.spark.sql.types.StructType) references[0] /* schema */)); /* 076 */ /* 077 */ return value_0; /* 078 */ } /* 079 */ /* 080 */ } 20/08/08 15:33:51.396 Thread-4 DEBUG CodeGenerator: /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificSafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificSafeProjection extends org.apache.spark.sql.catalyst.expressions.codegen.BaseProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private InternalRow mutableRow; /* 009 */ private boolean resultIsNull_0; /* 010 */ private InternalRow[] mutableStateArray_0 = new InternalRow[1]; /* 011 */ /* 012 */ public SpecificSafeProjection(Object[] references) { /* 013 */ this.references = references; /* 014 */ mutableRow = (InternalRow) references[references.length - 1]; /* 015 */ /* 016 */ /* 017 */ } /* 018 */ /* 019 */ public void initialize(int partitionIndex) { /* 020 */ /* 021 */ } /* 022 */ /* 023 */ public java.lang.Object apply(java.lang.Object _i) { /* 024 */ InternalRow i = (InternalRow) _i; /* 025 */ org.apache.spark.sql.Row value_4 = CreateExternalRow_0(i); /* 026 */ if (false) { /* 027 */ mutableRow.setNullAt(0); /* 028 */ } else { /* 029 */ /* 030 */ mutableRow.update(0, value_4); /* 031 */ } /* 032 */ /* 033 */ return mutableRow; /* 034 */ } /* 035 */ /* 036 */ /* 037 */ private org.apache.spark.sql.Row CreateExternalRow_0(InternalRow i) { /* 038 */ Object[] values_0 = new Object[1]; /* 039 */ /* 040 */ final org.apache.spark.ml.linalg.VectorUDT value_2 = false ? /* 041 */ null : new org.apache.spark.ml.linalg.VectorUDT(); /* 042 */ boolean isNull_1 = true; /* 043 */ org.apache.spark.ml.linalg.Vector value_1 = null; /* 044 */ if (!false) { /* 045 */ resultIsNull_0 = false; /* 046 */ if (!resultIsNull_0) { /* 047 */ boolean isNull_3 = i.isNullAt(0); /* 048 */ InternalRow value_3 = isNull_3 ? /* 049 */ null : (i.getStruct(0, 4)); /* 050 */ resultIsNull_0 = isNull_3; /* 051 */ mutableStateArray_0[0] = value_3; /* 052 */ } /* 053 */ /* 054 */ isNull_1 = resultIsNull_0; /* 055 */ if (!isNull_1) { /* 056 */ /* 057 */ Object funcResult_0 = null; /* 058 */ funcResult_0 = value_2.deserialize(mutableStateArray_0[0]); /* 059 */ /* 060 */ if (funcResult_0 != null) { /* 061 */ value_1 = (org.apache.spark.ml.linalg.Vector) funcResult_0; /* 062 */ } else { /* 063 */ isNull_1 = true; /* 064 */ } /* 065 */ /* 066 */ /* 067 */ } /* 068 */ } /* 069 */ if (isNull_1) { /* 070 */ values_0[0] = null; /* 071 */ } else { /* 072 */ values_0[0] = value_1; /* 073 */ } /* 074 */ /* 075 */ final org.apache.spark.sql.Row value_0 = new org.apache.spark.sql.catalyst.expressions.GenericRowWithSchema(values_0, ((org.apache.spark.sql.types.StructType) references[0] /* schema */)); /* 076 */ /* 077 */ return value_0; /* 078 */ } /* 079 */ /* 080 */ } 20/08/08 15:33:51.417 Thread-4 INFO CodeGenerator: Code generated in 24.893485 ms 20/08/08 15:33:51.440 Thread-4 DEBUG Analyzer$ResolveReferences: Resolving 'prediction to prediction#1 20/08/08 15:33:51.441 Thread-4 DEBUG Analyzer$ResolveReferences: Resolving 'normalizedFeatures to normalizedFeatures#11 20/08/08 15:33:51.441 Thread-4 DEBUG Analyzer$ResolveReferences: Resolving 'weightCol to weightCol#5 20/08/08 15:33:51.480 Executor task launch worker for task 0 DEBUG GenerateSafeProjection: code for createexternalrow(input[0, int, false], input[1, int, false], input[2, float, true], StructField(UDF(user),IntegerType,false), StructField(UDF(item),IntegerType,false), StructField(rating,FloatType,true)): /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificSafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificSafeProjection extends org.apache.spark.sql.catalyst.expressions.codegen.BaseProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private InternalRow mutableRow; /* 009 */ /* 010 */ /* 011 */ public SpecificSafeProjection(Object[] references) { /* 012 */ this.references = references; /* 013 */ mutableRow = (InternalRow) references[references.length - 1]; /* 014 */ /* 015 */ } /* 016 */ /* 017 */ public void initialize(int partitionIndex) { /* 018 */ /* 019 */ } /* 020 */ /* 021 */ public java.lang.Object apply(java.lang.Object _i) { /* 022 */ InternalRow i = (InternalRow) _i; /* 023 */ Object[] values_0 = new Object[3]; /* 024 */ /* 025 */ int value_1 = i.getInt(0); /* 026 */ if (false) { /* 027 */ values_0[0] = null; /* 028 */ } else { /* 029 */ values_0[0] = value_1; /* 030 */ } /* 031 */ /* 032 */ int value_2 = i.getInt(1); /* 033 */ if (false) { /* 034 */ values_0[1] = null; /* 035 */ } else { /* 036 */ values_0[1] = value_2; /* 037 */ } /* 038 */ /* 039 */ boolean isNull_3 = i.isNullAt(2); /* 040 */ float value_3 = isNull_3 ? /* 041 */ -1.0f : (i.getFloat(2)); /* 042 */ if (isNull_3) { /* 043 */ values_0[2] = null; /* 044 */ } else { /* 045 */ values_0[2] = value_3; /* 046 */ } /* 047 */ /* 048 */ final org.apache.spark.sql.Row value_0 = new org.apache.spark.sql.catalyst.expressions.GenericRowWithSchema(values_0, ((org.apache.spark.sql.types.StructType) references[0] /* schema */)); /* 049 */ if (false) { /* 050 */ mutableRow.setNullAt(0); /* 051 */ } else { /* 052 */ /* 053 */ mutableRow.update(0, value_0); /* 054 */ } /* 055 */ /* 056 */ return mutableRow; /* 057 */ } /* 058 */ /* 059 */ /* 060 */ } 20/08/08 15:33:51.485 Executor task launch worker for task 0 DEBUG CodeGenerator: /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificSafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificSafeProjection extends org.apache.spark.sql.catalyst.expressions.codegen.BaseProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private InternalRow mutableRow; /* 009 */ /* 010 */ /* 011 */ public SpecificSafeProjection(Object[] references) { /* 012 */ this.references = references; /* 013 */ mutableRow = (InternalRow) references[references.length - 1]; /* 014 */ /* 015 */ } /* 016 */ /* 017 */ public void initialize(int partitionIndex) { /* 018 */ /* 019 */ } /* 020 */ /* 021 */ public java.lang.Object apply(java.lang.Object _i) { /* 022 */ InternalRow i = (InternalRow) _i; /* 023 */ Object[] values_0 = new Object[3]; /* 024 */ /* 025 */ int value_1 = i.getInt(0); /* 026 */ if (false) { /* 027 */ values_0[0] = null; /* 028 */ } else { /* 029 */ values_0[0] = value_1; /* 030 */ } /* 031 */ /* 032 */ int value_2 = i.getInt(1); /* 033 */ if (false) { /* 034 */ values_0[1] = null; /* 035 */ } else { /* 036 */ values_0[1] = value_2; /* 037 */ } /* 038 */ /* 039 */ boolean isNull_3 = i.isNullAt(2); /* 040 */ float value_3 = isNull_3 ? /* 041 */ -1.0f : (i.getFloat(2)); /* 042 */ if (isNull_3) { /* 043 */ values_0[2] = null; /* 044 */ } else { /* 045 */ values_0[2] = value_3; /* 046 */ } /* 047 */ /* 048 */ final org.apache.spark.sql.Row value_0 = new org.apache.spark.sql.catalyst.expressions.GenericRowWithSchema(values_0, ((org.apache.spark.sql.types.StructType) references[0] /* schema */)); /* 049 */ if (false) { /* 050 */ mutableRow.setNullAt(0); /* 051 */ } else { /* 052 */ /* 053 */ mutableRow.update(0, value_0); /* 054 */ } /* 055 */ /* 056 */ return mutableRow; /* 057 */ } /* 058 */ /* 059 */ /* 060 */ } 20/08/08 15:33:51.504 Executor task launch worker for task 0 INFO CodeGenerator: Code generated in 24.076619 ms 20/08/08 15:33:51.539 Executor task launch worker for task 0 INFO Executor: Finished task 0.0 in stage 0.0 (TID 0). 1997 bytes result sent to driver 20/08/08 15:33:51.540 Executor task launch worker for task 0 DEBUG ExecutorMetricsPoller: removing (0, 0) from stageTCMP 20/08/08 15:33:51.542 dispatcher-event-loop-0 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_0.0, runningTasks: 0 20/08/08 15:33:51.543 dispatcher-event-loop-0 DEBUG TaskSetManager: No tasks for locality level NO_PREF, so moving to locality level ANY 20/08/08 15:33:51.548 task-result-getter-0 INFO TaskSetManager: Finished task 0.0 in stage 0.0 (TID 0) in 1666 ms on 192.168.1.225 (executor driver) (1/1) 20/08/08 15:33:51.550 task-result-getter-0 INFO TaskSchedulerImpl: Removed TaskSet 0.0, whose tasks have all completed, from pool 20/08/08 15:33:51.554 dag-scheduler-event-loop INFO PythonAccumulatorV2: Connected to AccumulatorServer at host: 127.0.0.1 port: 53041 20/08/08 15:33:51.559 dag-scheduler-event-loop INFO DAGScheduler: ResultStage 0 (isEmpty at ALS.scala:947) finished in 1.997 s 20/08/08 15:33:51.564 dag-scheduler-event-loop DEBUG DAGScheduler: After removal of stage 0, remaining stages = 0 20/08/08 15:33:51.565 dag-scheduler-event-loop INFO DAGScheduler: Job 0 is finished. Cancelling potential speculative or zombie tasks for this job 20/08/08 15:33:51.565 dag-scheduler-event-loop INFO TaskSchedulerImpl: Killing all running tasks in stage 0: Stage finished 20/08/08 15:33:51.568 Thread-4 INFO DAGScheduler: Job 0 finished: isEmpty at ALS.scala:947, took 2.119387 s 20/08/08 15:33:51.574 Thread-4 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$partitionRatings$1 20/08/08 15:33:51.576 Thread-4 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$partitionRatings$1) is now cleaned +++ 20/08/08 15:33:51.597 Thread-4 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$groupByKey$2 20/08/08 15:33:51.601 Thread-4 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$groupByKey$2) is now cleaned +++ 20/08/08 15:33:51.611 Thread-4 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$groupByKey$3 20/08/08 15:33:51.615 Thread-4 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$groupByKey$3) is now cleaned +++ 20/08/08 15:33:51.616 Thread-4 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$groupByKey$4 20/08/08 15:33:51.619 Thread-4 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$groupByKey$4) is now cleaned +++ 20/08/08 15:33:51.622 Thread-4 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$partitionRatings$7 20/08/08 15:33:51.624 Thread-4 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$partitionRatings$7) is now cleaned +++ 20/08/08 15:33:51.630 Thread-4 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$makeBlocks$1 20/08/08 15:33:51.632 Thread-4 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$makeBlocks$1) is now cleaned +++ 20/08/08 15:33:51.634 Thread-4 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$groupByKey$2 20/08/08 15:33:51.637 Thread-4 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$groupByKey$2) is now cleaned +++ 20/08/08 15:33:51.651 Thread-4 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$groupByKey$3 20/08/08 15:33:51.654 Thread-4 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$groupByKey$3) is now cleaned +++ 20/08/08 15:33:51.655 Thread-4 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$groupByKey$4 20/08/08 15:33:51.658 Thread-4 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$groupByKey$4) is now cleaned +++ 20/08/08 15:33:51.660 Thread-4 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$makeBlocks$5 20/08/08 15:33:51.662 Thread-4 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$makeBlocks$5) is now cleaned +++ 20/08/08 15:33:51.664 Thread-4 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$makeBlocks$7 20/08/08 15:33:51.666 Thread-4 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$makeBlocks$7) is now cleaned +++ 20/08/08 15:33:51.668 Thread-4 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$count$1$adapted 20/08/08 15:33:51.675 Thread-4 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$count$1$adapted) is now cleaned +++ 20/08/08 15:33:51.676 Thread-4 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$runJob$5 20/08/08 15:33:51.681 Thread-4 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$runJob$5) is now cleaned +++ 20/08/08 15:33:51.682 Thread-4 INFO SparkContext: Starting job: count at ALS.scala:960 20/08/08 15:33:51.682 Thread-4 DEBUG WholeStageCodegenExec: /* 001 */ public Object generate(Object[] references) { /* 002 */ return new GeneratedIteratorForCodegenStage1(references); /* 003 */ } /* 004 */ /* 005 */ // codegenStageId=1 /* 006 */ final class GeneratedIteratorForCodegenStage1 extends org.apache.spark.sql.execution.BufferedRowIterator { /* 007 */ private Object[] references; /* 008 */ private scala.collection.Iterator[] inputs; /* 009 */ private scala.collection.Iterator rdd_input_0; /* 010 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[] rdd_mutableStateArray_0 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[4]; /* 011 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[] rdd_mutableStateArray_1 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[4]; /* 012 */ /* 013 */ public GeneratedIteratorForCodegenStage1(Object[] references) { /* 014 */ this.references = references; /* 015 */ } /* 016 */ /* 017 */ public void init(int index, scala.collection.Iterator[] inputs) { /* 018 */ partitionIndex = index; /* 019 */ this.inputs = inputs; /* 020 */ rdd_input_0 = inputs[0]; /* 021 */ rdd_mutableStateArray_0[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(2, 32); /* 022 */ rdd_mutableStateArray_0[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(rdd_mutableStateArray_0[0], 4); /* 023 */ rdd_mutableStateArray_1[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[1], 4); /* 024 */ rdd_mutableStateArray_1[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[1], 8); /* 025 */ rdd_mutableStateArray_0[2] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(3, 32); /* 026 */ rdd_mutableStateArray_0[3] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(rdd_mutableStateArray_0[2], 4); /* 027 */ rdd_mutableStateArray_1[2] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[3], 4); /* 028 */ rdd_mutableStateArray_1[3] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[3], 8); /* 029 */ /* 030 */ } /* 031 */ /* 032 */ private void project_writeFields_0_1(InternalRow project_tmpInput_0) { /* 033 */ if ((project_tmpInput_0.isNullAt(3))) { /* 034 */ rdd_mutableStateArray_0[3].setNullAt(3); /* 035 */ } else { /* 036 */ // Remember the current cursor so that we can calculate how many bytes are /* 037 */ // written later. /* 038 */ final int project_previousCursor_2 = rdd_mutableStateArray_0[3].cursor(); /* 039 */ /* 040 */ final ArrayData project_tmpInput_2 = (project_tmpInput_0.getArray(3)); /* 041 */ if (project_tmpInput_2 instanceof UnsafeArrayData) { /* 042 */ rdd_mutableStateArray_0[3].write((UnsafeArrayData) project_tmpInput_2); /* 043 */ } else { /* 044 */ final int project_numElements_1 = project_tmpInput_2.numElements(); /* 045 */ rdd_mutableStateArray_1[3].initialize(project_numElements_1); /* 046 */ /* 047 */ for (int project_index_1 = 0; project_index_1 < project_numElements_1; project_index_1++) { /* 048 */ rdd_mutableStateArray_1[3].write(project_index_1, project_tmpInput_2.getDouble(project_index_1)); /* 049 */ } /* 050 */ } /* 051 */ /* 052 */ rdd_mutableStateArray_0[3].setOffsetAndSizeFromPreviousCursor(3, project_previousCursor_2); /* 053 */ } /* 054 */ /* 055 */ } /* 056 */ /* 057 */ private void project_writeFields_0_0(InternalRow project_tmpInput_0) { /* 058 */ rdd_mutableStateArray_0[3].write(0, (project_tmpInput_0.getByte(0))); /* 059 */ /* 060 */ if ((project_tmpInput_0.isNullAt(1))) { /* 061 */ rdd_mutableStateArray_0[3].setNullAt(1); /* 062 */ } else { /* 063 */ rdd_mutableStateArray_0[3].write(1, (project_tmpInput_0.getInt(1))); /* 064 */ } /* 065 */ /* 066 */ if ((project_tmpInput_0.isNullAt(2))) { /* 067 */ rdd_mutableStateArray_0[3].setNullAt(2); /* 068 */ } else { /* 069 */ // Remember the current cursor so that we can calculate how many bytes are /* 070 */ // written later. /* 071 */ final int project_previousCursor_1 = rdd_mutableStateArray_0[3].cursor(); /* 072 */ /* 073 */ final ArrayData project_tmpInput_1 = (project_tmpInput_0.getArray(2)); /* 074 */ if (project_tmpInput_1 instanceof UnsafeArrayData) { /* 075 */ rdd_mutableStateArray_0[3].write((UnsafeArrayData) project_tmpInput_1); /* 076 */ } else { /* 077 */ final int project_numElements_0 = project_tmpInput_1.numElements(); /* 078 */ rdd_mutableStateArray_1[2].initialize(project_numElements_0); /* 079 */ /* 080 */ for (int project_index_0 = 0; project_index_0 < project_numElements_0; project_index_0++) { /* 081 */ rdd_mutableStateArray_1[2].write(project_index_0, project_tmpInput_1.getInt(project_index_0)); /* 082 */ } /* 083 */ } /* 084 */ /* 085 */ rdd_mutableStateArray_0[3].setOffsetAndSizeFromPreviousCursor(2, project_previousCursor_1); /* 086 */ } /* 087 */ /* 088 */ } /* 089 */ /* 090 */ protected void processNext() throws java.io.IOException { /* 091 */ while ( rdd_input_0.hasNext()) { /* 092 */ InternalRow rdd_row_0 = (InternalRow) rdd_input_0.next(); /* 093 */ ((org.apache.spark.sql.execution.metric.SQLMetric) references[0] /* numOutputRows */).add(1); /* 094 */ boolean rdd_isNull_1 = rdd_row_0.isNullAt(1); /* 095 */ double rdd_value_1 = rdd_isNull_1 ? /* 096 */ -1.0 : (rdd_row_0.getDouble(1)); /* 097 */ boolean rdd_isNull_0 = rdd_row_0.isNullAt(0); /* 098 */ InternalRow rdd_value_0 = rdd_isNull_0 ? /* 099 */ null : (rdd_row_0.getStruct(0, 4)); /* 100 */ /* 101 */ Object project_arg_0 = null; /* 102 */ if (rdd_isNull_0) { /* 103 */ project_arg_0 = ((scala.Function1[]) references[1] /* converters */)[0].apply(null); /* 104 */ } else { /* 105 */ project_arg_0 = ((scala.Function1[]) references[1] /* converters */)[0].apply(rdd_value_0); /* 106 */ } /* 107 */ /* 108 */ InternalRow project_result_0 = null; /* 109 */ try { /* 110 */ project_result_0 = (InternalRow)((scala.Function1[]) references[1] /* converters */)[1].apply(((scala.Function1) references[3] /* udf */).apply(project_arg_0)); /* 111 */ } catch (Exception e) { /* 112 */ throw new org.apache.spark.SparkException(((java.lang.String) references[2] /* errMsg */), e); /* 113 */ } /* 114 */ /* 115 */ boolean project_isNull_1 = project_result_0 == null; /* 116 */ InternalRow project_value_1 = null; /* 117 */ if (!project_isNull_1) { /* 118 */ project_value_1 = project_result_0; /* 119 */ } /* 120 */ rdd_mutableStateArray_0[2].reset(); /* 121 */ /* 122 */ rdd_mutableStateArray_0[2].zeroOutNullBytes(); /* 123 */ /* 124 */ if (rdd_isNull_1) { /* 125 */ rdd_mutableStateArray_0[2].setNullAt(0); /* 126 */ } else { /* 127 */ rdd_mutableStateArray_0[2].write(0, rdd_value_1); /* 128 */ } /* 129 */ /* 130 */ if (project_isNull_1) { /* 131 */ rdd_mutableStateArray_0[2].setNullAt(1); /* 132 */ } else { /* 133 */ final InternalRow project_tmpInput_0 = project_value_1; /* 134 */ if (project_tmpInput_0 instanceof UnsafeRow) { /* 135 */ rdd_mutableStateArray_0[2].write(1, (UnsafeRow) project_tmpInput_0); /* 136 */ } else { /* 137 */ // Remember the current cursor so that we can calculate how many bytes are /* 138 */ // written later. /* 139 */ final int project_previousCursor_0 = rdd_mutableStateArray_0[2].cursor(); /* 140 */ /* 141 */ rdd_mutableStateArray_0[3].resetRowWriter(); /* 142 */ project_writeFields_0_0(project_tmpInput_0); /* 143 */ project_writeFields_0_1(project_tmpInput_0); /* 144 */ /* 145 */ rdd_mutableStateArray_0[2].setOffsetAndSizeFromPreviousCursor(1, project_previousCursor_0); /* 146 */ } /* 147 */ } /* 148 */ /* 149 */ rdd_mutableStateArray_0[2].write(2, 1.0D); /* 150 */ append((rdd_mutableStateArray_0[2].getRow())); /* 151 */ if (shouldStop()) return; /* 152 */ } /* 153 */ } /* 154 */ /* 155 */ } 20/08/08 15:33:51.686 dag-scheduler-event-loop DEBUG DAGScheduler: Merging stage rdd profiles: Set() 20/08/08 15:33:51.688 dag-scheduler-event-loop DEBUG DAGScheduler: Merging stage rdd profiles: Set() 20/08/08 15:33:51.690 dag-scheduler-event-loop INFO DAGScheduler: Registering RDD 14 (mapPartitions at ALS.scala:1356) as input to shuffle 1 20/08/08 15:33:51.692 dag-scheduler-event-loop DEBUG DAGScheduler: Merging stage rdd profiles: Set() 20/08/08 15:33:51.693 dag-scheduler-event-loop INFO DAGScheduler: Registering RDD 17 (map at ALS.scala:1599) as input to shuffle 0 20/08/08 15:33:51.694 dag-scheduler-event-loop INFO DAGScheduler: Got job 1 (count at ALS.scala:960) with 10 output partitions 20/08/08 15:33:51.694 dag-scheduler-event-loop INFO DAGScheduler: Final stage: ResultStage 3 (count at ALS.scala:960) 20/08/08 15:33:51.694 dag-scheduler-event-loop INFO DAGScheduler: Parents of final stage: List(ShuffleMapStage 2) 20/08/08 15:33:51.696 Thread-4 DEBUG CodeGenerator: /* 001 */ public Object generate(Object[] references) { /* 002 */ return new GeneratedIteratorForCodegenStage1(references); /* 003 */ } /* 004 */ /* 005 */ // codegenStageId=1 /* 006 */ final class GeneratedIteratorForCodegenStage1 extends org.apache.spark.sql.execution.BufferedRowIterator { /* 007 */ private Object[] references; /* 008 */ private scala.collection.Iterator[] inputs; /* 009 */ private scala.collection.Iterator rdd_input_0; /* 010 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[] rdd_mutableStateArray_0 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[4]; /* 011 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[] rdd_mutableStateArray_1 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[4]; /* 012 */ /* 013 */ public GeneratedIteratorForCodegenStage1(Object[] references) { /* 014 */ this.references = references; /* 015 */ } /* 016 */ /* 017 */ public void init(int index, scala.collection.Iterator[] inputs) { /* 018 */ partitionIndex = index; /* 019 */ this.inputs = inputs; /* 020 */ rdd_input_0 = inputs[0]; /* 021 */ rdd_mutableStateArray_0[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(2, 32); /* 022 */ rdd_mutableStateArray_0[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(rdd_mutableStateArray_0[0], 4); /* 023 */ rdd_mutableStateArray_1[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[1], 4); /* 024 */ rdd_mutableStateArray_1[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[1], 8); /* 025 */ rdd_mutableStateArray_0[2] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(3, 32); /* 026 */ rdd_mutableStateArray_0[3] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(rdd_mutableStateArray_0[2], 4); /* 027 */ rdd_mutableStateArray_1[2] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[3], 4); /* 028 */ rdd_mutableStateArray_1[3] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[3], 8); /* 029 */ /* 030 */ } /* 031 */ /* 032 */ private void project_writeFields_0_1(InternalRow project_tmpInput_0) { /* 033 */ if ((project_tmpInput_0.isNullAt(3))) { /* 034 */ rdd_mutableStateArray_0[3].setNullAt(3); /* 035 */ } else { /* 036 */ // Remember the current cursor so that we can calculate how many bytes are /* 037 */ // written later. /* 038 */ final int project_previousCursor_2 = rdd_mutableStateArray_0[3].cursor(); /* 039 */ /* 040 */ final ArrayData project_tmpInput_2 = (project_tmpInput_0.getArray(3)); /* 041 */ if (project_tmpInput_2 instanceof UnsafeArrayData) { /* 042 */ rdd_mutableStateArray_0[3].write((UnsafeArrayData) project_tmpInput_2); /* 043 */ } else { /* 044 */ final int project_numElements_1 = project_tmpInput_2.numElements(); /* 045 */ rdd_mutableStateArray_1[3].initialize(project_numElements_1); /* 046 */ /* 047 */ for (int project_index_1 = 0; project_index_1 < project_numElements_1; project_index_1++) { /* 048 */ rdd_mutableStateArray_1[3].write(project_index_1, project_tmpInput_2.getDouble(project_index_1)); /* 049 */ } /* 050 */ } /* 051 */ /* 052 */ rdd_mutableStateArray_0[3].setOffsetAndSizeFromPreviousCursor(3, project_previousCursor_2); /* 053 */ } /* 054 */ /* 055 */ } /* 056 */ /* 057 */ private void project_writeFields_0_0(InternalRow project_tmpInput_0) { /* 058 */ rdd_mutableStateArray_0[3].write(0, (project_tmpInput_0.getByte(0))); /* 059 */ /* 060 */ if ((project_tmpInput_0.isNullAt(1))) { /* 061 */ rdd_mutableStateArray_0[3].setNullAt(1); /* 062 */ } else { /* 063 */ rdd_mutableStateArray_0[3].write(1, (project_tmpInput_0.getInt(1))); /* 064 */ } /* 065 */ /* 066 */ if ((project_tmpInput_0.isNullAt(2))) { /* 067 */ rdd_mutableStateArray_0[3].setNullAt(2); /* 068 */ } else { /* 069 */ // Remember the current cursor so that we can calculate how many bytes are /* 070 */ // written later. /* 071 */ final int project_previousCursor_1 = rdd_mutableStateArray_0[3].cursor(); /* 072 */ /* 073 */ final ArrayData project_tmpInput_1 = (project_tmpInput_0.getArray(2)); /* 074 */ if (project_tmpInput_1 instanceof UnsafeArrayData) { /* 075 */ rdd_mutableStateArray_0[3].write((UnsafeArrayData) project_tmpInput_1); /* 076 */ } else { /* 077 */ final int project_numElements_0 = project_tmpInput_1.numElements(); /* 078 */ rdd_mutableStateArray_1[2].initialize(project_numElements_0); /* 079 */ /* 080 */ for (int project_index_0 = 0; project_index_0 < project_numElements_0; project_index_0++) { /* 081 */ rdd_mutableStateArray_1[2].write(project_index_0, project_tmpInput_1.getInt(project_index_0)); /* 082 */ } /* 083 */ } /* 084 */ /* 085 */ rdd_mutableStateArray_0[3].setOffsetAndSizeFromPreviousCursor(2, project_previousCursor_1); /* 086 */ } /* 087 */ /* 088 */ } /* 089 */ /* 090 */ protected void processNext() throws java.io.IOException { /* 091 */ while ( rdd_input_0.hasNext()) { /* 092 */ InternalRow rdd_row_0 = (InternalRow) rdd_input_0.next(); /* 093 */ ((org.apache.spark.sql.execution.metric.SQLMetric) references[0] /* numOutputRows */).add(1); /* 094 */ boolean rdd_isNull_1 = rdd_row_0.isNullAt(1); /* 095 */ double rdd_value_1 = rdd_isNull_1 ? /* 096 */ -1.0 : (rdd_row_0.getDouble(1)); /* 097 */ boolean rdd_isNull_0 = rdd_row_0.isNullAt(0); /* 098 */ InternalRow rdd_value_0 = rdd_isNull_0 ? /* 099 */ null : (rdd_row_0.getStruct(0, 4)); /* 100 */ /* 101 */ Object project_arg_0 = null; /* 102 */ if (rdd_isNull_0) { /* 103 */ project_arg_0 = ((scala.Function1[]) references[1] /* converters */)[0].apply(null); /* 104 */ } else { /* 105 */ project_arg_0 = ((scala.Function1[]) references[1] /* converters */)[0].apply(rdd_value_0); /* 106 */ } /* 107 */ /* 108 */ InternalRow project_result_0 = null; /* 109 */ try { /* 110 */ project_result_0 = (InternalRow)((scala.Function1[]) references[1] /* converters */)[1].apply(((scala.Function1) references[3] /* udf */).apply(project_arg_0)); /* 111 */ } catch (Exception e) { /* 112 */ throw new org.apache.spark.SparkException(((java.lang.String) references[2] /* errMsg */), e); /* 113 */ } /* 114 */ /* 115 */ boolean project_isNull_1 = project_result_0 == null; /* 116 */ InternalRow project_value_1 = null; /* 117 */ if (!project_isNull_1) { /* 118 */ project_value_1 = project_result_0; /* 119 */ } /* 120 */ rdd_mutableStateArray_0[2].reset(); /* 121 */ /* 122 */ rdd_mutableStateArray_0[2].zeroOutNullBytes(); /* 123 */ /* 124 */ if (rdd_isNull_1) { /* 125 */ rdd_mutableStateArray_0[2].setNullAt(0); /* 126 */ } else { /* 127 */ rdd_mutableStateArray_0[2].write(0, rdd_value_1); /* 128 */ } /* 129 */ /* 130 */ if (project_isNull_1) { /* 131 */ rdd_mutableStateArray_0[2].setNullAt(1); /* 132 */ } else { /* 133 */ final InternalRow project_tmpInput_0 = project_value_1; /* 134 */ if (project_tmpInput_0 instanceof UnsafeRow) { /* 135 */ rdd_mutableStateArray_0[2].write(1, (UnsafeRow) project_tmpInput_0); /* 136 */ } else { /* 137 */ // Remember the current cursor so that we can calculate how many bytes are /* 138 */ // written later. /* 139 */ final int project_previousCursor_0 = rdd_mutableStateArray_0[2].cursor(); /* 140 */ /* 141 */ rdd_mutableStateArray_0[3].resetRowWriter(); /* 142 */ project_writeFields_0_0(project_tmpInput_0); /* 143 */ project_writeFields_0_1(project_tmpInput_0); /* 144 */ /* 145 */ rdd_mutableStateArray_0[2].setOffsetAndSizeFromPreviousCursor(1, project_previousCursor_0); /* 146 */ } /* 147 */ } /* 148 */ /* 149 */ rdd_mutableStateArray_0[2].write(2, 1.0D); /* 150 */ append((rdd_mutableStateArray_0[2].getRow())); /* 151 */ if (shouldStop()) return; /* 152 */ } /* 153 */ } /* 154 */ /* 155 */ } 20/08/08 15:33:51.708 dag-scheduler-event-loop INFO DAGScheduler: Missing parents: List(ShuffleMapStage 2) 20/08/08 15:33:51.709 dag-scheduler-event-loop DEBUG DAGScheduler: submitStage(ResultStage 3 (name=count at ALS.scala:960;jobs=1)) 20/08/08 15:33:51.709 dag-scheduler-event-loop DEBUG DAGScheduler: missing: List(ShuffleMapStage 2) 20/08/08 15:33:51.710 dag-scheduler-event-loop DEBUG DAGScheduler: submitStage(ShuffleMapStage 2 (name=map at ALS.scala:1599;jobs=1)) 20/08/08 15:33:51.712 dag-scheduler-event-loop DEBUG DAGScheduler: missing: List(ShuffleMapStage 1) 20/08/08 15:33:51.712 dag-scheduler-event-loop DEBUG DAGScheduler: submitStage(ShuffleMapStage 1 (name=mapPartitions at ALS.scala:1356;jobs=1)) 20/08/08 15:33:51.713 dag-scheduler-event-loop DEBUG DAGScheduler: missing: List() 20/08/08 15:33:51.713 dag-scheduler-event-loop INFO DAGScheduler: Submitting ShuffleMapStage 1 (MapPartitionsRDD[14] at mapPartitions at ALS.scala:1356), which has no missing parents 20/08/08 15:33:51.713 dag-scheduler-event-loop DEBUG DAGScheduler: submitMissingTasks(ShuffleMapStage 1) 20/08/08 15:33:51.736 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_1 stored as values in memory (estimated size 30.8 KiB, free 366.2 MiB) 20/08/08 15:33:51.736 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_1 locally took 1 ms 20/08/08 15:33:51.737 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_1 without replication took 1 ms 20/08/08 15:33:51.738 Thread-4 INFO CodeGenerator: Code generated in 55.410715 ms 20/08/08 15:33:51.738 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_1_piece0 stored as bytes in memory (estimated size 14.6 KiB, free 366.2 MiB) 20/08/08 15:33:51.738 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master broadcast_1_piece0 for BlockManagerId(driver, 192.168.1.225, 38159, None) 20/08/08 15:33:51.739 Thread-4 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$doExecute$4$adapted 20/08/08 15:33:51.739 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added broadcast_1_piece0 in memory on 192.168.1.225:38159 (size: 14.6 KiB, free: 366.3 MiB) 20/08/08 15:33:51.740 Thread-4 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$doExecute$4$adapted) is now cleaned +++ 20/08/08 15:33:51.741 dag-scheduler-event-loop DEBUG BlockManagerMaster: Updated info of block broadcast_1_piece0 20/08/08 15:33:51.741 dag-scheduler-event-loop DEBUG BlockManager: Told master about block broadcast_1_piece0 20/08/08 15:33:51.741 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_1_piece0 locally took 3 ms 20/08/08 15:33:51.741 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_1_piece0 without replication took 3 ms 20/08/08 15:33:51.742 dag-scheduler-event-loop INFO SparkContext: Created broadcast 1 from broadcast at DAGScheduler.scala:1348 20/08/08 15:33:51.746 dag-scheduler-event-loop INFO DAGScheduler: Submitting 4 missing tasks from ShuffleMapStage 1 (MapPartitionsRDD[14] at mapPartitions at ALS.scala:1356) (first 15 tasks are for partitions Vector(0, 1, 2, 3)) 20/08/08 15:33:51.746 dag-scheduler-event-loop INFO TaskSchedulerImpl: Adding task set 1.0 with 4 tasks resource profile 0 20/08/08 15:33:51.747 dag-scheduler-event-loop DEBUG TaskSetManager: Epoch for TaskSet 1.0: 0 20/08/08 15:33:51.747 dag-scheduler-event-loop DEBUG TaskSetManager: Adding pending tasks took 0 ms 20/08/08 15:33:51.747 dag-scheduler-event-loop DEBUG TaskSetManager: Valid locality levels for TaskSet 1.0: NO_PREF, ANY 20/08/08 15:33:51.748 dispatcher-event-loop-1 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_1.0, runningTasks: 0 20/08/08 15:33:51.750 dispatcher-event-loop-1 INFO TaskSetManager: Starting task 0.0 in stage 1.0 (TID 1, 192.168.1.225, executor driver, partition 0, PROCESS_LOCAL, 7362 bytes) taskResourceAssignments Map() 20/08/08 15:33:51.751 dispatcher-event-loop-1 INFO TaskSetManager: Starting task 1.0 in stage 1.0 (TID 2, 192.168.1.225, executor driver, partition 1, PROCESS_LOCAL, 7402 bytes) taskResourceAssignments Map() 20/08/08 15:33:51.751 dispatcher-event-loop-1 INFO TaskSetManager: Starting task 2.0 in stage 1.0 (TID 3, 192.168.1.225, executor driver, partition 2, PROCESS_LOCAL, 7362 bytes) taskResourceAssignments Map() 20/08/08 15:33:51.751 dispatcher-event-loop-1 INFO TaskSetManager: Starting task 3.0 in stage 1.0 (TID 4, 192.168.1.225, executor driver, partition 3, PROCESS_LOCAL, 7402 bytes) taskResourceAssignments Map() 20/08/08 15:33:51.752 Executor task launch worker for task 1 INFO Executor: Running task 0.0 in stage 1.0 (TID 1) 20/08/08 15:33:51.760 Executor task launch worker for task 1 DEBUG ExecutorMetricsPoller: stageTCMP: (1, 0) -> 1 20/08/08 15:33:51.762 Executor task launch worker for task 2 INFO Executor: Running task 1.0 in stage 1.0 (TID 2) 20/08/08 15:33:51.763 Executor task launch worker for task 1 DEBUG BlockManager: Getting local block broadcast_1 20/08/08 15:33:51.763 Executor task launch worker for task 4 INFO Executor: Running task 3.0 in stage 1.0 (TID 4) 20/08/08 15:33:51.763 Executor task launch worker for task 1 DEBUG BlockManager: Level for block broadcast_1 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:51.763 Executor task launch worker for task 2 DEBUG ExecutorMetricsPoller: stageTCMP: (1, 0) -> 2 20/08/08 15:33:51.764 Executor task launch worker for task 4 DEBUG ExecutorMetricsPoller: stageTCMP: (1, 0) -> 3 20/08/08 15:33:51.765 Executor task launch worker for task 3 INFO Executor: Running task 2.0 in stage 1.0 (TID 3) 20/08/08 15:33:51.775 Executor task launch worker for task 3 DEBUG ExecutorMetricsPoller: stageTCMP: (1, 0) -> 4 20/08/08 15:33:51.816 Thread-4 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$rdd$1 20/08/08 15:33:51.820 Thread-4 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$rdd$1) is now cleaned +++ 20/08/08 15:33:51.833 Executor task launch worker for task 0 INFO PythonRunner: Times: total = 636, boot = 595, init = 40, finish = 1 20/08/08 15:33:51.846 Thread-4 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$computeClusterStats$5 20/08/08 15:33:51.847 Thread-4 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$computeClusterStats$5) is now cleaned +++ 20/08/08 15:33:51.866 Executor task launch worker for task 0 INFO Executor: Finished task 0.0 in stage 0.0 (TID 0). 2085 bytes result sent to driver 20/08/08 15:33:51.869 Executor task launch worker for task 2 DEBUG GenerateSafeProjection: code for mapobjects(lambdavariable(MapObject, StringType, true, -1), lambdavariable(MapObject, StringType, true, -1).toString, input[0, array, true], Some(interface scala.collection.Seq)): /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificSafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificSafeProjection extends org.apache.spark.sql.catalyst.expressions.codegen.BaseProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private InternalRow mutableRow; /* 009 */ private UTF8String value_MapObject_lambda_variable_1; /* 010 */ private boolean isNull_MapObject_lambda_variable_1; /* 011 */ private boolean globalIsNull_0; /* 012 */ /* 013 */ public SpecificSafeProjection(Object[] references) { /* 014 */ this.references = references; /* 015 */ mutableRow = (InternalRow) references[references.length - 1]; /* 016 */ /* 017 */ /* 018 */ } /* 019 */ /* 020 */ public void initialize(int partitionIndex) { /* 021 */ /* 022 */ } /* 023 */ /* 024 */ public java.lang.Object apply(java.lang.Object _i) { /* 025 */ InternalRow i = (InternalRow) _i; /* 026 */ scala.collection.Seq value_3 = MapObjects_0(i); /* 027 */ if (globalIsNull_0) { /* 028 */ mutableRow.setNullAt(0); /* 029 */ } else { /* 030 */ /* 031 */ mutableRow.update(0, value_3); /* 032 */ } /* 033 */ /* 034 */ return mutableRow; /* 035 */ } /* 036 */ /* 037 */ /* 038 */ private scala.collection.Seq MapObjects_0(InternalRow i) { /* 039 */ boolean isNull_1 = i.isNullAt(0); /* 040 */ ArrayData value_1 = isNull_1 ? /* 041 */ null : (i.getArray(0)); /* 042 */ scala.collection.Seq value_0 = null; /* 043 */ /* 044 */ if (!isNull_1) { /* 045 */ /* 046 */ int dataLength_0 = value_1.numElements(); /* 047 */ /* 048 */ scala.collection.mutable.Builder collectionBuilder_0 = scala.collection.Seq$.MODULE$.newBuilder(); /* 049 */ collectionBuilder_0.sizeHint(dataLength_0); /* 050 */ /* 051 */ /* 052 */ int loopIndex_0 = 0; /* 053 */ /* 054 */ while (loopIndex_0 < dataLength_0) { /* 055 */ value_MapObject_lambda_variable_1 = (UTF8String) (value_1.getUTF8String(loopIndex_0)); /* 056 */ isNull_MapObject_lambda_variable_1 = value_1.isNullAt(loopIndex_0); /* 057 */ /* 058 */ boolean isNull_2 = true; /* 059 */ java.lang.String value_2 = null; /* 060 */ if (!isNull_MapObject_lambda_variable_1) { /* 061 */ /* 062 */ isNull_2 = false; /* 063 */ if (!isNull_2) { /* 064 */ /* 065 */ Object funcResult_0 = null; /* 066 */ funcResult_0 = value_MapObject_lambda_variable_1.toString(); /* 067 */ value_2 = (java.lang.String) funcResult_0; /* 068 */ /* 069 */ } /* 070 */ } /* 071 */ if (isNull_2) { /* 072 */ collectionBuilder_0.$plus$eq(null); /* 073 */ } else { /* 074 */ collectionBuilder_0.$plus$eq(value_2); /* 075 */ } /* 076 */ /* 077 */ loopIndex_0 += 1; /* 078 */ } /* 079 */ /* 080 */ value_0 = (scala.collection.Seq) collectionBuilder_0.result(); /* 081 */ } /* 082 */ globalIsNull_0 = isNull_1; /* 083 */ return value_0; /* 084 */ } /* 085 */ /* 086 */ } 20/08/08 15:33:51.879 Thread-4 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$computeClusterStats$6 20/08/08 15:33:51.880 Thread-4 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$computeClusterStats$6) is now cleaned +++ 20/08/08 15:33:51.885 Thread-4 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$aggregateByKey$3 20/08/08 15:33:51.887 Executor task launch worker for task 0 DEBUG ExecutorMetricsPoller: stageTCMP: (0, 0) -> 3 20/08/08 15:33:51.888 Thread-4 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$aggregateByKey$3) is now cleaned +++ 20/08/08 15:33:51.873 Executor task launch worker for task 3 DEBUG GenerateSafeProjection: code for mapobjects(lambdavariable(MapObject, StringType, true, -1), lambdavariable(MapObject, StringType, true, -1).toString, input[0, array, true], Some(interface scala.collection.Seq)): /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificSafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificSafeProjection extends org.apache.spark.sql.catalyst.expressions.codegen.BaseProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private InternalRow mutableRow; /* 009 */ private UTF8String value_MapObject_lambda_variable_1; /* 010 */ private boolean isNull_MapObject_lambda_variable_1; /* 011 */ private boolean globalIsNull_0; /* 012 */ /* 013 */ public SpecificSafeProjection(Object[] references) { /* 014 */ this.references = references; /* 015 */ mutableRow = (InternalRow) references[references.length - 1]; /* 016 */ /* 017 */ /* 018 */ } /* 019 */ /* 020 */ public void initialize(int partitionIndex) { /* 021 */ /* 022 */ } /* 023 */ /* 024 */ public java.lang.Object apply(java.lang.Object _i) { /* 025 */ InternalRow i = (InternalRow) _i; /* 026 */ scala.collection.Seq value_3 = MapObjects_0(i); /* 027 */ if (globalIsNull_0) { /* 028 */ mutableRow.setNullAt(0); /* 029 */ } else { /* 030 */ /* 031 */ mutableRow.update(0, value_3); /* 032 */ } /* 033 */ /* 034 */ return mutableRow; /* 035 */ } /* 036 */ /* 037 */ /* 038 */ private scala.collection.Seq MapObjects_0(InternalRow i) { /* 039 */ boolean isNull_1 = i.isNullAt(0); /* 040 */ ArrayData value_1 = isNull_1 ? /* 041 */ null : (i.getArray(0)); /* 042 */ scala.collection.Seq value_0 = null; /* 043 */ /* 044 */ if (!isNull_1) { /* 045 */ /* 046 */ int dataLength_0 = value_1.numElements(); /* 047 */ /* 048 */ scala.collection.mutable.Builder collectionBuilder_0 = scala.collection.Seq$.MODULE$.newBuilder(); /* 049 */ collectionBuilder_0.sizeHint(dataLength_0); /* 050 */ /* 051 */ /* 052 */ int loopIndex_0 = 0; /* 053 */ /* 054 */ while (loopIndex_0 < dataLength_0) { /* 055 */ value_MapObject_lambda_variable_1 = (UTF8String) (value_1.getUTF8String(loopIndex_0)); /* 056 */ isNull_MapObject_lambda_variable_1 = value_1.isNullAt(loopIndex_0); /* 057 */ /* 058 */ boolean isNull_2 = true; /* 059 */ java.lang.String value_2 = null; /* 060 */ if (!isNull_MapObject_lambda_variable_1) { /* 061 */ /* 062 */ isNull_2 = false; /* 063 */ if (!isNull_2) { /* 064 */ /* 065 */ Object funcResult_0 = null; /* 066 */ funcResult_0 = value_MapObject_lambda_variable_1.toString(); /* 067 */ value_2 = (java.lang.String) funcResult_0; /* 068 */ /* 069 */ } /* 070 */ } /* 071 */ if (isNull_2) { /* 072 */ collectionBuilder_0.$plus$eq(null); /* 073 */ } else { /* 074 */ collectionBuilder_0.$plus$eq(value_2); /* 075 */ } /* 076 */ /* 077 */ loopIndex_0 += 1; /* 078 */ } /* 079 */ /* 080 */ value_0 = (scala.collection.Seq) collectionBuilder_0.result(); /* 081 */ } /* 082 */ globalIsNull_0 = isNull_1; /* 083 */ return value_0; /* 084 */ } /* 085 */ /* 086 */ } 20/08/08 15:33:51.888 dispatcher-event-loop-0 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_0.0, runningTasks: 3 20/08/08 15:33:51.896 Thread-4 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$computeClusterStats$6 20/08/08 15:33:51.897 Thread-4 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$computeClusterStats$6) is now cleaned +++ 20/08/08 15:33:51.897 Thread-4 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$computeClusterStats$7 20/08/08 15:33:51.898 Thread-4 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$computeClusterStats$7) is now cleaned +++ 20/08/08 15:33:51.873 Executor task launch worker for task 1 DEBUG GenerateSafeProjection: code for mapobjects(lambdavariable(MapObject, StringType, true, -1), lambdavariable(MapObject, StringType, true, -1).toString, input[0, array, true], Some(interface scala.collection.Seq)): /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificSafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificSafeProjection extends org.apache.spark.sql.catalyst.expressions.codegen.BaseProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private InternalRow mutableRow; /* 009 */ private UTF8String value_MapObject_lambda_variable_1; /* 010 */ private boolean isNull_MapObject_lambda_variable_1; /* 011 */ private boolean globalIsNull_0; /* 012 */ /* 013 */ public SpecificSafeProjection(Object[] references) { /* 014 */ this.references = references; /* 015 */ mutableRow = (InternalRow) references[references.length - 1]; /* 016 */ /* 017 */ /* 018 */ } /* 019 */ /* 020 */ public void initialize(int partitionIndex) { /* 021 */ /* 022 */ } /* 023 */ /* 024 */ public java.lang.Object apply(java.lang.Object _i) { /* 025 */ InternalRow i = (InternalRow) _i; /* 026 */ scala.collection.Seq value_3 = MapObjects_0(i); /* 027 */ if (globalIsNull_0) { /* 028 */ mutableRow.setNullAt(0); /* 029 */ } else { /* 030 */ /* 031 */ mutableRow.update(0, value_3); /* 032 */ } /* 033 */ /* 034 */ return mutableRow; /* 035 */ } /* 036 */ /* 037 */ /* 038 */ private scala.collection.Seq MapObjects_0(InternalRow i) { /* 039 */ boolean isNull_1 = i.isNullAt(0); /* 040 */ ArrayData value_1 = isNull_1 ? /* 041 */ null : (i.getArray(0)); /* 042 */ scala.collection.Seq value_0 = null; /* 043 */ /* 044 */ if (!isNull_1) { /* 045 */ /* 046 */ int dataLength_0 = value_1.numElements(); /* 047 */ /* 048 */ scala.collection.mutable.Builder collectionBuilder_0 = scala.collection.Seq$.MODULE$.newBuilder(); /* 049 */ collectionBuilder_0.sizeHint(dataLength_0); /* 050 */ /* 051 */ /* 052 */ int loopIndex_0 = 0; /* 053 */ /* 054 */ while (loopIndex_0 < dataLength_0) { /* 055 */ value_MapObject_lambda_variable_1 = (UTF8String) (value_1.getUTF8String(loopIndex_0)); /* 056 */ isNull_MapObject_lambda_variable_1 = value_1.isNullAt(loopIndex_0); /* 057 */ /* 058 */ boolean isNull_2 = true; /* 059 */ java.lang.String value_2 = null; /* 060 */ if (!isNull_MapObject_lambda_variable_1) { /* 061 */ /* 062 */ isNull_2 = false; /* 063 */ if (!isNull_2) { /* 064 */ /* 065 */ Object funcResult_0 = null; /* 066 */ funcResult_0 = value_MapObject_lambda_variable_1.toString(); /* 067 */ value_2 = (java.lang.String) funcResult_0; /* 068 */ /* 069 */ } /* 070 */ } /* 071 */ if (isNull_2) { /* 072 */ collectionBuilder_0.$plus$eq(null); /* 073 */ } else { /* 074 */ collectionBuilder_0.$plus$eq(value_2); /* 075 */ } /* 076 */ /* 077 */ loopIndex_0 += 1; /* 078 */ } /* 079 */ /* 080 */ value_0 = (scala.collection.Seq) collectionBuilder_0.result(); /* 081 */ } /* 082 */ globalIsNull_0 = isNull_1; /* 083 */ return value_0; /* 084 */ } /* 085 */ /* 086 */ } 20/08/08 15:33:51.898 dispatcher-event-loop-0 DEBUG TaskSetManager: No tasks for locality level NO_PREF, so moving to locality level ANY 20/08/08 15:33:51.902 Thread-4 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$collect$2 20/08/08 15:33:51.905 Thread-4 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$collect$2) is now cleaned +++ 20/08/08 15:33:51.914 Thread-4 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$runJob$5 20/08/08 15:33:51.917 Thread-4 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$runJob$5) is now cleaned +++ 20/08/08 15:33:51.917 Thread-4 INFO SparkContext: Starting job: collectAsMap at ClusteringMetrics.scala:515 20/08/08 15:33:51.910 Executor task launch worker for task 2 DEBUG CodeGenerator: /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificSafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificSafeProjection extends org.apache.spark.sql.catalyst.expressions.codegen.BaseProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private InternalRow mutableRow; /* 009 */ private UTF8String value_MapObject_lambda_variable_1; /* 010 */ private boolean isNull_MapObject_lambda_variable_1; /* 011 */ private boolean globalIsNull_0; /* 012 */ /* 013 */ public SpecificSafeProjection(Object[] references) { /* 014 */ this.references = references; /* 015 */ mutableRow = (InternalRow) references[references.length - 1]; /* 016 */ /* 017 */ /* 018 */ } /* 019 */ /* 020 */ public void initialize(int partitionIndex) { /* 021 */ /* 022 */ } /* 023 */ /* 024 */ public java.lang.Object apply(java.lang.Object _i) { /* 025 */ InternalRow i = (InternalRow) _i; /* 026 */ scala.collection.Seq value_3 = MapObjects_0(i); /* 027 */ if (globalIsNull_0) { /* 028 */ mutableRow.setNullAt(0); /* 029 */ } else { /* 030 */ /* 031 */ mutableRow.update(0, value_3); /* 032 */ } /* 033 */ /* 034 */ return mutableRow; /* 035 */ } /* 036 */ /* 037 */ /* 038 */ private scala.collection.Seq MapObjects_0(InternalRow i) { /* 039 */ boolean isNull_1 = i.isNullAt(0); /* 040 */ ArrayData value_1 = isNull_1 ? /* 041 */ null : (i.getArray(0)); /* 042 */ scala.collection.Seq value_0 = null; /* 043 */ /* 044 */ if (!isNull_1) { /* 045 */ /* 046 */ int dataLength_0 = value_1.numElements(); /* 047 */ /* 048 */ scala.collection.mutable.Builder collectionBuilder_0 = scala.collection.Seq$.MODULE$.newBuilder(); /* 049 */ collectionBuilder_0.sizeHint(dataLength_0); /* 050 */ /* 051 */ /* 052 */ int loopIndex_0 = 0; /* 053 */ /* 054 */ while (loopIndex_0 < dataLength_0) { /* 055 */ value_MapObject_lambda_variable_1 = (UTF8String) (value_1.getUTF8String(loopIndex_0)); /* 056 */ isNull_MapObject_lambda_variable_1 = value_1.isNullAt(loopIndex_0); /* 057 */ /* 058 */ boolean isNull_2 = true; /* 059 */ java.lang.String value_2 = null; /* 060 */ if (!isNull_MapObject_lambda_variable_1) { /* 061 */ /* 062 */ isNull_2 = false; /* 063 */ if (!isNull_2) { /* 064 */ /* 065 */ Object funcResult_0 = null; /* 066 */ funcResult_0 = value_MapObject_lambda_variable_1.toString(); /* 067 */ value_2 = (java.lang.String) funcResult_0; /* 068 */ /* 069 */ } /* 070 */ } /* 071 */ if (isNull_2) { /* 072 */ collectionBuilder_0.$plus$eq(null); /* 073 */ } else { /* 074 */ collectionBuilder_0.$plus$eq(value_2); /* 075 */ } /* 076 */ /* 077 */ loopIndex_0 += 1; /* 078 */ } /* 079 */ /* 080 */ value_0 = (scala.collection.Seq) collectionBuilder_0.result(); /* 081 */ } /* 082 */ globalIsNull_0 = isNull_1; /* 083 */ return value_0; /* 084 */ } /* 085 */ /* 086 */ } 20/08/08 15:33:51.922 dag-scheduler-event-loop DEBUG SortShuffleManager: Can't use serialized shuffle for shuffle 0 because the serializer, org.apache.spark.serializer.JavaSerializer, does not support object relocation 20/08/08 15:33:51.924 task-result-getter-0 INFO TaskSetManager: Finished task 0.0 in stage 0.0 (TID 0) in 1685 ms on 192.168.1.225 (executor driver) (1/4) 20/08/08 15:33:51.925 dag-scheduler-event-loop DEBUG DAGScheduler: Merging stage rdd profiles: Set() 20/08/08 15:33:51.927 dag-scheduler-event-loop DEBUG DAGScheduler: Merging stage rdd profiles: Set() 20/08/08 15:33:51.929 dag-scheduler-event-loop INFO DAGScheduler: Registering RDD 11 (map at ClusteringMetrics.scala:500) as input to shuffle 0 20/08/08 15:33:51.931 dag-scheduler-event-loop INFO PythonAccumulatorV2: Connected to AccumulatorServer at host: 127.0.0.1 port: 41095 20/08/08 15:33:51.932 dag-scheduler-event-loop INFO DAGScheduler: Got job 1 (collectAsMap at ClusteringMetrics.scala:515) with 4 output partitions 20/08/08 15:33:51.932 dag-scheduler-event-loop INFO DAGScheduler: Final stage: ResultStage 2 (collectAsMap at ClusteringMetrics.scala:515) 20/08/08 15:33:51.932 dag-scheduler-event-loop INFO DAGScheduler: Parents of final stage: List(ShuffleMapStage 1) 20/08/08 15:33:51.934 dag-scheduler-event-loop INFO DAGScheduler: Missing parents: List(ShuffleMapStage 1) 20/08/08 15:33:51.943 dag-scheduler-event-loop DEBUG DAGScheduler: submitStage(ResultStage 2 (name=collectAsMap at ClusteringMetrics.scala:515;jobs=1)) 20/08/08 15:33:51.943 dag-scheduler-event-loop DEBUG DAGScheduler: missing: List(ShuffleMapStage 1) 20/08/08 15:33:51.947 Executor task launch worker for task 2 INFO CodeGenerator: Code generated in 77.851314 ms 20/08/08 15:33:51.950 dag-scheduler-event-loop DEBUG DAGScheduler: submitStage(ShuffleMapStage 1 (name=map at ClusteringMetrics.scala:500;jobs=1)) 20/08/08 15:33:51.951 dag-scheduler-event-loop DEBUG DAGScheduler: missing: List() 20/08/08 15:33:51.951 dag-scheduler-event-loop INFO DAGScheduler: Submitting ShuffleMapStage 1 (MapPartitionsRDD[11] at map at ClusteringMetrics.scala:500), which has no missing parents 20/08/08 15:33:51.951 dag-scheduler-event-loop DEBUG DAGScheduler: submitMissingTasks(ShuffleMapStage 1) 20/08/08 15:33:51.958 Executor task launch worker for task 3 DEBUG BlockManager: Getting local block broadcast_0 20/08/08 15:33:51.958 Executor task launch worker for task 3 DEBUG BlockManager: Level for block broadcast_0 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:51.984 Executor task launch worker for task 1 DEBUG GenerateUnsafeProjection: code for newInstance(class org.apache.spark.ml.linalg.VectorUDT).serialize: /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificUnsafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificUnsafeProjection extends org.apache.spark.sql.catalyst.expressions.UnsafeProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private boolean resultIsNull_0; /* 009 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[] mutableStateArray_1 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[2]; /* 010 */ private org.apache.spark.ml.linalg.Vector[] mutableStateArray_0 = new org.apache.spark.ml.linalg.Vector[1]; /* 011 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[] mutableStateArray_2 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[2]; /* 012 */ /* 013 */ public SpecificUnsafeProjection(Object[] references) { /* 014 */ this.references = references; /* 015 */ /* 016 */ mutableStateArray_1[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(1, 32); /* 017 */ mutableStateArray_1[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(mutableStateArray_1[0], 4); /* 018 */ mutableStateArray_2[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(mutableStateArray_1[1], 4); /* 019 */ mutableStateArray_2[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(mutableStateArray_1[1], 8); /* 020 */ /* 021 */ } /* 022 */ /* 023 */ public void initialize(int partitionIndex) { /* 024 */ /* 025 */ } /* 026 */ /* 027 */ // Scala.Function1 need this /* 028 */ public java.lang.Object apply(java.lang.Object row) { /* 029 */ return apply((InternalRow) row); /* 030 */ } /* 031 */ /* 032 */ public UnsafeRow apply(InternalRow i) { /* 033 */ mutableStateArray_1[0].reset(); /* 034 */ /* 035 */ /* 036 */ mutableStateArray_1[0].zeroOutNullBytes(); /* 037 */ /* 038 */ final org.apache.spark.ml.linalg.VectorUDT value_1 = false ? /* 039 */ null : new org.apache.spark.ml.linalg.VectorUDT(); /* 040 */ boolean isNull_0 = true; /* 041 */ InternalRow value_0 = null; /* 042 */ if (!false) { /* 043 */ resultIsNull_0 = false; /* 044 */ if (!resultIsNull_0) { /* 045 */ boolean isNull_2 = i.isNullAt(0); /* 046 */ org.apache.spark.ml.linalg.Vector value_2 = isNull_2 ? /* 047 */ null : ((org.apache.spark.ml.linalg.Vector)i.get(0, null)); /* 048 */ resultIsNull_0 = isNull_2; /* 049 */ mutableStateArray_0[0] = value_2; /* 050 */ } /* 051 */ /* 052 */ isNull_0 = resultIsNull_0; /* 053 */ if (!isNull_0) { /* 054 */ /* 055 */ Object funcResult_0 = null; /* 056 */ funcResult_0 = value_1.serialize(mutableStateArray_0[0]); /* 057 */ /* 058 */ if (funcResult_0 != null) { /* 059 */ value_0 = (InternalRow) funcResult_0; /* 060 */ } else { /* 061 */ isNull_0 = true; /* 062 */ } /* 063 */ /* 064 */ /* 065 */ } /* 066 */ } /* 067 */ if (isNull_0) { /* 068 */ mutableStateArray_1[0].setNullAt(0); /* 069 */ } else { /* 070 */ final InternalRow tmpInput_0 = value_0; /* 071 */ if (tmpInput_0 instanceof UnsafeRow) { /* 072 */ mutableStateArray_1[0].write(0, (UnsafeRow) tmpInput_0); /* 073 */ } else { /* 074 */ // Remember the current cursor so that we can calculate how many bytes are /* 075 */ // written later. /* 076 */ final int previousCursor_0 = mutableStateArray_1[0].cursor(); /* 077 */ /* 078 */ mutableStateArray_1[1].resetRowWriter(); /* 079 */ /* 080 */ /* 081 */ mutableStateArray_1[1].write(0, (tmpInput_0.getByte(0))); /* 082 */ /* 083 */ /* 084 */ if ((tmpInput_0.isNullAt(1))) { /* 085 */ mutableStateArray_1[1].setNullAt(1); /* 086 */ } else { /* 087 */ mutableStateArray_1[1].write(1, (tmpInput_0.getInt(1))); /* 088 */ } /* 089 */ /* 090 */ /* 091 */ if ((tmpInput_0.isNullAt(2))) { /* 092 */ mutableStateArray_1[1].setNullAt(2); /* 093 */ } else { /* 094 */ // Remember the current cursor so that we can calculate how many bytes are /* 095 */ // written later. /* 096 */ final int previousCursor_1 = mutableStateArray_1[1].cursor(); /* 097 */ /* 098 */ final ArrayData tmpInput_1 = (tmpInput_0.getArray(2)); /* 099 */ if (tmpInput_1 instanceof UnsafeArrayData) { /* 100 */ mutableStateArray_1[1].write((UnsafeArrayData) tmpInput_1); /* 101 */ } else { /* 102 */ final int numElements_0 = tmpInput_1.numElements(); /* 103 */ mutableStateArray_2[0].initialize(numElements_0); /* 104 */ /* 105 */ for (int index_0 = 0; index_0 < numElements_0; index_0++) { /* 106 */ mutableStateArray_2[0].write(index_0, tmpInput_1.getInt(index_0)); /* 107 */ } /* 108 */ } /* 109 */ /* 110 */ mutableStateArray_1[1].setOffsetAndSizeFromPreviousCursor(2, previousCursor_1); /* 111 */ } /* 112 */ /* 113 */ /* 114 */ if ((tmpInput_0.isNullAt(3))) { /* 115 */ mutableStateArray_1[1].setNullAt(3); /* 116 */ } else { /* 117 */ // Remember the current cursor so that we can calculate how many bytes are /* 118 */ // written later. /* 119 */ final int previousCursor_2 = mutableStateArray_1[1].cursor(); /* 120 */ /* 121 */ final ArrayData tmpInput_2 = (tmpInput_0.getArray(3)); /* 122 */ if (tmpInput_2 instanceof UnsafeArrayData) { /* 123 */ mutableStateArray_1[1].write((UnsafeArrayData) tmpInput_2); /* 124 */ } else { /* 125 */ final int numElements_1 = tmpInput_2.numElements(); /* 126 */ mutableStateArray_2[1].initialize(numElements_1); /* 127 */ /* 128 */ for (int index_1 = 0; index_1 < numElements_1; index_1++) { /* 129 */ mutableStateArray_2[1].write(index_1, tmpInput_2.getDouble(index_1)); /* 130 */ } /* 131 */ } /* 132 */ /* 133 */ mutableStateArray_1[1].setOffsetAndSizeFromPreviousCursor(3, previousCursor_2); /* 134 */ } /* 135 */ /* 136 */ /* 137 */ mutableStateArray_1[0].setOffsetAndSizeFromPreviousCursor(0, previousCursor_0); /* 138 */ } /* 139 */ } /* 140 */ return (mutableStateArray_1[0].getRow()); /* 141 */ } /* 142 */ /* 143 */ /* 144 */ } 20/08/08 15:33:51.988 Executor task launch worker for task 2 DEBUG GenerateUnsafeProjection: code for newInstance(class org.apache.spark.ml.linalg.VectorUDT).serialize: /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificUnsafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificUnsafeProjection extends org.apache.spark.sql.catalyst.expressions.UnsafeProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private boolean resultIsNull_0; /* 009 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[] mutableStateArray_1 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[2]; /* 010 */ private org.apache.spark.ml.linalg.Vector[] mutableStateArray_0 = new org.apache.spark.ml.linalg.Vector[1]; /* 011 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[] mutableStateArray_2 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[2]; /* 012 */ /* 013 */ public SpecificUnsafeProjection(Object[] references) { /* 014 */ this.references = references; /* 015 */ /* 016 */ mutableStateArray_1[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(1, 32); /* 017 */ mutableStateArray_1[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(mutableStateArray_1[0], 4); /* 018 */ mutableStateArray_2[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(mutableStateArray_1[1], 4); /* 019 */ mutableStateArray_2[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(mutableStateArray_1[1], 8); /* 020 */ /* 021 */ } /* 022 */ /* 023 */ public void initialize(int partitionIndex) { /* 024 */ /* 025 */ } /* 026 */ /* 027 */ // Scala.Function1 need this /* 028 */ public java.lang.Object apply(java.lang.Object row) { /* 029 */ return apply((InternalRow) row); /* 030 */ } /* 031 */ /* 032 */ public UnsafeRow apply(InternalRow i) { /* 033 */ mutableStateArray_1[0].reset(); /* 034 */ /* 035 */ /* 036 */ mutableStateArray_1[0].zeroOutNullBytes(); /* 037 */ /* 038 */ final org.apache.spark.ml.linalg.VectorUDT value_1 = false ? /* 039 */ null : new org.apache.spark.ml.linalg.VectorUDT(); /* 040 */ boolean isNull_0 = true; /* 041 */ InternalRow value_0 = null; /* 042 */ if (!false) { /* 043 */ resultIsNull_0 = false; /* 044 */ if (!resultIsNull_0) { /* 045 */ boolean isNull_2 = i.isNullAt(0); /* 046 */ org.apache.spark.ml.linalg.Vector value_2 = isNull_2 ? /* 047 */ null : ((org.apache.spark.ml.linalg.Vector)i.get(0, null)); /* 048 */ resultIsNull_0 = isNull_2; /* 049 */ mutableStateArray_0[0] = value_2; /* 050 */ } /* 051 */ /* 052 */ isNull_0 = resultIsNull_0; /* 053 */ if (!isNull_0) { /* 054 */ /* 055 */ Object funcResult_0 = null; /* 056 */ funcResult_0 = value_1.serialize(mutableStateArray_0[0]); /* 057 */ /* 058 */ if (funcResult_0 != null) { /* 059 */ value_0 = (InternalRow) funcResult_0; /* 060 */ } else { /* 061 */ isNull_0 = true; /* 062 */ } /* 063 */ /* 064 */ /* 065 */ } /* 066 */ } /* 067 */ if (isNull_0) { /* 068 */ mutableStateArray_1[0].setNullAt(0); /* 069 */ } else { /* 070 */ final InternalRow tmpInput_0 = value_0; /* 071 */ if (tmpInput_0 instanceof UnsafeRow) { /* 072 */ mutableStateArray_1[0].write(0, (UnsafeRow) tmpInput_0); /* 073 */ } else { /* 074 */ // Remember the current cursor so that we can calculate how many bytes are /* 075 */ // written later. /* 076 */ final int previousCursor_0 = mutableStateArray_1[0].cursor(); /* 077 */ /* 078 */ mutableStateArray_1[1].resetRowWriter(); /* 079 */ /* 080 */ /* 081 */ mutableStateArray_1[1].write(0, (tmpInput_0.getByte(0))); /* 082 */ /* 083 */ /* 084 */ if ((tmpInput_0.isNullAt(1))) { /* 085 */ mutableStateArray_1[1].setNullAt(1); /* 086 */ } else { /* 087 */ mutableStateArray_1[1].write(1, (tmpInput_0.getInt(1))); /* 088 */ } /* 089 */ /* 090 */ /* 091 */ if ((tmpInput_0.isNullAt(2))) { /* 092 */ mutableStateArray_1[1].setNullAt(2); /* 093 */ } else { /* 094 */ // Remember the current cursor so that we can calculate how many bytes are /* 095 */ // written later. /* 096 */ final int previousCursor_1 = mutableStateArray_1[1].cursor(); /* 097 */ /* 098 */ final ArrayData tmpInput_1 = (tmpInput_0.getArray(2)); /* 099 */ if (tmpInput_1 instanceof UnsafeArrayData) { /* 100 */ mutableStateArray_1[1].write((UnsafeArrayData) tmpInput_1); /* 101 */ } else { /* 102 */ final int numElements_0 = tmpInput_1.numElements(); /* 103 */ mutableStateArray_2[0].initialize(numElements_0); /* 104 */ /* 105 */ for (int index_0 = 0; index_0 < numElements_0; index_0++) { /* 106 */ mutableStateArray_2[0].write(index_0, tmpInput_1.getInt(index_0)); /* 107 */ } /* 108 */ } /* 109 */ /* 110 */ mutableStateArray_1[1].setOffsetAndSizeFromPreviousCursor(2, previousCursor_1); /* 111 */ } /* 112 */ /* 113 */ /* 114 */ if ((tmpInput_0.isNullAt(3))) { /* 115 */ mutableStateArray_1[1].setNullAt(3); /* 116 */ } else { /* 117 */ // Remember the current cursor so that we can calculate how many bytes are /* 118 */ // written later. /* 119 */ final int previousCursor_2 = mutableStateArray_1[1].cursor(); /* 120 */ /* 121 */ final ArrayData tmpInput_2 = (tmpInput_0.getArray(3)); /* 122 */ if (tmpInput_2 instanceof UnsafeArrayData) { /* 123 */ mutableStateArray_1[1].write((UnsafeArrayData) tmpInput_2); /* 124 */ } else { /* 125 */ final int numElements_1 = tmpInput_2.numElements(); /* 126 */ mutableStateArray_2[1].initialize(numElements_1); /* 127 */ /* 128 */ for (int index_1 = 0; index_1 < numElements_1; index_1++) { /* 129 */ mutableStateArray_2[1].write(index_1, tmpInput_2.getDouble(index_1)); /* 130 */ } /* 131 */ } /* 132 */ /* 133 */ mutableStateArray_1[1].setOffsetAndSizeFromPreviousCursor(3, previousCursor_2); /* 134 */ } /* 135 */ /* 136 */ /* 137 */ mutableStateArray_1[0].setOffsetAndSizeFromPreviousCursor(0, previousCursor_0); /* 138 */ } /* 139 */ } /* 140 */ return (mutableStateArray_1[0].getRow()); /* 141 */ } /* 142 */ /* 143 */ /* 144 */ } 20/08/08 15:33:51.996 Executor task launch worker for task 1 DEBUG CodeGenerator: /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificUnsafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificUnsafeProjection extends org.apache.spark.sql.catalyst.expressions.UnsafeProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private boolean resultIsNull_0; /* 009 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[] mutableStateArray_1 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[2]; /* 010 */ private org.apache.spark.ml.linalg.Vector[] mutableStateArray_0 = new org.apache.spark.ml.linalg.Vector[1]; /* 011 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[] mutableStateArray_2 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[2]; /* 012 */ /* 013 */ public SpecificUnsafeProjection(Object[] references) { /* 014 */ this.references = references; /* 015 */ /* 016 */ mutableStateArray_1[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(1, 32); /* 017 */ mutableStateArray_1[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(mutableStateArray_1[0], 4); /* 018 */ mutableStateArray_2[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(mutableStateArray_1[1], 4); /* 019 */ mutableStateArray_2[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(mutableStateArray_1[1], 8); /* 020 */ /* 021 */ } /* 022 */ /* 023 */ public void initialize(int partitionIndex) { /* 024 */ /* 025 */ } /* 026 */ /* 027 */ // Scala.Function1 need this /* 028 */ public java.lang.Object apply(java.lang.Object row) { /* 029 */ return apply((InternalRow) row); /* 030 */ } /* 031 */ /* 032 */ public UnsafeRow apply(InternalRow i) { /* 033 */ mutableStateArray_1[0].reset(); /* 034 */ /* 035 */ /* 036 */ mutableStateArray_1[0].zeroOutNullBytes(); /* 037 */ /* 038 */ final org.apache.spark.ml.linalg.VectorUDT value_1 = false ? /* 039 */ null : new org.apache.spark.ml.linalg.VectorUDT(); /* 040 */ boolean isNull_0 = true; /* 041 */ InternalRow value_0 = null; /* 042 */ if (!false) { /* 043 */ resultIsNull_0 = false; /* 044 */ if (!resultIsNull_0) { /* 045 */ boolean isNull_2 = i.isNullAt(0); /* 046 */ org.apache.spark.ml.linalg.Vector value_2 = isNull_2 ? /* 047 */ null : ((org.apache.spark.ml.linalg.Vector)i.get(0, null)); /* 048 */ resultIsNull_0 = isNull_2; /* 049 */ mutableStateArray_0[0] = value_2; /* 050 */ } /* 051 */ /* 052 */ isNull_0 = resultIsNull_0; /* 053 */ if (!isNull_0) { /* 054 */ /* 055 */ Object funcResult_0 = null; /* 056 */ funcResult_0 = value_1.serialize(mutableStateArray_0[0]); /* 057 */ /* 058 */ if (funcResult_0 != null) { /* 059 */ value_0 = (InternalRow) funcResult_0; /* 060 */ } else { /* 061 */ isNull_0 = true; /* 062 */ } /* 063 */ /* 064 */ /* 065 */ } /* 066 */ } /* 067 */ if (isNull_0) { /* 068 */ mutableStateArray_1[0].setNullAt(0); /* 069 */ } else { /* 070 */ final InternalRow tmpInput_0 = value_0; /* 071 */ if (tmpInput_0 instanceof UnsafeRow) { /* 072 */ mutableStateArray_1[0].write(0, (UnsafeRow) tmpInput_0); /* 073 */ } else { /* 074 */ // Remember the current cursor so that we can calculate how many bytes are /* 075 */ // written later. /* 076 */ final int previousCursor_0 = mutableStateArray_1[0].cursor(); /* 077 */ /* 078 */ mutableStateArray_1[1].resetRowWriter(); /* 079 */ /* 080 */ /* 081 */ mutableStateArray_1[1].write(0, (tmpInput_0.getByte(0))); /* 082 */ /* 083 */ /* 084 */ if ((tmpInput_0.isNullAt(1))) { /* 085 */ mutableStateArray_1[1].setNullAt(1); /* 086 */ } else { /* 087 */ mutableStateArray_1[1].write(1, (tmpInput_0.getInt(1))); /* 088 */ } /* 089 */ /* 090 */ /* 091 */ if ((tmpInput_0.isNullAt(2))) { /* 092 */ mutableStateArray_1[1].setNullAt(2); /* 093 */ } else { /* 094 */ // Remember the current cursor so that we can calculate how many bytes are /* 095 */ // written later. /* 096 */ final int previousCursor_1 = mutableStateArray_1[1].cursor(); /* 097 */ /* 098 */ final ArrayData tmpInput_1 = (tmpInput_0.getArray(2)); /* 099 */ if (tmpInput_1 instanceof UnsafeArrayData) { /* 100 */ mutableStateArray_1[1].write((UnsafeArrayData) tmpInput_1); /* 101 */ } else { /* 102 */ final int numElements_0 = tmpInput_1.numElements(); /* 103 */ mutableStateArray_2[0].initialize(numElements_0); /* 104 */ /* 105 */ for (int index_0 = 0; index_0 < numElements_0; index_0++) { /* 106 */ mutableStateArray_2[0].write(index_0, tmpInput_1.getInt(index_0)); /* 107 */ } /* 108 */ } /* 109 */ /* 110 */ mutableStateArray_1[1].setOffsetAndSizeFromPreviousCursor(2, previousCursor_1); /* 111 */ } /* 112 */ /* 113 */ /* 114 */ if ((tmpInput_0.isNullAt(3))) { /* 115 */ mutableStateArray_1[1].setNullAt(3); /* 116 */ } else { /* 117 */ // Remember the current cursor so that we can calculate how many bytes are /* 118 */ // written later. /* 119 */ final int previousCursor_2 = mutableStateArray_1[1].cursor(); /* 120 */ /* 121 */ final ArrayData tmpInput_2 = (tmpInput_0.getArray(3)); /* 122 */ if (tmpInput_2 instanceof UnsafeArrayData) { /* 123 */ mutableStateArray_1[1].write((UnsafeArrayData) tmpInput_2); /* 124 */ } else { /* 125 */ final int numElements_1 = tmpInput_2.numElements(); /* 126 */ mutableStateArray_2[1].initialize(numElements_1); /* 127 */ /* 128 */ for (int index_1 = 0; index_1 < numElements_1; index_1++) { /* 129 */ mutableStateArray_2[1].write(index_1, tmpInput_2.getDouble(index_1)); /* 130 */ } /* 131 */ } /* 132 */ /* 133 */ mutableStateArray_1[1].setOffsetAndSizeFromPreviousCursor(3, previousCursor_2); /* 134 */ } /* 135 */ /* 136 */ /* 137 */ mutableStateArray_1[0].setOffsetAndSizeFromPreviousCursor(0, previousCursor_0); /* 138 */ } /* 139 */ } /* 140 */ return (mutableStateArray_1[0].getRow()); /* 141 */ } /* 142 */ /* 143 */ /* 144 */ } 20/08/08 15:33:51.997 Executor task launch worker for task 3 DEBUG GenerateUnsafeProjection: code for newInstance(class org.apache.spark.ml.linalg.VectorUDT).serialize: /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificUnsafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificUnsafeProjection extends org.apache.spark.sql.catalyst.expressions.UnsafeProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private boolean resultIsNull_0; /* 009 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[] mutableStateArray_1 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[2]; /* 010 */ private org.apache.spark.ml.linalg.Vector[] mutableStateArray_0 = new org.apache.spark.ml.linalg.Vector[1]; /* 011 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[] mutableStateArray_2 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[2]; /* 012 */ /* 013 */ public SpecificUnsafeProjection(Object[] references) { /* 014 */ this.references = references; /* 015 */ /* 016 */ mutableStateArray_1[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(1, 32); /* 017 */ mutableStateArray_1[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(mutableStateArray_1[0], 4); /* 018 */ mutableStateArray_2[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(mutableStateArray_1[1], 4); /* 019 */ mutableStateArray_2[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(mutableStateArray_1[1], 8); /* 020 */ /* 021 */ } /* 022 */ /* 023 */ public void initialize(int partitionIndex) { /* 024 */ /* 025 */ } /* 026 */ /* 027 */ // Scala.Function1 need this /* 028 */ public java.lang.Object apply(java.lang.Object row) { /* 029 */ return apply((InternalRow) row); /* 030 */ } /* 031 */ /* 032 */ public UnsafeRow apply(InternalRow i) { /* 033 */ mutableStateArray_1[0].reset(); /* 034 */ /* 035 */ /* 036 */ mutableStateArray_1[0].zeroOutNullBytes(); /* 037 */ /* 038 */ final org.apache.spark.ml.linalg.VectorUDT value_1 = false ? /* 039 */ null : new org.apache.spark.ml.linalg.VectorUDT(); /* 040 */ boolean isNull_0 = true; /* 041 */ InternalRow value_0 = null; /* 042 */ if (!false) { /* 043 */ resultIsNull_0 = false; /* 044 */ if (!resultIsNull_0) { /* 045 */ boolean isNull_2 = i.isNullAt(0); /* 046 */ org.apache.spark.ml.linalg.Vector value_2 = isNull_2 ? /* 047 */ null : ((org.apache.spark.ml.linalg.Vector)i.get(0, null)); /* 048 */ resultIsNull_0 = isNull_2; /* 049 */ mutableStateArray_0[0] = value_2; /* 050 */ } /* 051 */ /* 052 */ isNull_0 = resultIsNull_0; /* 053 */ if (!isNull_0) { /* 054 */ /* 055 */ Object funcResult_0 = null; /* 056 */ funcResult_0 = value_1.serialize(mutableStateArray_0[0]); /* 057 */ /* 058 */ if (funcResult_0 != null) { /* 059 */ value_0 = (InternalRow) funcResult_0; /* 060 */ } else { /* 061 */ isNull_0 = true; /* 062 */ } /* 063 */ /* 064 */ /* 065 */ } /* 066 */ } /* 067 */ if (isNull_0) { /* 068 */ mutableStateArray_1[0].setNullAt(0); /* 069 */ } else { /* 070 */ final InternalRow tmpInput_0 = value_0; /* 071 */ if (tmpInput_0 instanceof UnsafeRow) { /* 072 */ mutableStateArray_1[0].write(0, (UnsafeRow) tmpInput_0); /* 073 */ } else { /* 074 */ // Remember the current cursor so that we can calculate how many bytes are /* 075 */ // written later. /* 076 */ final int previousCursor_0 = mutableStateArray_1[0].cursor(); /* 077 */ /* 078 */ mutableStateArray_1[1].resetRowWriter(); /* 079 */ /* 080 */ /* 081 */ mutableStateArray_1[1].write(0, (tmpInput_0.getByte(0))); /* 082 */ /* 083 */ /* 084 */ if ((tmpInput_0.isNullAt(1))) { /* 085 */ mutableStateArray_1[1].setNullAt(1); /* 086 */ } else { /* 087 */ mutableStateArray_1[1].write(1, (tmpInput_0.getInt(1))); /* 088 */ } /* 089 */ /* 090 */ /* 091 */ if ((tmpInput_0.isNullAt(2))) { /* 092 */ mutableStateArray_1[1].setNullAt(2); /* 093 */ } else { /* 094 */ // Remember the current cursor so that we can calculate how many bytes are /* 095 */ // written later. /* 096 */ final int previousCursor_1 = mutableStateArray_1[1].cursor(); /* 097 */ /* 098 */ final ArrayData tmpInput_1 = (tmpInput_0.getArray(2)); /* 099 */ if (tmpInput_1 instanceof UnsafeArrayData) { /* 100 */ mutableStateArray_1[1].write((UnsafeArrayData) tmpInput_1); /* 101 */ } else { /* 102 */ final int numElements_0 = tmpInput_1.numElements(); /* 103 */ mutableStateArray_2[0].initialize(numElements_0); /* 104 */ /* 105 */ for (int index_0 = 0; index_0 < numElements_0; index_0++) { /* 106 */ mutableStateArray_2[0].write(index_0, tmpInput_1.getInt(index_0)); /* 107 */ } /* 108 */ } /* 109 */ /* 110 */ mutableStateArray_1[1].setOffsetAndSizeFromPreviousCursor(2, previousCursor_1); /* 111 */ } /* 112 */ /* 113 */ /* 114 */ if ((tmpInput_0.isNullAt(3))) { /* 115 */ mutableStateArray_1[1].setNullAt(3); /* 116 */ } else { /* 117 */ // Remember the current cursor so that we can calculate how many bytes are /* 118 */ // written later. /* 119 */ final int previousCursor_2 = mutableStateArray_1[1].cursor(); /* 120 */ /* 121 */ final ArrayData tmpInput_2 = (tmpInput_0.getArray(3)); /* 122 */ if (tmpInput_2 instanceof UnsafeArrayData) { /* 123 */ mutableStateArray_1[1].write((UnsafeArrayData) tmpInput_2); /* 124 */ } else { /* 125 */ final int numElements_1 = tmpInput_2.numElements(); /* 126 */ mutableStateArray_2[1].initialize(numElements_1); /* 127 */ /* 128 */ for (int index_1 = 0; index_1 < numElements_1; index_1++) { /* 129 */ mutableStateArray_2[1].write(index_1, tmpInput_2.getDouble(index_1)); /* 130 */ } /* 131 */ } /* 132 */ /* 133 */ mutableStateArray_1[1].setOffsetAndSizeFromPreviousCursor(3, previousCursor_2); /* 134 */ } /* 135 */ /* 136 */ /* 137 */ mutableStateArray_1[0].setOffsetAndSizeFromPreviousCursor(0, previousCursor_0); /* 138 */ } /* 139 */ } /* 140 */ return (mutableStateArray_1[0].getRow()); /* 141 */ } /* 142 */ /* 143 */ /* 144 */ } 20/08/08 15:33:52.050 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_1 stored as values in memory (estimated size 29.0 KiB, free 366.3 MiB) 20/08/08 15:33:52.050 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_1 locally took 1 ms 20/08/08 15:33:52.050 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_1 without replication took 1 ms 20/08/08 15:33:52.052 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_1_piece0 stored as bytes in memory (estimated size 12.9 KiB, free 366.2 MiB) 20/08/08 15:33:52.052 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master broadcast_1_piece0 for BlockManagerId(driver, 192.168.1.225, 36391, None) 20/08/08 15:33:52.053 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added broadcast_1_piece0 in memory on 192.168.1.225:36391 (size: 12.9 KiB, free: 366.3 MiB) 20/08/08 15:33:52.053 dag-scheduler-event-loop DEBUG BlockManagerMaster: Updated info of block broadcast_1_piece0 20/08/08 15:33:52.053 dag-scheduler-event-loop DEBUG BlockManager: Told master about block broadcast_1_piece0 20/08/08 15:33:52.054 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_1_piece0 locally took 2 ms 20/08/08 15:33:52.054 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_1_piece0 without replication took 2 ms 20/08/08 15:33:52.054 dag-scheduler-event-loop INFO SparkContext: Created broadcast 1 from broadcast at DAGScheduler.scala:1348 20/08/08 15:33:52.061 dag-scheduler-event-loop INFO DAGScheduler: Submitting 4 missing tasks from ShuffleMapStage 1 (MapPartitionsRDD[11] at map at ClusteringMetrics.scala:500) (first 15 tasks are for partitions Vector(0, 1, 2, 3)) 20/08/08 15:33:52.062 dag-scheduler-event-loop INFO TaskSchedulerImpl: Adding task set 1.0 with 4 tasks resource profile 0 20/08/08 15:33:52.062 dag-scheduler-event-loop DEBUG TaskSetManager: Epoch for TaskSet 1.0: 0 20/08/08 15:33:52.062 dag-scheduler-event-loop DEBUG TaskSetManager: Adding pending tasks took 0 ms 20/08/08 15:33:52.063 dag-scheduler-event-loop DEBUG TaskSetManager: Valid locality levels for TaskSet 1.0: NO_PREF, ANY 20/08/08 15:33:52.063 dispatcher-event-loop-1 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_1.0, runningTasks: 0 20/08/08 15:33:52.065 dispatcher-event-loop-1 INFO TaskSetManager: Starting task 0.0 in stage 1.0 (TID 1, 192.168.1.225, executor driver, partition 0, PROCESS_LOCAL, 7387 bytes) taskResourceAssignments Map() 20/08/08 15:33:52.066 dispatcher-event-loop-1 INFO TaskSetManager: Starting task 1.0 in stage 1.0 (TID 2, 192.168.1.225, executor driver, partition 1, PROCESS_LOCAL, 7452 bytes) taskResourceAssignments Map() 20/08/08 15:33:52.066 dispatcher-event-loop-1 INFO TaskSetManager: Starting task 2.0 in stage 1.0 (TID 3, 192.168.1.225, executor driver, partition 2, PROCESS_LOCAL, 7387 bytes) taskResourceAssignments Map() 20/08/08 15:33:52.067 dispatcher-event-loop-1 INFO TaskSetManager: Starting task 3.0 in stage 1.0 (TID 4, 192.168.1.225, executor driver, partition 3, PROCESS_LOCAL, 7452 bytes) taskResourceAssignments Map() 20/08/08 15:33:52.067 Executor task launch worker for task 1 INFO Executor: Running task 0.0 in stage 1.0 (TID 1) 20/08/08 15:33:52.068 Executor task launch worker for task 1 DEBUG ExecutorMetricsPoller: stageTCMP: (1, 0) -> 1 20/08/08 15:33:52.070 Executor task launch worker for task 1 DEBUG BlockManager: Getting local block broadcast_1 20/08/08 15:33:52.070 Executor task launch worker for task 1 DEBUG BlockManager: Level for block broadcast_1 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:52.072 Executor task launch worker for task 1 INFO CodeGenerator: Code generated in 87.556095 ms 20/08/08 15:33:52.076 Executor task launch worker for task 2 INFO Executor: Running task 1.0 in stage 1.0 (TID 2) 20/08/08 15:33:52.080 Executor task launch worker for task 3 INFO PythonRunner: Times: total = 615, boot = 569, init = 46, finish = 0 20/08/08 15:33:52.080 Executor task launch worker for task 2 INFO PythonRunner: Times: total = 613, boot = 585, init = 28, finish = 0 20/08/08 15:33:52.082 Executor task launch worker for task 1 INFO PythonRunner: Times: total = 594, boot = 560, init = 34, finish = 0 20/08/08 15:33:52.085 Executor task launch worker for task 2 INFO Executor: Finished task 2.0 in stage 0.0 (TID 2). 2103 bytes result sent to driver 20/08/08 15:33:52.086 Executor task launch worker for task 2 DEBUG ExecutorMetricsPoller: stageTCMP: (0, 0) -> 2 20/08/08 15:33:52.085 Executor task launch worker for task 1 INFO Executor: Finished task 1.0 in stage 0.0 (TID 1). 2117 bytes result sent to driver 20/08/08 15:33:52.087 Executor task launch worker for task 3 INFO Executor: Finished task 3.0 in stage 0.0 (TID 3). 2091 bytes result sent to driver 20/08/08 15:33:52.087 Executor task launch worker for task 1 DEBUG ExecutorMetricsPoller: stageTCMP: (0, 0) -> 1 20/08/08 15:33:52.088 Executor task launch worker for task 3 DEBUG ExecutorMetricsPoller: removing (0, 0) from stageTCMP 20/08/08 15:33:52.090 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(8) 20/08/08 15:33:52.090 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 8 20/08/08 15:33:52.091 Executor task launch worker for task 4 INFO Executor: Running task 3.0 in stage 1.0 (TID 4) 20/08/08 15:33:52.091 Executor task launch worker for task 3 INFO Executor: Running task 2.0 in stage 1.0 (TID 3) 20/08/08 15:33:52.093 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 8 20/08/08 15:33:52.093 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(14) 20/08/08 15:33:52.093 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 14 20/08/08 15:33:52.093 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 14 20/08/08 15:33:52.093 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(15) 20/08/08 15:33:52.093 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 15 20/08/08 15:33:52.093 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 15 20/08/08 15:33:52.093 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(25) 20/08/08 15:33:52.093 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 25 20/08/08 15:33:52.093 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 25 20/08/08 15:33:52.093 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(27) 20/08/08 15:33:52.093 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 27 20/08/08 15:33:52.093 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 27 20/08/08 15:33:52.093 dispatcher-event-loop-1 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_0.0, runningTasks: 2 20/08/08 15:33:52.094 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(9) 20/08/08 15:33:52.094 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 9 20/08/08 15:33:52.094 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 9 20/08/08 15:33:52.094 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(18) 20/08/08 15:33:52.094 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 18 20/08/08 15:33:52.094 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 18 20/08/08 15:33:52.094 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(7) 20/08/08 15:33:52.094 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 7 20/08/08 15:33:52.094 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 7 20/08/08 15:33:52.094 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(11) 20/08/08 15:33:52.094 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 11 20/08/08 15:33:52.094 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 11 20/08/08 15:33:52.094 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanBroadcast(0) 20/08/08 15:33:52.096 Executor task launch worker for task 4 DEBUG ExecutorMetricsPoller: stageTCMP: (1, 0) -> 2 20/08/08 15:33:52.097 dispatcher-event-loop-1 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_0.0, runningTasks: 1 20/08/08 15:33:52.099 Executor task launch worker for task 3 DEBUG ExecutorMetricsPoller: stageTCMP: (1, 0) -> 3 20/08/08 15:33:52.099 dispatcher-event-loop-1 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_0.0, runningTasks: 0 20/08/08 15:33:52.099 Executor task launch worker for task 2 DEBUG ExecutorMetricsPoller: stageTCMP: (1, 0) -> 4 20/08/08 15:33:52.099 task-result-getter-1 INFO TaskSetManager: Finished task 2.0 in stage 0.0 (TID 2) in 1863 ms on 192.168.1.225 (executor driver) (2/4) 20/08/08 15:33:52.103 task-result-getter-2 INFO TaskSetManager: Finished task 1.0 in stage 0.0 (TID 1) in 1867 ms on 192.168.1.225 (executor driver) (3/4) 20/08/08 15:33:52.105 Spark Context Cleaner DEBUG ContextCleaner: Cleaning broadcast 0 20/08/08 15:33:52.108 task-result-getter-3 INFO TaskSetManager: Finished task 3.0 in stage 0.0 (TID 3) in 1870 ms on 192.168.1.225 (executor driver) (4/4) 20/08/08 15:33:52.109 Spark Context Cleaner DEBUG TorrentBroadcast: Unpersisting TorrentBroadcast 0 20/08/08 15:33:52.110 task-result-getter-3 INFO TaskSchedulerImpl: Removed TaskSet 0.0, whose tasks have all completed, from pool 20/08/08 15:33:52.111 dag-scheduler-event-loop INFO DAGScheduler: ResultStage 0 ( at /usr/lib/python3.6/runpy.py:85) finished in 2.066 s 20/08/08 15:33:52.116 dag-scheduler-event-loop DEBUG DAGScheduler: After removal of stage 0, remaining stages = 0 20/08/08 15:33:52.116 dag-scheduler-event-loop INFO DAGScheduler: Job 0 is finished. Cancelling potential speculative or zombie tasks for this job 20/08/08 15:33:52.117 dag-scheduler-event-loop INFO TaskSchedulerImpl: Killing all running tasks in stage 0: Stage finished 20/08/08 15:33:52.119 Thread-4 INFO DAGScheduler: Job 0 finished: at /usr/lib/python3.6/runpy.py:85, took 2.237404 s 20/08/08 15:33:52.138 main DEBUG PythonGatewayServer: Exiting due to broken pipe from Python driver 20/08/08 15:33:52.143 Thread-1 INFO ShutdownHookManager: Shutdown hook called 20/08/08 15:33:52.144 Thread-1 INFO ShutdownHookManager: Deleting directory /home/jenkins/workspace/spark-master-test-python-arm/python/target/c1079ab9-e95b-42b9-9d66-c20bb67941db/spark-41940c30-9959-4a27-9410-0477b764002e 20/08/08 15:33:52.147 Thread-1 INFO ShutdownHookManager: Deleting directory /home/jenkins/workspace/spark-master-test-python-arm/python/target/c1079ab9-e95b-42b9-9d66-c20bb67941db/spark-3c380b3e-bad9-4179-b58d-fb5610c338dc/pyspark-277ce81e-dc11-4c75-95d3-fc353081e62c 20/08/08 15:33:52.150 Thread-1 INFO ShutdownHookManager: Deleting directory /home/jenkins/workspace/spark-master-test-python-arm/python/target/c1079ab9-e95b-42b9-9d66-c20bb67941db/spark-3c380b3e-bad9-4179-b58d-fb5610c338dc 20/08/08 15:33:52.163 block-manager-storage-async-thread-pool-0 DEBUG BlockManagerStorageEndpoint: removing broadcast 0 20/08/08 15:33:52.164 block-manager-storage-async-thread-pool-0 DEBUG BlockManager: Removing broadcast 0 20/08/08 15:33:52.167 block-manager-storage-async-thread-pool-0 DEBUG BlockManager: Removing block broadcast_0_piece0 20/08/08 15:33:52.185 block-manager-storage-async-thread-pool-0 DEBUG MemoryStore: Block broadcast_0_piece0 of size 13532 dropped from memory (free 384018390) 20/08/08 15:33:52.186 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master broadcast_0_piece0 for BlockManagerId(driver, 192.168.1.225, 38159, None) 20/08/08 15:33:52.187 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Removed broadcast_0_piece0 on 192.168.1.225:38159 in memory (size: 13.2 KiB, free: 366.3 MiB) 20/08/08 15:33:52.187 block-manager-storage-async-thread-pool-0 DEBUG BlockManagerMaster: Updated info of block broadcast_0_piece0 20/08/08 15:33:52.187 block-manager-storage-async-thread-pool-0 DEBUG BlockManager: Told master about block broadcast_0_piece0 20/08/08 15:33:52.188 block-manager-storage-async-thread-pool-0 DEBUG BlockManager: Removing block broadcast_0 20/08/08 15:33:52.188 block-manager-storage-async-thread-pool-0 DEBUG MemoryStore: Block broadcast_0 of size 28568 dropped from memory (free 384046958) 20/08/08 15:33:52.197 block-manager-storage-async-thread-pool-2 DEBUG BlockManagerStorageEndpoint: Done removing broadcast 0, response is 0 20/08/08 15:33:52.200 block-manager-storage-async-thread-pool-2 DEBUG BlockManagerStorageEndpoint: Sent response: 0 to 192.168.1.225:44433 20/08/08 15:33:52.200 Spark Context Cleaner DEBUG ContextCleaner: Cleaned broadcast 0 20/08/08 15:33:52.201 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(21) 20/08/08 15:33:52.201 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 21 20/08/08 15:33:52.201 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 21 20/08/08 15:33:52.201 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(23) 20/08/08 15:33:52.201 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 23 20/08/08 15:33:52.201 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 23 20/08/08 15:33:52.201 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(24) 20/08/08 15:33:52.201 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 24 20/08/08 15:33:52.201 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 24 20/08/08 15:33:52.201 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(26) 20/08/08 15:33:52.201 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 26 20/08/08 15:33:52.201 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 26 20/08/08 15:33:52.201 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(29) 20/08/08 15:33:52.201 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 29 20/08/08 15:33:52.201 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 29 20/08/08 15:33:52.201 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(16) 20/08/08 15:33:52.201 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 16 20/08/08 15:33:52.202 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 16 20/08/08 15:33:52.202 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(20) 20/08/08 15:33:52.202 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 20 20/08/08 15:33:52.202 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 20 20/08/08 15:33:52.202 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(6) 20/08/08 15:33:52.202 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 6 20/08/08 15:33:52.202 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 6 20/08/08 15:33:52.202 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(10) 20/08/08 15:33:52.202 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 10 20/08/08 15:33:52.202 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 10 20/08/08 15:33:52.202 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(12) 20/08/08 15:33:52.202 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 12 20/08/08 15:33:52.202 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 12 20/08/08 15:33:52.202 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(13) 20/08/08 15:33:52.203 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 13 20/08/08 15:33:52.203 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 13 20/08/08 15:33:52.203 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(17) 20/08/08 15:33:52.203 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 17 20/08/08 15:33:52.203 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 17 20/08/08 15:33:52.203 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(19) 20/08/08 15:33:52.203 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 19 20/08/08 15:33:52.203 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 19 20/08/08 15:33:52.203 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(22) 20/08/08 15:33:52.203 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 22 20/08/08 15:33:52.203 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 22 20/08/08 15:33:52.203 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(5) 20/08/08 15:33:52.203 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 5 20/08/08 15:33:52.203 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 5 20/08/08 15:33:52.203 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(28) 20/08/08 15:33:52.203 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 28 20/08/08 15:33:52.203 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 28 20/08/08 15:33:52.211 Executor task launch worker for task 3 DEBUG GenerateSafeProjection: code for createexternalrow(input[0, int, false], input[1, int, false], input[2, float, true], StructField(UDF(user),IntegerType,false), StructField(UDF(item),IntegerType,false), StructField(rating,FloatType,true)): /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificSafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificSafeProjection extends org.apache.spark.sql.catalyst.expressions.codegen.BaseProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private InternalRow mutableRow; /* 009 */ /* 010 */ /* 011 */ public SpecificSafeProjection(Object[] references) { /* 012 */ this.references = references; /* 013 */ mutableRow = (InternalRow) references[references.length - 1]; /* 014 */ /* 015 */ } /* 016 */ /* 017 */ public void initialize(int partitionIndex) { /* 018 */ /* 019 */ } /* 020 */ /* 021 */ public java.lang.Object apply(java.lang.Object _i) { /* 022 */ InternalRow i = (InternalRow) _i; /* 023 */ Object[] values_0 = new Object[3]; /* 024 */ /* 025 */ int value_1 = i.getInt(0); /* 026 */ if (false) { /* 027 */ values_0[0] = null; /* 028 */ } else { /* 029 */ values_0[0] = value_1; /* 030 */ } /* 031 */ /* 032 */ int value_2 = i.getInt(1); /* 033 */ if (false) { /* 034 */ values_0[1] = null; /* 035 */ } else { /* 036 */ values_0[1] = value_2; /* 037 */ } /* 038 */ /* 039 */ boolean isNull_3 = i.isNullAt(2); /* 040 */ float value_3 = isNull_3 ? /* 041 */ -1.0f : (i.getFloat(2)); /* 042 */ if (isNull_3) { /* 043 */ values_0[2] = null; /* 044 */ } else { /* 045 */ values_0[2] = value_3; /* 046 */ } /* 047 */ /* 048 */ final org.apache.spark.sql.Row value_0 = new org.apache.spark.sql.catalyst.expressions.GenericRowWithSchema(values_0, ((org.apache.spark.sql.types.StructType) references[0] /* schema */)); /* 049 */ if (false) { /* 050 */ mutableRow.setNullAt(0); /* 051 */ } else { /* 052 */ /* 053 */ mutableRow.update(0, value_0); /* 054 */ } /* 055 */ /* 056 */ return mutableRow; /* 057 */ } /* 058 */ /* 059 */ /* 060 */ } 20/08/08 15:33:52.235 Executor task launch worker for task 1 DEBUG GenerateSafeProjection: code for createexternalrow(input[0, int, false], input[1, int, false], input[2, float, true], StructField(UDF(user),IntegerType,false), StructField(UDF(item),IntegerType,false), StructField(rating,FloatType,true)): /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificSafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificSafeProjection extends org.apache.spark.sql.catalyst.expressions.codegen.BaseProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private InternalRow mutableRow; /* 009 */ /* 010 */ /* 011 */ public SpecificSafeProjection(Object[] references) { /* 012 */ this.references = references; /* 013 */ mutableRow = (InternalRow) references[references.length - 1]; /* 014 */ /* 015 */ } /* 016 */ /* 017 */ public void initialize(int partitionIndex) { /* 018 */ /* 019 */ } /* 020 */ /* 021 */ public java.lang.Object apply(java.lang.Object _i) { /* 022 */ InternalRow i = (InternalRow) _i; /* 023 */ Object[] values_0 = new Object[3]; /* 024 */ /* 025 */ int value_1 = i.getInt(0); /* 026 */ if (false) { /* 027 */ values_0[0] = null; /* 028 */ } else { /* 029 */ values_0[0] = value_1; /* 030 */ } /* 031 */ /* 032 */ int value_2 = i.getInt(1); /* 033 */ if (false) { /* 034 */ values_0[1] = null; /* 035 */ } else { /* 036 */ values_0[1] = value_2; /* 037 */ } /* 038 */ /* 039 */ boolean isNull_3 = i.isNullAt(2); /* 040 */ float value_3 = isNull_3 ? /* 041 */ -1.0f : (i.getFloat(2)); /* 042 */ if (isNull_3) { /* 043 */ values_0[2] = null; /* 044 */ } else { /* 045 */ values_0[2] = value_3; /* 046 */ } /* 047 */ /* 048 */ final org.apache.spark.sql.Row value_0 = new org.apache.spark.sql.catalyst.expressions.GenericRowWithSchema(values_0, ((org.apache.spark.sql.types.StructType) references[0] /* schema */)); /* 049 */ if (false) { /* 050 */ mutableRow.setNullAt(0); /* 051 */ } else { /* 052 */ /* 053 */ mutableRow.update(0, value_0); /* 054 */ } /* 055 */ /* 056 */ return mutableRow; /* 057 */ } /* 058 */ /* 059 */ /* 060 */ } 20/08/08 15:33:52.242 Executor task launch worker for task 2 DEBUG GenerateSafeProjection: code for createexternalrow(input[0, int, false], input[1, int, false], input[2, float, true], StructField(UDF(user),IntegerType,false), StructField(UDF(item),IntegerType,false), StructField(rating,FloatType,true)): /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificSafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificSafeProjection extends org.apache.spark.sql.catalyst.expressions.codegen.BaseProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private InternalRow mutableRow; /* 009 */ /* 010 */ /* 011 */ public SpecificSafeProjection(Object[] references) { /* 012 */ this.references = references; /* 013 */ mutableRow = (InternalRow) references[references.length - 1]; /* 014 */ /* 015 */ } /* 016 */ /* 017 */ public void initialize(int partitionIndex) { /* 018 */ /* 019 */ } /* 020 */ /* 021 */ public java.lang.Object apply(java.lang.Object _i) { /* 022 */ InternalRow i = (InternalRow) _i; /* 023 */ Object[] values_0 = new Object[3]; /* 024 */ /* 025 */ int value_1 = i.getInt(0); /* 026 */ if (false) { /* 027 */ values_0[0] = null; /* 028 */ } else { /* 029 */ values_0[0] = value_1; /* 030 */ } /* 031 */ /* 032 */ int value_2 = i.getInt(1); /* 033 */ if (false) { /* 034 */ values_0[1] = null; /* 035 */ } else { /* 036 */ values_0[1] = value_2; /* 037 */ } /* 038 */ /* 039 */ boolean isNull_3 = i.isNullAt(2); /* 040 */ float value_3 = isNull_3 ? /* 041 */ -1.0f : (i.getFloat(2)); /* 042 */ if (isNull_3) { /* 043 */ values_0[2] = null; /* 044 */ } else { /* 045 */ values_0[2] = value_3; /* 046 */ } /* 047 */ /* 048 */ final org.apache.spark.sql.Row value_0 = new org.apache.spark.sql.catalyst.expressions.GenericRowWithSchema(values_0, ((org.apache.spark.sql.types.StructType) references[0] /* schema */)); /* 049 */ if (false) { /* 050 */ mutableRow.setNullAt(0); /* 051 */ } else { /* 052 */ /* 053 */ mutableRow.update(0, value_0); /* 054 */ } /* 055 */ /* 056 */ return mutableRow; /* 057 */ } /* 058 */ /* 059 */ /* 060 */ } 20/08/08 15:33:52.245 Executor task launch worker for task 4 DEBUG GenerateSafeProjection: code for createexternalrow(input[0, int, false], input[1, int, false], input[2, float, true], StructField(UDF(user),IntegerType,false), StructField(UDF(item),IntegerType,false), StructField(rating,FloatType,true)): /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificSafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificSafeProjection extends org.apache.spark.sql.catalyst.expressions.codegen.BaseProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private InternalRow mutableRow; /* 009 */ /* 010 */ /* 011 */ public SpecificSafeProjection(Object[] references) { /* 012 */ this.references = references; /* 013 */ mutableRow = (InternalRow) references[references.length - 1]; /* 014 */ /* 015 */ } /* 016 */ /* 017 */ public void initialize(int partitionIndex) { /* 018 */ /* 019 */ } /* 020 */ /* 021 */ public java.lang.Object apply(java.lang.Object _i) { /* 022 */ InternalRow i = (InternalRow) _i; /* 023 */ Object[] values_0 = new Object[3]; /* 024 */ /* 025 */ int value_1 = i.getInt(0); /* 026 */ if (false) { /* 027 */ values_0[0] = null; /* 028 */ } else { /* 029 */ values_0[0] = value_1; /* 030 */ } /* 031 */ /* 032 */ int value_2 = i.getInt(1); /* 033 */ if (false) { /* 034 */ values_0[1] = null; /* 035 */ } else { /* 036 */ values_0[1] = value_2; /* 037 */ } /* 038 */ /* 039 */ boolean isNull_3 = i.isNullAt(2); /* 040 */ float value_3 = isNull_3 ? /* 041 */ -1.0f : (i.getFloat(2)); /* 042 */ if (isNull_3) { /* 043 */ values_0[2] = null; /* 044 */ } else { /* 045 */ values_0[2] = value_3; /* 046 */ } /* 047 */ /* 048 */ final org.apache.spark.sql.Row value_0 = new org.apache.spark.sql.catalyst.expressions.GenericRowWithSchema(values_0, ((org.apache.spark.sql.types.StructType) references[0] /* schema */)); /* 049 */ if (false) { /* 050 */ mutableRow.setNullAt(0); /* 051 */ } else { /* 052 */ /* 053 */ mutableRow.update(0, value_0); /* 054 */ } /* 055 */ /* 056 */ return mutableRow; /* 057 */ } /* 058 */ /* 059 */ /* 060 */ } 20/08/08 15:33:52.277 Executor task launch worker for task 3 INFO PythonRunner: Times: total = 56, boot = 7, init = 49, finish = 0 20/08/08 15:33:52.278 Executor task launch worker for task 1 INFO PythonRunner: Times: total = 78, boot = 14, init = 64, finish = 0 20/08/08 15:33:52.282 Executor task launch worker for task 2 INFO PythonRunner: Times: total = 78, boot = 11, init = 67, finish = 0 20/08/08 15:33:52.283 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(21) 20/08/08 15:33:52.284 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 21 20/08/08 15:33:52.286 Executor task launch worker for task 4 INFO PythonRunner: Times: total = 87, boot = 29, init = 58, finish = 0 20/08/08 15:33:52.287 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 21 20/08/08 15:33:52.287 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(19) 20/08/08 15:33:52.287 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 19 20/08/08 15:33:52.287 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 19 20/08/08 15:33:52.288 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(37) 20/08/08 15:33:52.288 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 37 20/08/08 15:33:52.288 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 37 20/08/08 15:33:52.288 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(28) 20/08/08 15:33:52.288 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 28 20/08/08 15:33:52.288 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 28 20/08/08 15:33:52.288 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(34) 20/08/08 15:33:52.288 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 34 20/08/08 15:33:52.288 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 34 20/08/08 15:33:52.288 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(16) 20/08/08 15:33:52.288 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 16 20/08/08 15:33:52.288 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 16 20/08/08 15:33:52.288 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(1) 20/08/08 15:33:52.288 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 1 20/08/08 15:33:52.288 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 1 20/08/08 15:33:52.288 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(36) 20/08/08 15:33:52.288 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 36 20/08/08 15:33:52.288 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 36 20/08/08 15:33:52.288 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(35) 20/08/08 15:33:52.288 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 35 20/08/08 15:33:52.288 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 35 20/08/08 15:33:52.288 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(22) 20/08/08 15:33:52.288 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 22 20/08/08 15:33:52.288 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 22 20/08/08 15:33:52.289 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(17) 20/08/08 15:33:52.292 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 17 20/08/08 15:33:52.293 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 17 20/08/08 15:33:52.293 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(4) 20/08/08 15:33:52.293 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 4 20/08/08 15:33:52.293 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 4 20/08/08 15:33:52.293 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(6) 20/08/08 15:33:52.293 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 6 20/08/08 15:33:52.293 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 6 20/08/08 15:33:52.293 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(14) 20/08/08 15:33:52.293 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 14 20/08/08 15:33:52.293 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 14 20/08/08 15:33:52.293 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(30) 20/08/08 15:33:52.293 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 30 20/08/08 15:33:52.293 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 30 20/08/08 15:33:52.293 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(26) 20/08/08 15:33:52.293 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 26 20/08/08 15:33:52.294 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 26 20/08/08 15:33:52.294 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(12) 20/08/08 15:33:52.294 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 12 20/08/08 15:33:52.294 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 12 20/08/08 15:33:52.294 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(31) 20/08/08 15:33:52.294 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 31 20/08/08 15:33:52.294 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 31 20/08/08 15:33:52.294 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(8) 20/08/08 15:33:52.294 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 8 20/08/08 15:33:52.294 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 8 20/08/08 15:33:52.294 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(15) 20/08/08 15:33:52.294 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 15 20/08/08 15:33:52.294 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 15 20/08/08 15:33:52.294 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(10) 20/08/08 15:33:52.294 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 10 20/08/08 15:33:52.294 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 10 20/08/08 15:33:52.294 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(2) 20/08/08 15:33:52.294 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 2 20/08/08 15:33:52.294 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 2 20/08/08 15:33:52.294 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(25) 20/08/08 15:33:52.294 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 25 20/08/08 15:33:52.294 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 25 20/08/08 15:33:52.294 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(9) 20/08/08 15:33:52.294 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 9 20/08/08 15:33:52.295 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 9 20/08/08 15:33:52.295 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(20) 20/08/08 15:33:52.295 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 20 20/08/08 15:33:52.295 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 20 20/08/08 15:33:52.295 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanBroadcast(0) 20/08/08 15:33:52.295 Thread-4 DEBUG CountVectorizerModel: Input schema: {"type":"struct","fields":[{"name":"id","type":"long","nullable":true,"metadata":{}},{"name":"words","type":{"type":"array","elementType":"string","containsNull":true},"nullable":true,"metadata":{}},{"name":"expected","type":{"type":"udt","class":"org.apache.spark.ml.linalg.VectorUDT","pyClass":"pyspark.ml.linalg.VectorUDT","sqlType":{"type":"struct","fields":[{"name":"type","type":"byte","nullable":false,"metadata":{}},{"name":"size","type":"integer","nullable":true,"metadata":{}},{"name":"indices","type":{"type":"array","elementType":"integer","containsNull":false},"nullable":true,"metadata":{}},{"name":"values","type":{"type":"array","elementType":"double","containsNull":false},"nullable":true,"metadata":{}}]}},"nullable":true,"metadata":{}}]} 20/08/08 15:33:52.295 Spark Context Cleaner DEBUG ContextCleaner: Cleaning broadcast 0 20/08/08 15:33:52.296 Thread-4 DEBUG CountVectorizerModel: Expected output schema: {"type":"struct","fields":[{"name":"id","type":"long","nullable":true,"metadata":{}},{"name":"words","type":{"type":"array","elementType":"string","containsNull":true},"nullable":true,"metadata":{}},{"name":"expected","type":{"type":"udt","class":"org.apache.spark.ml.linalg.VectorUDT","pyClass":"pyspark.ml.linalg.VectorUDT","sqlType":{"type":"struct","fields":[{"name":"type","type":"byte","nullable":false,"metadata":{}},{"name":"size","type":"integer","nullable":true,"metadata":{}},{"name":"indices","type":{"type":"array","elementType":"integer","containsNull":false},"nullable":true,"metadata":{}},{"name":"values","type":{"type":"array","elementType":"double","containsNull":false},"nullable":true,"metadata":{}}]}},"nullable":true,"metadata":{}},{"name":"CountVectorizerModel_593874e9ac1f__output","type":{"type":"udt","class":"org.apache.spark.ml.linalg.VectorUDT","pyClass":"pyspark.ml.linalg.VectorUDT","sqlType":{"type":"struct","fields":[{"name":"type","type":"byte","nullable":false,"metadata":{}},{"name":"size","type":"integer","nullable":true,"metadata":{}},{"name":"indices","type":{"type":"array","elementType":"integer","containsNull":false},"nullable":true,"metadata":{}},{"name":"values","type":{"type":"array","elementType":"double","containsNull":false},"nullable":true,"metadata":{}}]}},"nullable":false,"metadata":{"ml_attr":{"attrs":{},"num_attrs":3}}}]} 20/08/08 15:33:52.298 Thread-4 INFO MemoryStore: Block broadcast_2 stored as values in memory (estimated size 256.0 B, free 366.3 MiB) 20/08/08 15:33:52.299 Thread-4 DEBUG BlockManager: Put block broadcast_2 locally took 1 ms 20/08/08 15:33:52.299 Thread-4 DEBUG BlockManager: Putting block broadcast_2 without replication took 1 ms 20/08/08 15:33:52.300 Thread-4 INFO MemoryStore: Block broadcast_2_piece0 stored as bytes in memory (estimated size 264.0 B, free 366.3 MiB) 20/08/08 15:33:52.301 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master broadcast_2_piece0 for BlockManagerId(driver, 192.168.1.225, 36433, None) 20/08/08 15:33:52.302 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added broadcast_2_piece0 in memory on 192.168.1.225:36433 (size: 264.0 B, free: 366.3 MiB) 20/08/08 15:33:52.304 Thread-4 DEBUG BlockManagerMaster: Updated info of block broadcast_2_piece0 20/08/08 15:33:52.304 Thread-4 DEBUG BlockManager: Told master about block broadcast_2_piece0 20/08/08 15:33:52.304 Thread-4 DEBUG BlockManager: Put block broadcast_2_piece0 locally took 4 ms 20/08/08 15:33:52.304 Thread-4 DEBUG BlockManager: Putting block broadcast_2_piece0 without replication took 4 ms 20/08/08 15:33:52.305 Thread-4 INFO SparkContext: Created broadcast 2 from broadcast at CountVectorizer.scala:306 20/08/08 15:33:52.308 Spark Context Cleaner DEBUG TorrentBroadcast: Unpersisting TorrentBroadcast 0 20/08/08 15:33:52.316 Executor task launch worker for task 3 INFO Executor: Finished task 2.0 in stage 1.0 (TID 3). 2073 bytes result sent to driver 20/08/08 15:33:52.317 Executor task launch worker for task 3 DEBUG ExecutorMetricsPoller: stageTCMP: (1, 0) -> 3 20/08/08 15:33:52.318 Executor task launch worker for task 4 INFO Executor: Finished task 3.0 in stage 1.0 (TID 4). 2030 bytes result sent to driver 20/08/08 15:33:52.318 Executor task launch worker for task 4 DEBUG ExecutorMetricsPoller: stageTCMP: (1, 0) -> 2 20/08/08 15:33:52.329 dispatcher-event-loop-1 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_1.0, runningTasks: 3 20/08/08 15:33:52.330 dispatcher-event-loop-1 DEBUG TaskSetManager: No tasks for locality level NO_PREF, so moving to locality level ANY 20/08/08 15:33:52.332 dispatcher-event-loop-1 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_1.0, runningTasks: 2 20/08/08 15:33:52.334 task-result-getter-2 INFO TaskSetManager: Finished task 3.0 in stage 1.0 (TID 4) in 583 ms on 192.168.1.225 (executor driver) (1/4) 20/08/08 15:33:52.335 Executor task launch worker for task 1 INFO Executor: Finished task 0.0 in stage 1.0 (TID 1). 2030 bytes result sent to driver 20/08/08 15:33:52.335 Executor task launch worker for task 1 DEBUG ExecutorMetricsPoller: stageTCMP: (1, 0) -> 1 20/08/08 15:33:52.335 dag-scheduler-event-loop DEBUG DAGScheduler: ShuffleMapTask finished on driver 20/08/08 15:33:52.336 Executor task launch worker for task 2 INFO Executor: Finished task 1.0 in stage 1.0 (TID 2). 2030 bytes result sent to driver 20/08/08 15:33:52.336 Executor task launch worker for task 2 DEBUG ExecutorMetricsPoller: removing (1, 0) from stageTCMP 20/08/08 15:33:52.337 dispatcher-event-loop-0 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_1.0, runningTasks: 1 20/08/08 15:33:52.338 dispatcher-event-loop-0 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_1.0, runningTasks: 0 20/08/08 15:33:52.339 task-result-getter-3 INFO TaskSetManager: Finished task 0.0 in stage 1.0 (TID 1) in 591 ms on 192.168.1.225 (executor driver) (2/4) 20/08/08 15:33:52.340 Thread-4 DEBUG Analyzer$ResolveReferences: Resolving 'words to words#1 20/08/08 15:33:52.340 dag-scheduler-event-loop DEBUG DAGScheduler: ShuffleMapTask finished on driver 20/08/08 15:33:52.342 task-result-getter-3 INFO TaskSetManager: Finished task 1.0 in stage 1.0 (TID 2) in 592 ms on 192.168.1.225 (executor driver) (3/4) 20/08/08 15:33:52.343 dag-scheduler-event-loop DEBUG DAGScheduler: ShuffleMapTask finished on driver 20/08/08 15:33:52.345 task-result-getter-1 INFO TaskSetManager: Finished task 2.0 in stage 1.0 (TID 3) in 594 ms on 192.168.1.225 (executor driver) (4/4) 20/08/08 15:33:52.346 dag-scheduler-event-loop DEBUG DAGScheduler: ShuffleMapTask finished on driver 20/08/08 15:33:52.346 dag-scheduler-event-loop INFO DAGScheduler: ShuffleMapStage 1 (mapPartitions at ALS.scala:1356) finished in 0.626 s 20/08/08 15:33:52.347 dag-scheduler-event-loop INFO DAGScheduler: looking for newly runnable stages 20/08/08 15:33:52.347 dag-scheduler-event-loop INFO DAGScheduler: running: Set() 20/08/08 15:33:52.348 dag-scheduler-event-loop INFO DAGScheduler: waiting: Set(ShuffleMapStage 2, ResultStage 3) 20/08/08 15:33:52.348 dag-scheduler-event-loop INFO DAGScheduler: failed: Set() 20/08/08 15:33:52.349 dag-scheduler-event-loop DEBUG MapOutputTrackerMaster: Increasing epoch to 1 20/08/08 15:33:52.352 dag-scheduler-event-loop DEBUG DAGScheduler: submitStage(ShuffleMapStage 2 (name=map at ALS.scala:1599;jobs=1)) 20/08/08 15:33:52.353 dag-scheduler-event-loop DEBUG DAGScheduler: missing: List() 20/08/08 15:33:52.353 dag-scheduler-event-loop INFO DAGScheduler: Submitting ShuffleMapStage 2 (MapPartitionsRDD[17] at map at ALS.scala:1599), which has no missing parents 20/08/08 15:33:52.353 dag-scheduler-event-loop DEBUG DAGScheduler: submitMissingTasks(ShuffleMapStage 2) 20/08/08 15:33:52.346 task-result-getter-1 INFO TaskSchedulerImpl: Removed TaskSet 1.0, whose tasks have all completed, from pool 20/08/08 15:33:52.360 block-manager-storage-async-thread-pool-0 DEBUG BlockManagerStorageEndpoint: removing broadcast 0 20/08/08 15:33:52.370 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_2 stored as values in memory (estimated size 32.3 KiB, free 366.2 MiB) 20/08/08 15:33:52.370 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_2 locally took 1 ms 20/08/08 15:33:52.370 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_2 without replication took 1 ms 20/08/08 15:33:52.372 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_2_piece0 stored as bytes in memory (estimated size 15.2 KiB, free 366.2 MiB) 20/08/08 15:33:52.372 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master broadcast_2_piece0 for BlockManagerId(driver, 192.168.1.225, 38159, None) 20/08/08 15:33:52.373 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added broadcast_2_piece0 in memory on 192.168.1.225:38159 (size: 15.2 KiB, free: 366.3 MiB) 20/08/08 15:33:52.374 block-manager-storage-async-thread-pool-0 DEBUG BlockManager: Removing broadcast 0 20/08/08 15:33:52.376 dag-scheduler-event-loop DEBUG BlockManagerMaster: Updated info of block broadcast_2_piece0 20/08/08 15:33:52.376 dag-scheduler-event-loop DEBUG BlockManager: Told master about block broadcast_2_piece0 20/08/08 15:33:52.376 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_2_piece0 locally took 4 ms 20/08/08 15:33:52.376 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_2_piece0 without replication took 4 ms 20/08/08 15:33:52.376 dag-scheduler-event-loop INFO SparkContext: Created broadcast 2 from broadcast at DAGScheduler.scala:1348 20/08/08 15:33:52.377 dag-scheduler-event-loop INFO DAGScheduler: Submitting 4 missing tasks from ShuffleMapStage 2 (MapPartitionsRDD[17] at map at ALS.scala:1599) (first 15 tasks are for partitions Vector(0, 1, 2, 3)) 20/08/08 15:33:52.377 dag-scheduler-event-loop INFO TaskSchedulerImpl: Adding task set 2.0 with 4 tasks resource profile 0 20/08/08 15:33:52.378 dag-scheduler-event-loop DEBUG TaskSetManager: Epoch for TaskSet 2.0: 1 20/08/08 15:33:52.380 dag-scheduler-event-loop DEBUG TaskSetManager: Adding pending tasks took 1 ms 20/08/08 15:33:52.380 dag-scheduler-event-loop DEBUG TaskSetManager: Valid locality levels for TaskSet 2.0: NODE_LOCAL, ANY 20/08/08 15:33:52.381 dispatcher-event-loop-3 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_2.0, runningTasks: 0 20/08/08 15:33:52.384 dispatcher-event-loop-3 INFO TaskSetManager: Starting task 0.0 in stage 2.0 (TID 5, 192.168.1.225, executor driver, partition 0, NODE_LOCAL, 7132 bytes) taskResourceAssignments Map() 20/08/08 15:33:52.384 dispatcher-event-loop-3 INFO TaskSetManager: Starting task 1.0 in stage 2.0 (TID 6, 192.168.1.225, executor driver, partition 1, NODE_LOCAL, 7132 bytes) taskResourceAssignments Map() 20/08/08 15:33:52.385 dispatcher-event-loop-3 INFO TaskSetManager: Starting task 2.0 in stage 2.0 (TID 7, 192.168.1.225, executor driver, partition 2, NODE_LOCAL, 7132 bytes) taskResourceAssignments Map() 20/08/08 15:33:52.385 dispatcher-event-loop-3 INFO TaskSetManager: Starting task 3.0 in stage 2.0 (TID 8, 192.168.1.225, executor driver, partition 3, NODE_LOCAL, 7132 bytes) taskResourceAssignments Map() 20/08/08 15:33:52.386 Executor task launch worker for task 7 INFO Executor: Running task 2.0 in stage 2.0 (TID 7) 20/08/08 15:33:52.386 Executor task launch worker for task 8 INFO Executor: Running task 3.0 in stage 2.0 (TID 8) 20/08/08 15:33:52.386 Executor task launch worker for task 6 INFO Executor: Running task 1.0 in stage 2.0 (TID 6) 20/08/08 15:33:52.386 Executor task launch worker for task 5 INFO Executor: Running task 0.0 in stage 2.0 (TID 5) 20/08/08 15:33:52.387 Executor task launch worker for task 8 DEBUG ExecutorMetricsPoller: stageTCMP: (2, 0) -> 1 20/08/08 15:33:52.387 Executor task launch worker for task 5 DEBUG ExecutorMetricsPoller: stageTCMP: (2, 0) -> 3 20/08/08 15:33:52.387 Executor task launch worker for task 6 DEBUG ExecutorMetricsPoller: stageTCMP: (2, 0) -> 2 20/08/08 15:33:52.387 Executor task launch worker for task 7 DEBUG ExecutorMetricsPoller: stageTCMP: (2, 0) -> 4 20/08/08 15:33:52.388 Executor task launch worker for task 5 DEBUG BlockManager: Getting local block broadcast_2 20/08/08 15:33:52.388 Executor task launch worker for task 5 DEBUG BlockManager: Level for block broadcast_2 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:52.389 block-manager-storage-async-thread-pool-0 DEBUG BlockManager: Removing block broadcast_0_piece0 20/08/08 15:33:52.397 Thread-4 DEBUG Analyzer$ResolveReferences: Resolving 'CountVectorizerModel_593874e9ac1f__output to CountVectorizerModel_593874e9ac1f__output#25 20/08/08 15:33:52.400 Executor task launch worker for task 8 DEBUG BlockManager: Getting local block rdd_16_3 20/08/08 15:33:52.400 Executor task launch worker for task 8 DEBUG BlockManager: Block rdd_16_3 was not found 20/08/08 15:33:52.403 Executor task launch worker for task 8 DEBUG BlockManager: Getting remote block rdd_16_3 20/08/08 15:33:52.405 Executor task launch worker for task 7 DEBUG BlockManager: Getting local block rdd_16_2 20/08/08 15:33:52.405 Executor task launch worker for task 7 DEBUG BlockManager: Block rdd_16_2 was not found 20/08/08 15:33:52.405 Executor task launch worker for task 7 DEBUG BlockManager: Getting remote block rdd_16_2 20/08/08 15:33:52.406 block-manager-storage-async-thread-pool-0 DEBUG MemoryStore: Block broadcast_0_piece0 of size 7169 dropped from memory (free 384035037) 20/08/08 15:33:52.406 Executor task launch worker for task 7 DEBUG BlockManager: Block rdd_16_2 is unknown by block manager master 20/08/08 15:33:52.407 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master broadcast_0_piece0 for BlockManagerId(driver, 192.168.1.225, 36391, None) 20/08/08 15:33:52.409 Executor task launch worker for task 8 DEBUG BlockManager: Block rdd_16_3 is unknown by block manager master 20/08/08 15:33:52.409 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Removed broadcast_0_piece0 on 192.168.1.225:36391 in memory (size: 7.0 KiB, free: 366.3 MiB) 20/08/08 15:33:52.409 Executor task launch worker for task 6 DEBUG BlockManager: Getting local block rdd_16_1 20/08/08 15:33:52.409 Executor task launch worker for task 6 DEBUG BlockManager: Block rdd_16_1 was not found 20/08/08 15:33:52.409 Executor task launch worker for task 6 DEBUG BlockManager: Getting remote block rdd_16_1 20/08/08 15:33:52.410 block-manager-storage-async-thread-pool-0 DEBUG BlockManagerMaster: Updated info of block broadcast_0_piece0 20/08/08 15:33:52.410 block-manager-storage-async-thread-pool-0 DEBUG BlockManager: Told master about block broadcast_0_piece0 20/08/08 15:33:52.410 Executor task launch worker for task 6 DEBUG BlockManager: Block rdd_16_1 is unknown by block manager master 20/08/08 15:33:52.410 block-manager-storage-async-thread-pool-0 DEBUG BlockManager: Removing block broadcast_0 20/08/08 15:33:52.411 block-manager-storage-async-thread-pool-0 DEBUG MemoryStore: Block broadcast_0 of size 15432 dropped from memory (free 384050469) 20/08/08 15:33:52.413 Executor task launch worker for task 7 DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 1 20/08/08 15:33:52.416 Executor task launch worker for task 7 DEBUG MapOutputTrackerMaster: Convert map statuses for shuffle 1, mappers 0-4, partitions 2-3 20/08/08 15:33:52.418 Executor task launch worker for task 6 DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 1 20/08/08 15:33:52.418 Executor task launch worker for task 6 DEBUG MapOutputTrackerMaster: Convert map statuses for shuffle 1, mappers 0-4, partitions 1-2 20/08/08 15:33:52.418 Executor task launch worker for task 5 DEBUG BlockManager: Getting local block rdd_16_0 20/08/08 15:33:52.418 Executor task launch worker for task 5 DEBUG BlockManager: Block rdd_16_0 was not found 20/08/08 15:33:52.418 Executor task launch worker for task 5 DEBUG BlockManager: Getting remote block rdd_16_0 20/08/08 15:33:52.425 Executor task launch worker for task 5 DEBUG BlockManager: Block rdd_16_0 is unknown by block manager master 20/08/08 15:33:52.425 Executor task launch worker for task 5 DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 1 20/08/08 15:33:52.425 Executor task launch worker for task 5 DEBUG MapOutputTrackerMaster: Convert map statuses for shuffle 1, mappers 0-4, partitions 0-1 20/08/08 15:33:52.433 block-manager-storage-async-thread-pool-2 DEBUG BlockManagerStorageEndpoint: Done removing broadcast 0, response is 0 20/08/08 15:33:52.412 Executor task launch worker for task 8 DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 1 20/08/08 15:33:52.435 Executor task launch worker for task 5 DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 20/08/08 15:33:52.438 Executor task launch worker for task 7 DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 20/08/08 15:33:52.440 Executor task launch worker for task 8 DEBUG MapOutputTrackerMaster: Convert map statuses for shuffle 1, mappers 0-4, partitions 3-4 20/08/08 15:33:52.445 block-manager-storage-async-thread-pool-2 DEBUG BlockManagerStorageEndpoint: Sent response: 0 to 192.168.1.225:37029 20/08/08 15:33:52.446 Executor task launch worker for task 8 DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 20/08/08 15:33:52.446 Executor task launch worker for task 6 DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 20/08/08 15:33:52.453 Executor task launch worker for task 7 INFO ShuffleBlockFetcherIterator: Getting 2 (668.0 B) non-empty blocks including 2 (668.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 20/08/08 15:33:52.454 Executor task launch worker for task 7 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 16 ms 20/08/08 15:33:52.455 Executor task launch worker for task 7 DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_1_2_2,1), (shuffle_1_3_2,2) 20/08/08 15:33:52.456 Executor task launch worker for task 7 INFO BlockManager: Getting local shuffle block shuffle_1_2_2 20/08/08 15:33:52.458 Executor task launch worker for task 8 INFO ShuffleBlockFetcherIterator: Getting 2 (668.0 B) non-empty blocks including 2 (668.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 20/08/08 15:33:52.458 Executor task launch worker for task 6 INFO ShuffleBlockFetcherIterator: Getting 1 (334.0 B) non-empty blocks including 1 (334.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 20/08/08 15:33:52.458 Executor task launch worker for task 5 INFO ShuffleBlockFetcherIterator: Getting 1 (334.0 B) non-empty blocks including 1 (334.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 20/08/08 15:33:52.458 Executor task launch worker for task 6 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 11 ms 20/08/08 15:33:52.458 Executor task launch worker for task 5 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 24 ms 20/08/08 15:33:52.458 Executor task launch worker for task 8 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 11 ms 20/08/08 15:33:52.458 Executor task launch worker for task 5 DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_1_4_0,3) 20/08/08 15:33:52.458 Executor task launch worker for task 6 DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_1_1_1,0) 20/08/08 15:33:52.458 Executor task launch worker for task 5 INFO BlockManager: Getting local shuffle block shuffle_1_4_0 20/08/08 15:33:52.458 Executor task launch worker for task 8 DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_1_2_3,1), (shuffle_1_4_3,3) 20/08/08 15:33:52.458 Executor task launch worker for task 6 INFO BlockManager: Getting local shuffle block shuffle_1_1_1 20/08/08 15:33:52.458 Executor task launch worker for task 8 INFO BlockManager: Getting local shuffle block shuffle_1_2_3 20/08/08 15:33:52.460 Executor task launch worker for task 8 INFO BlockManager: Getting local shuffle block shuffle_1_4_3 20/08/08 15:33:52.460 Executor task launch worker for task 7 INFO BlockManager: Getting local shuffle block shuffle_1_3_2 20/08/08 15:33:52.460 Executor task launch worker for task 6 DEBUG ShuffleBlockFetcherIterator: Got local blocks in 13 ms 20/08/08 15:33:52.461 Executor task launch worker for task 7 DEBUG ShuffleBlockFetcherIterator: Got local blocks in 22 ms 20/08/08 15:33:52.463 Thread-4 DEBUG WholeStageCodegenExec: /* 001 */ public Object generate(Object[] references) { /* 002 */ return new GeneratedIteratorForCodegenStage1(references); /* 003 */ } /* 004 */ /* 005 */ // codegenStageId=1 /* 006 */ final class GeneratedIteratorForCodegenStage1 extends org.apache.spark.sql.execution.BufferedRowIterator { /* 007 */ private Object[] references; /* 008 */ private scala.collection.Iterator[] inputs; /* 009 */ private scala.collection.Iterator rdd_input_0; /* 010 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[] rdd_mutableStateArray_0 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[4]; /* 011 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[] rdd_mutableStateArray_1 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[5]; /* 012 */ /* 013 */ public GeneratedIteratorForCodegenStage1(Object[] references) { /* 014 */ this.references = references; /* 015 */ } /* 016 */ /* 017 */ public void init(int index, scala.collection.Iterator[] inputs) { /* 018 */ partitionIndex = index; /* 019 */ this.inputs = inputs; /* 020 */ wholestagecodegen_init_0_0(); /* 021 */ wholestagecodegen_init_0_1(); /* 022 */ /* 023 */ } /* 024 */ /* 025 */ private void project_writeFields_0_1(InternalRow project_tmpInput_0) { /* 026 */ if ((project_tmpInput_0.isNullAt(3))) { /* 027 */ rdd_mutableStateArray_0[3].setNullAt(3); /* 028 */ } else { /* 029 */ // Remember the current cursor so that we can calculate how many bytes are /* 030 */ // written later. /* 031 */ final int project_previousCursor_2 = rdd_mutableStateArray_0[3].cursor(); /* 032 */ /* 033 */ final ArrayData project_tmpInput_2 = (project_tmpInput_0.getArray(3)); /* 034 */ if (project_tmpInput_2 instanceof UnsafeArrayData) { /* 035 */ rdd_mutableStateArray_0[3].write((UnsafeArrayData) project_tmpInput_2); /* 036 */ } else { /* 037 */ final int project_numElements_1 = project_tmpInput_2.numElements(); /* 038 */ rdd_mutableStateArray_1[4].initialize(project_numElements_1); /* 039 */ /* 040 */ for (int project_index_1 = 0; project_index_1 < project_numElements_1; project_index_1++) { /* 041 */ rdd_mutableStateArray_1[4].write(project_index_1, project_tmpInput_2.getDouble(project_index_1)); /* 042 */ } /* 043 */ } /* 044 */ /* 045 */ rdd_mutableStateArray_0[3].setOffsetAndSizeFromPreviousCursor(3, project_previousCursor_2); /* 046 */ } /* 047 */ /* 048 */ } /* 049 */ /* 050 */ private void wholestagecodegen_init_0_1() { /* 051 */ rdd_mutableStateArray_1[4] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[3], 8); /* 052 */ /* 053 */ } /* 054 */ /* 055 */ private void project_writeFields_0_0(InternalRow project_tmpInput_0) { /* 056 */ rdd_mutableStateArray_0[3].write(0, (project_tmpInput_0.getByte(0))); /* 057 */ /* 058 */ if ((project_tmpInput_0.isNullAt(1))) { /* 059 */ rdd_mutableStateArray_0[3].setNullAt(1); /* 060 */ } else { /* 061 */ rdd_mutableStateArray_0[3].write(1, (project_tmpInput_0.getInt(1))); /* 062 */ } /* 063 */ /* 064 */ if ((project_tmpInput_0.isNullAt(2))) { /* 065 */ rdd_mutableStateArray_0[3].setNullAt(2); /* 066 */ } else { /* 067 */ // Remember the current cursor so that we can calculate how many bytes are /* 068 */ // written later. /* 069 */ final int project_previousCursor_1 = rdd_mutableStateArray_0[3].cursor(); /* 070 */ /* 071 */ final ArrayData project_tmpInput_1 = (project_tmpInput_0.getArray(2)); /* 072 */ if (project_tmpInput_1 instanceof UnsafeArrayData) { /* 073 */ rdd_mutableStateArray_0[3].write((UnsafeArrayData) project_tmpInput_1); /* 074 */ } else { /* 075 */ final int project_numElements_0 = project_tmpInput_1.numElements(); /* 076 */ rdd_mutableStateArray_1[3].initialize(project_numElements_0); /* 077 */ /* 078 */ for (int project_index_0 = 0; project_index_0 < project_numElements_0; project_index_0++) { /* 079 */ rdd_mutableStateArray_1[3].write(project_index_0, project_tmpInput_1.getInt(project_index_0)); /* 080 */ } /* 081 */ } /* 082 */ /* 083 */ rdd_mutableStateArray_0[3].setOffsetAndSizeFromPreviousCursor(2, project_previousCursor_1); /* 084 */ } /* 085 */ /* 086 */ } /* 087 */ /* 088 */ protected void processNext() throws java.io.IOException { /* 089 */ while ( rdd_input_0.hasNext()) { /* 090 */ InternalRow rdd_row_0 = (InternalRow) rdd_input_0.next(); /* 091 */ ((org.apache.spark.sql.execution.metric.SQLMetric) references[0] /* numOutputRows */).add(1); /* 092 */ boolean rdd_isNull_1 = rdd_row_0.isNullAt(1); /* 093 */ ArrayData rdd_value_1 = rdd_isNull_1 ? /* 094 */ null : (rdd_row_0.getArray(1)); /* 095 */ /* 096 */ Object project_arg_0 = null; /* 097 */ if (rdd_isNull_1) { /* 098 */ project_arg_0 = ((scala.Function1[]) references[1] /* converters */)[0].apply(null); /* 099 */ } else { /* 100 */ project_arg_0 = ((scala.Function1[]) references[1] /* converters */)[0].apply(rdd_value_1); /* 101 */ } /* 102 */ /* 103 */ InternalRow project_result_0 = null; /* 104 */ try { /* 105 */ project_result_0 = (InternalRow)((scala.Function1[]) references[1] /* converters */)[1].apply(((scala.Function1) references[3] /* udf */).apply(project_arg_0)); /* 106 */ } catch (Exception e) { /* 107 */ throw new org.apache.spark.SparkException(((java.lang.String) references[2] /* errMsg */), e); /* 108 */ } /* 109 */ /* 110 */ boolean project_isNull_0 = project_result_0 == null; /* 111 */ InternalRow project_value_0 = null; /* 112 */ if (!project_isNull_0) { /* 113 */ project_value_0 = project_result_0; /* 114 */ } /* 115 */ rdd_mutableStateArray_0[2].reset(); /* 116 */ /* 117 */ rdd_mutableStateArray_0[2].zeroOutNullBytes(); /* 118 */ /* 119 */ if (project_isNull_0) { /* 120 */ rdd_mutableStateArray_0[2].setNullAt(0); /* 121 */ } else { /* 122 */ final InternalRow project_tmpInput_0 = project_value_0; /* 123 */ if (project_tmpInput_0 instanceof UnsafeRow) { /* 124 */ rdd_mutableStateArray_0[2].write(0, (UnsafeRow) project_tmpInput_0); /* 125 */ } else { /* 126 */ // Remember the current cursor so that we can calculate how many bytes are /* 127 */ // written later. /* 128 */ final int project_previousCursor_0 = rdd_mutableStateArray_0[2].cursor(); /* 129 */ /* 130 */ rdd_mutableStateArray_0[3].resetRowWriter(); /* 131 */ project_writeFields_0_0(project_tmpInput_0); /* 132 */ project_writeFields_0_1(project_tmpInput_0); /* 133 */ /* 134 */ rdd_mutableStateArray_0[2].setOffsetAndSizeFromPreviousCursor(0, project_previousCursor_0); /* 135 */ } /* 136 */ } /* 137 */ append((rdd_mutableStateArray_0[2].getRow())); /* 138 */ if (shouldStop()) return; /* 139 */ } /* 140 */ } /* 141 */ /* 142 */ private void wholestagecodegen_init_0_0() { /* 143 */ rdd_input_0 = inputs[0]; /* 144 */ rdd_mutableStateArray_0[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(3, 64); /* 145 */ rdd_mutableStateArray_1[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[0], 8); /* 146 */ rdd_mutableStateArray_0[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(rdd_mutableStateArray_0[0], 4); /* 147 */ rdd_mutableStateArray_1[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[1], 4); /* 148 */ rdd_mutableStateArray_1[2] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[1], 8); /* 149 */ rdd_mutableStateArray_0[2] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(1, 32); /* 150 */ rdd_mutableStateArray_0[3] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(rdd_mutableStateArray_0[2], 4); /* 151 */ rdd_mutableStateArray_1[3] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[3], 4); /* 152 */ /* 153 */ } /* 154 */ /* 155 */ } 20/08/08 15:33:52.466 Spark Context Cleaner DEBUG ContextCleaner: Cleaned broadcast 0 20/08/08 15:33:52.466 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(11) 20/08/08 15:33:52.466 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 11 20/08/08 15:33:52.467 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 11 20/08/08 15:33:52.467 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(23) 20/08/08 15:33:52.467 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 23 20/08/08 15:33:52.467 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 23 20/08/08 15:33:52.467 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(29) 20/08/08 15:33:52.467 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 29 20/08/08 15:33:52.467 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 29 20/08/08 15:33:52.467 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(24) 20/08/08 15:33:52.467 Thread-4 DEBUG CodeGenerator: /* 001 */ public Object generate(Object[] references) { /* 002 */ return new GeneratedIteratorForCodegenStage1(references); /* 003 */ } /* 004 */ /* 005 */ // codegenStageId=1 /* 006 */ final class GeneratedIteratorForCodegenStage1 extends org.apache.spark.sql.execution.BufferedRowIterator { /* 007 */ private Object[] references; /* 008 */ private scala.collection.Iterator[] inputs; /* 009 */ private scala.collection.Iterator rdd_input_0; /* 010 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[] rdd_mutableStateArray_0 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[4]; /* 011 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[] rdd_mutableStateArray_1 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[5]; /* 012 */ /* 013 */ public GeneratedIteratorForCodegenStage1(Object[] references) { /* 014 */ this.references = references; /* 015 */ } /* 016 */ /* 017 */ public void init(int index, scala.collection.Iterator[] inputs) { /* 018 */ partitionIndex = index; /* 019 */ this.inputs = inputs; /* 020 */ wholestagecodegen_init_0_0(); /* 021 */ wholestagecodegen_init_0_1(); /* 022 */ /* 023 */ } /* 024 */ /* 025 */ private void project_writeFields_0_1(InternalRow project_tmpInput_0) { /* 026 */ if ((project_tmpInput_0.isNullAt(3))) { /* 027 */ rdd_mutableStateArray_0[3].setNullAt(3); /* 028 */ } else { /* 029 */ // Remember the current cursor so that we can calculate how many bytes are /* 030 */ // written later. /* 031 */ final int project_previousCursor_2 = rdd_mutableStateArray_0[3].cursor(); /* 032 */ /* 033 */ final ArrayData project_tmpInput_2 = (project_tmpInput_0.getArray(3)); /* 034 */ if (project_tmpInput_2 instanceof UnsafeArrayData) { /* 035 */ rdd_mutableStateArray_0[3].write((UnsafeArrayData) project_tmpInput_2); /* 036 */ } else { /* 037 */ final int project_numElements_1 = project_tmpInput_2.numElements(); /* 038 */ rdd_mutableStateArray_1[4].initialize(project_numElements_1); /* 039 */ /* 040 */ for (int project_index_1 = 0; project_index_1 < project_numElements_1; project_index_1++) { /* 041 */ rdd_mutableStateArray_1[4].write(project_index_1, project_tmpInput_2.getDouble(project_index_1)); /* 042 */ } /* 043 */ } /* 044 */ /* 045 */ rdd_mutableStateArray_0[3].setOffsetAndSizeFromPreviousCursor(3, project_previousCursor_2); /* 046 */ } /* 047 */ /* 048 */ } /* 049 */ /* 050 */ private void wholestagecodegen_init_0_1() { /* 051 */ rdd_mutableStateArray_1[4] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[3], 8); /* 052 */ /* 053 */ } /* 054 */ /* 055 */ private void project_writeFields_0_0(InternalRow project_tmpInput_0) { /* 056 */ rdd_mutableStateArray_0[3].write(0, (project_tmpInput_0.getByte(0))); /* 057 */ /* 058 */ if ((project_tmpInput_0.isNullAt(1))) { /* 059 */ rdd_mutableStateArray_0[3].setNullAt(1); /* 060 */ } else { /* 061 */ rdd_mutableStateArray_0[3].write(1, (project_tmpInput_0.getInt(1))); /* 062 */ } /* 063 */ /* 064 */ if ((project_tmpInput_0.isNullAt(2))) { /* 065 */ rdd_mutableStateArray_0[3].setNullAt(2); /* 066 */ } else { /* 067 */ // Remember the current cursor so that we can calculate how many bytes are /* 068 */ // written later. /* 069 */ final int project_previousCursor_1 = rdd_mutableStateArray_0[3].cursor(); /* 070 */ /* 071 */ final ArrayData project_tmpInput_1 = (project_tmpInput_0.getArray(2)); /* 072 */ if (project_tmpInput_1 instanceof UnsafeArrayData) { /* 073 */ rdd_mutableStateArray_0[3].write((UnsafeArrayData) project_tmpInput_1); /* 074 */ } else { /* 075 */ final int project_numElements_0 = project_tmpInput_1.numElements(); /* 076 */ rdd_mutableStateArray_1[3].initialize(project_numElements_0); /* 077 */ /* 078 */ for (int project_index_0 = 0; project_index_0 < project_numElements_0; project_index_0++) { /* 079 */ rdd_mutableStateArray_1[3].write(project_index_0, project_tmpInput_1.getInt(project_index_0)); /* 080 */ } /* 081 */ } /* 082 */ /* 083 */ rdd_mutableStateArray_0[3].setOffsetAndSizeFromPreviousCursor(2, project_previousCursor_1); /* 084 */ } /* 085 */ /* 086 */ } /* 087 */ /* 088 */ protected void processNext() throws java.io.IOException { /* 089 */ while ( rdd_input_0.hasNext()) { /* 090 */ InternalRow rdd_row_0 = (InternalRow) rdd_input_0.next(); /* 091 */ ((org.apache.spark.sql.execution.metric.SQLMetric) references[0] /* numOutputRows */).add(1); /* 092 */ boolean rdd_isNull_1 = rdd_row_0.isNullAt(1); /* 093 */ ArrayData rdd_value_1 = rdd_isNull_1 ? /* 094 */ null : (rdd_row_0.getArray(1)); /* 095 */ /* 096 */ Object project_arg_0 = null; /* 097 */ if (rdd_isNull_1) { /* 098 */ project_arg_0 = ((scala.Function1[]) references[1] /* converters */)[0].apply(null); /* 099 */ } else { /* 100 */ project_arg_0 = ((scala.Function1[]) references[1] /* converters */)[0].apply(rdd_value_1); /* 101 */ } /* 102 */ /* 103 */ InternalRow project_result_0 = null; /* 104 */ try { /* 105 */ project_result_0 = (InternalRow)((scala.Function1[]) references[1] /* converters */)[1].apply(((scala.Function1) references[3] /* udf */).apply(project_arg_0)); /* 106 */ } catch (Exception e) { /* 107 */ throw new org.apache.spark.SparkException(((java.lang.String) references[2] /* errMsg */), e); /* 108 */ } /* 109 */ /* 110 */ boolean project_isNull_0 = project_result_0 == null; /* 111 */ InternalRow project_value_0 = null; /* 112 */ if (!project_isNull_0) { /* 113 */ project_value_0 = project_result_0; /* 114 */ } /* 115 */ rdd_mutableStateArray_0[2].reset(); /* 116 */ /* 117 */ rdd_mutableStateArray_0[2].zeroOutNullBytes(); /* 118 */ /* 119 */ if (project_isNull_0) { /* 120 */ rdd_mutableStateArray_0[2].setNullAt(0); /* 121 */ } else { /* 122 */ final InternalRow project_tmpInput_0 = project_value_0; /* 123 */ if (project_tmpInput_0 instanceof UnsafeRow) { /* 124 */ rdd_mutableStateArray_0[2].write(0, (UnsafeRow) project_tmpInput_0); /* 125 */ } else { /* 126 */ // Remember the current cursor so that we can calculate how many bytes are /* 127 */ // written later. /* 128 */ final int project_previousCursor_0 = rdd_mutableStateArray_0[2].cursor(); /* 129 */ /* 130 */ rdd_mutableStateArray_0[3].resetRowWriter(); /* 131 */ project_writeFields_0_0(project_tmpInput_0); /* 132 */ project_writeFields_0_1(project_tmpInput_0); /* 133 */ /* 134 */ rdd_mutableStateArray_0[2].setOffsetAndSizeFromPreviousCursor(0, project_previousCursor_0); /* 135 */ } /* 136 */ } /* 137 */ append((rdd_mutableStateArray_0[2].getRow())); /* 138 */ if (shouldStop()) return; /* 139 */ } /* 140 */ } /* 141 */ /* 142 */ private void wholestagecodegen_init_0_0() { /* 143 */ rdd_input_0 = inputs[0]; /* 144 */ rdd_mutableStateArray_0[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(3, 64); /* 145 */ rdd_mutableStateArray_1[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[0], 8); /* 146 */ rdd_mutableStateArray_0[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(rdd_mutableStateArray_0[0], 4); /* 147 */ rdd_mutableStateArray_1[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[1], 4); /* 148 */ rdd_mutableStateArray_1[2] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[1], 8); /* 149 */ rdd_mutableStateArray_0[2] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(1, 20/08/08 15:33:52.467 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 24 32); /* 150 */ rdd_mutableStateArray_0[3] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(rdd_mutableStateArray_0[2], 4); /* 151 */ rdd_mutableStateArray_1[3] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[3], 4); /* 152 */ /* 153 */ } /* 154 */ /* 155 */ } 20/08/08 15:33:52.467 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 24 20/08/08 15:33:52.467 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(5) 20/08/08 15:33:52.467 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 5 20/08/08 15:33:52.467 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 5 20/08/08 15:33:52.467 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(3) 20/08/08 15:33:52.467 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 3 20/08/08 15:33:52.467 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 3 20/08/08 15:33:52.467 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(33) 20/08/08 15:33:52.467 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 33 20/08/08 15:33:52.467 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 33 20/08/08 15:33:52.467 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(13) 20/08/08 15:33:52.467 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 13 20/08/08 15:33:52.468 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 13 20/08/08 15:33:52.468 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(18) 20/08/08 15:33:52.468 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 18 20/08/08 15:33:52.468 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 18 20/08/08 15:33:52.468 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(27) 20/08/08 15:33:52.468 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 27 20/08/08 15:33:52.468 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 27 20/08/08 15:33:52.468 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(7) 20/08/08 15:33:52.468 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 7 20/08/08 15:33:52.468 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 7 20/08/08 15:33:52.468 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(32) 20/08/08 15:33:52.468 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 32 20/08/08 15:33:52.468 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 32 20/08/08 15:33:52.469 Executor task launch worker for task 5 DEBUG ShuffleBlockFetcherIterator: Got local blocks in 35 ms 20/08/08 15:33:52.469 Executor task launch worker for task 8 DEBUG ShuffleBlockFetcherIterator: Got local blocks in 23 ms 20/08/08 15:33:52.492 Thread-4 INFO CodeGenerator: Code generated in 28.272597 ms 20/08/08 15:33:52.493 Thread-4 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$doExecute$4$adapted 20/08/08 15:33:52.494 Thread-4 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$doExecute$4$adapted) is now cleaned +++ 20/08/08 15:33:52.498 Thread-4 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$collect$2 20/08/08 15:33:52.501 Thread-4 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$collect$2) is now cleaned +++ 20/08/08 15:33:52.504 Thread-4 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$runJob$5 20/08/08 15:33:52.506 Thread-4 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$runJob$5) is now cleaned +++ 20/08/08 15:33:52.508 Thread-4 INFO SparkContext: Starting job: at /usr/lib/python3.6/runpy.py:85 20/08/08 15:33:52.509 dag-scheduler-event-loop DEBUG DAGScheduler: Merging stage rdd profiles: Set() 20/08/08 15:33:52.510 dag-scheduler-event-loop INFO DAGScheduler: Got job 1 ( at /usr/lib/python3.6/runpy.py:85) with 4 output partitions 20/08/08 15:33:52.510 dag-scheduler-event-loop INFO DAGScheduler: Final stage: ResultStage 1 ( at /usr/lib/python3.6/runpy.py:85) 20/08/08 15:33:52.510 dag-scheduler-event-loop INFO DAGScheduler: Parents of final stage: List() 20/08/08 15:33:52.510 dag-scheduler-event-loop INFO DAGScheduler: Missing parents: List() 20/08/08 15:33:52.511 dag-scheduler-event-loop DEBUG DAGScheduler: submitStage(ResultStage 1 (name= at /usr/lib/python3.6/runpy.py:85;jobs=1)) 20/08/08 15:33:52.511 dag-scheduler-event-loop DEBUG DAGScheduler: missing: List() 20/08/08 15:33:52.511 dag-scheduler-event-loop INFO DAGScheduler: Submitting ResultStage 1 (MapPartitionsRDD[8] at at /usr/lib/python3.6/runpy.py:85), which has no missing parents 20/08/08 15:33:52.511 dag-scheduler-event-loop DEBUG DAGScheduler: submitMissingTasks(ResultStage 1) 20/08/08 15:33:52.512 Executor task launch worker for task 6 DEBUG TaskMemoryManager: Task 6 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@51ba0203 20/08/08 15:33:52.516 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_3 stored as values in memory (estimated size 27.8 KiB, free 366.2 MiB) 20/08/08 15:33:52.516 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_3 locally took 1 ms 20/08/08 15:33:52.516 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_3 without replication took 1 ms 20/08/08 15:33:52.512 Executor task launch worker for task 8 DEBUG TaskMemoryManager: Task 8 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@5d8b0ed2 20/08/08 15:33:52.518 Executor task launch worker for task 6 INFO MemoryStore: Block rdd_16_1 stored as values in memory (estimated size 176.0 B, free 366.2 MiB) 20/08/08 15:33:52.518 Executor task launch worker for task 8 INFO MemoryStore: Block rdd_16_3 stored as values in memory (estimated size 328.0 B, free 366.2 MiB) 20/08/08 15:33:52.518 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_16_3 for BlockManagerId(driver, 192.168.1.225, 38159, None) 20/08/08 15:33:52.519 Executor task launch worker for task 5 DEBUG TaskMemoryManager: Task 5 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@5fba4f9e 20/08/08 15:33:52.519 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_16_3 in memory on 192.168.1.225:38159 (size: 328.0 B, free: 366.3 MiB) 20/08/08 15:33:52.519 Executor task launch worker for task 8 DEBUG BlockManagerMaster: Updated info of block rdd_16_3 20/08/08 15:33:52.519 Executor task launch worker for task 8 DEBUG BlockManager: Told master about block rdd_16_3 20/08/08 15:33:52.519 Executor task launch worker for task 8 DEBUG BlockManager: Put block rdd_16_3 locally took 109 ms 20/08/08 15:33:52.519 Executor task launch worker for task 5 INFO MemoryStore: Block rdd_16_0 stored as values in memory (estimated size 176.0 B, free 366.2 MiB) 20/08/08 15:33:52.519 Executor task launch worker for task 7 DEBUG TaskMemoryManager: Task 7 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@191114c5 20/08/08 15:33:52.520 Executor task launch worker for task 7 INFO MemoryStore: Block rdd_16_2 stored as values in memory (estimated size 328.0 B, free 366.2 MiB) 20/08/08 15:33:52.520 Executor task launch worker for task 8 DEBUG BlockManager: Putting block rdd_16_3 without replication took 111 ms 20/08/08 15:33:52.520 Executor task launch worker for task 8 DEBUG BlockManager: Getting local block rdd_16_3 20/08/08 15:33:52.522 Executor task launch worker for task 8 DEBUG BlockManager: Level for block rdd_16_3 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:52.530 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_16_0 for BlockManagerId(driver, 192.168.1.225, 38159, None) 20/08/08 15:33:52.531 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_16_0 in memory on 192.168.1.225:38159 (size: 176.0 B, free: 366.3 MiB) 20/08/08 15:33:52.531 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_16_2 for BlockManagerId(driver, 192.168.1.225, 38159, None) 20/08/08 15:33:52.531 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_3_piece0 stored as bytes in memory (estimated size 12.4 KiB, free 366.2 MiB) 20/08/08 15:33:52.532 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_16_2 in memory on 192.168.1.225:38159 (size: 328.0 B, free: 366.3 MiB) 20/08/08 15:33:52.532 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master broadcast_3_piece0 for BlockManagerId(driver, 192.168.1.225, 36433, None) 20/08/08 15:33:52.532 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_16_1 for BlockManagerId(driver, 192.168.1.225, 38159, None) 20/08/08 15:33:52.532 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_16_1 in memory on 192.168.1.225:38159 (size: 176.0 B, free: 366.3 MiB) 20/08/08 15:33:52.532 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added broadcast_3_piece0 in memory on 192.168.1.225:36433 (size: 12.4 KiB, free: 366.3 MiB) 20/08/08 15:33:52.533 dag-scheduler-event-loop DEBUG BlockManagerMaster: Updated info of block broadcast_3_piece0 20/08/08 15:33:52.533 dag-scheduler-event-loop DEBUG BlockManager: Told master about block broadcast_3_piece0 20/08/08 15:33:52.533 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_3_piece0 locally took 2 ms 20/08/08 15:33:52.533 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_3_piece0 without replication took 2 ms 20/08/08 15:33:52.533 dag-scheduler-event-loop INFO SparkContext: Created broadcast 3 from broadcast at DAGScheduler.scala:1348 20/08/08 15:33:52.533 Executor task launch worker for task 5 DEBUG BlockManagerMaster: Updated info of block rdd_16_0 20/08/08 15:33:52.533 Executor task launch worker for task 5 DEBUG BlockManager: Told master about block rdd_16_0 20/08/08 15:33:52.534 Executor task launch worker for task 5 DEBUG BlockManager: Put block rdd_16_0 locally took 108 ms 20/08/08 15:33:52.534 Executor task launch worker for task 5 DEBUG BlockManager: Putting block rdd_16_0 without replication took 108 ms 20/08/08 15:33:52.534 Executor task launch worker for task 5 DEBUG BlockManager: Getting local block rdd_16_0 20/08/08 15:33:52.534 Executor task launch worker for task 5 DEBUG BlockManager: Level for block rdd_16_0 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:52.534 dag-scheduler-event-loop INFO DAGScheduler: Submitting 4 missing tasks from ResultStage 1 (MapPartitionsRDD[8] at at /usr/lib/python3.6/runpy.py:85) (first 15 tasks are for partitions Vector(0, 1, 2, 3)) 20/08/08 15:33:52.534 dag-scheduler-event-loop INFO TaskSchedulerImpl: Adding task set 1.0 with 4 tasks resource profile 0 20/08/08 15:33:52.535 dag-scheduler-event-loop DEBUG TaskSetManager: Epoch for TaskSet 1.0: 0 20/08/08 15:33:52.535 dag-scheduler-event-loop DEBUG TaskSetManager: Adding pending tasks took 0 ms 20/08/08 15:33:52.535 dag-scheduler-event-loop DEBUG TaskSetManager: Valid locality levels for TaskSet 1.0: NO_PREF, ANY 20/08/08 15:33:52.535 Executor task launch worker for task 8 DEBUG ALS: Converting to local indices took 0.010023472 seconds. 20/08/08 15:33:52.535 Executor task launch worker for task 6 DEBUG BlockManagerMaster: Updated info of block rdd_16_1 20/08/08 15:33:52.535 Executor task launch worker for task 6 DEBUG BlockManager: Told master about block rdd_16_1 20/08/08 15:33:52.535 Executor task launch worker for task 6 DEBUG BlockManager: Put block rdd_16_1 locally took 125 ms 20/08/08 15:33:52.535 Executor task launch worker for task 6 DEBUG BlockManager: Putting block rdd_16_1 without replication took 125 ms 20/08/08 15:33:52.535 Executor task launch worker for task 6 DEBUG BlockManager: Getting local block rdd_16_1 20/08/08 15:33:52.536 Executor task launch worker for task 6 DEBUG BlockManager: Level for block rdd_16_1 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:52.538 dispatcher-event-loop-0 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_1.0, runningTasks: 0 20/08/08 15:33:52.539 dispatcher-event-loop-0 INFO TaskSetManager: Starting task 0.0 in stage 1.0 (TID 4, 192.168.1.225, executor driver, partition 0, PROCESS_LOCAL, 7333 bytes) taskResourceAssignments Map() 20/08/08 15:33:52.539 dispatcher-event-loop-0 INFO TaskSetManager: Starting task 1.0 in stage 1.0 (TID 5, 192.168.1.225, executor driver, partition 1, PROCESS_LOCAL, 7421 bytes) taskResourceAssignments Map() 20/08/08 15:33:52.540 dispatcher-event-loop-0 INFO TaskSetManager: Starting task 2.0 in stage 1.0 (TID 6, 192.168.1.225, executor driver, partition 2, PROCESS_LOCAL, 7396 bytes) taskResourceAssignments Map() 20/08/08 15:33:52.540 dispatcher-event-loop-0 INFO TaskSetManager: Starting task 3.0 in stage 1.0 (TID 7, 192.168.1.225, executor driver, partition 3, PROCESS_LOCAL, 7385 bytes) taskResourceAssignments Map() 20/08/08 15:33:52.541 Executor task launch worker for task 4 INFO Executor: Running task 0.0 in stage 1.0 (TID 4) 20/08/08 15:33:52.541 Executor task launch worker for task 5 INFO Executor: Running task 1.0 in stage 1.0 (TID 5) 20/08/08 15:33:52.541 Executor task launch worker for task 7 INFO Executor: Running task 3.0 in stage 1.0 (TID 7) 20/08/08 15:33:52.542 Executor task launch worker for task 4 DEBUG ExecutorMetricsPoller: stageTCMP: (1, 0) -> 1 20/08/08 15:33:52.542 Executor task launch worker for task 5 DEBUG ExecutorMetricsPoller: stageTCMP: (1, 0) -> 3 20/08/08 15:33:52.542 Executor task launch worker for task 7 DEBUG ExecutorMetricsPoller: stageTCMP: (1, 0) -> 2 20/08/08 15:33:52.543 Executor task launch worker for task 4 DEBUG BlockManager: Getting local block broadcast_3 20/08/08 15:33:52.543 Executor task launch worker for task 4 DEBUG BlockManager: Level for block broadcast_3 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:52.544 Executor task launch worker for task 6 INFO Executor: Running task 2.0 in stage 1.0 (TID 6) 20/08/08 15:33:52.545 Executor task launch worker for task 6 DEBUG ExecutorMetricsPoller: stageTCMP: (1, 0) -> 4 20/08/08 15:33:52.547 Executor task launch worker for task 6 DEBUG ALS: Converting to local indices took 0.009865771 seconds. 20/08/08 15:33:52.549 Executor task launch worker for task 7 DEBUG BlockManagerMaster: Updated info of block rdd_16_2 20/08/08 15:33:52.538 Executor task launch worker for task 5 DEBUG ALS: Converting to local indices took 0.001998353 seconds. 20/08/08 15:33:52.549 Executor task launch worker for task 7 DEBUG BlockManager: Told master about block rdd_16_2 20/08/08 15:33:52.549 Executor task launch worker for task 7 DEBUG BlockManager: Put block rdd_16_2 locally took 142 ms 20/08/08 15:33:52.549 Executor task launch worker for task 7 DEBUG BlockManager: Putting block rdd_16_2 without replication took 142 ms 20/08/08 15:33:52.549 Executor task launch worker for task 7 DEBUG BlockManager: Getting local block rdd_16_2 20/08/08 15:33:52.549 Executor task launch worker for task 7 DEBUG BlockManager: Level for block rdd_16_2 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:52.550 Executor task launch worker for task 6 INFO Executor: Finished task 1.0 in stage 2.0 (TID 6). 2337 bytes result sent to driver 20/08/08 15:33:52.550 Executor task launch worker for task 6 DEBUG ExecutorMetricsPoller: stageTCMP: (2, 0) -> 3 20/08/08 15:33:52.551 dispatcher-event-loop-1 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_2.0, runningTasks: 3 20/08/08 15:33:52.551 dispatcher-event-loop-1 DEBUG TaskSetManager: No tasks for locality level NODE_LOCAL, so moving to locality level ANY 20/08/08 15:33:52.552 Executor task launch worker for task 5 INFO Executor: Finished task 0.0 in stage 2.0 (TID 5). 2337 bytes result sent to driver 20/08/08 15:33:52.552 task-result-getter-2 INFO TaskSetManager: Finished task 1.0 in stage 2.0 (TID 6) in 168 ms on 192.168.1.225 (executor driver) (1/4) 20/08/08 15:33:52.552 Executor task launch worker for task 7 DEBUG ALS: Converting to local indices took 0.001391929 seconds. 20/08/08 15:33:52.553 dag-scheduler-event-loop DEBUG DAGScheduler: ShuffleMapTask finished on driver 20/08/08 15:33:52.554 Executor task launch worker for task 7 DEBUG ALS: Converting to local indices took 0.001396908 seconds. 20/08/08 15:33:52.555 dispatcher-event-loop-2 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_2.0, runningTasks: 2 20/08/08 15:33:52.556 Executor task launch worker for task 5 DEBUG ExecutorMetricsPoller: stageTCMP: (2, 0) -> 2 20/08/08 15:33:52.556 task-result-getter-0 INFO TaskSetManager: Finished task 0.0 in stage 2.0 (TID 5) in 174 ms on 192.168.1.225 (executor driver) (2/4) 20/08/08 15:33:52.557 Executor task launch worker for task 7 INFO Executor: Finished task 2.0 in stage 2.0 (TID 7). 2337 bytes result sent to driver 20/08/08 15:33:52.557 Executor task launch worker for task 7 DEBUG ExecutorMetricsPoller: stageTCMP: (2, 0) -> 1 20/08/08 15:33:52.557 dag-scheduler-event-loop DEBUG DAGScheduler: ShuffleMapTask finished on driver 20/08/08 15:33:52.557 Executor task launch worker for task 8 DEBUG ALS: Converting to local indices took 0.021233252 seconds. 20/08/08 15:33:52.557 dispatcher-event-loop-0 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_2.0, runningTasks: 1 20/08/08 15:33:52.559 task-result-getter-3 INFO TaskSetManager: Finished task 2.0 in stage 2.0 (TID 7) in 174 ms on 192.168.1.225 (executor driver) (3/4) 20/08/08 15:33:52.560 Executor task launch worker for task 8 INFO Executor: Finished task 3.0 in stage 2.0 (TID 8). 2337 bytes result sent to driver 20/08/08 15:33:52.560 dag-scheduler-event-loop DEBUG DAGScheduler: ShuffleMapTask finished on driver 20/08/08 15:33:52.560 Executor task launch worker for task 8 DEBUG ExecutorMetricsPoller: removing (2, 0) from stageTCMP 20/08/08 15:33:52.560 dispatcher-event-loop-3 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_2.0, runningTasks: 0 20/08/08 15:33:52.561 task-result-getter-1 INFO TaskSetManager: Finished task 3.0 in stage 2.0 (TID 8) in 176 ms on 192.168.1.225 (executor driver) (4/4) 20/08/08 15:33:52.561 task-result-getter-1 INFO TaskSchedulerImpl: Removed TaskSet 2.0, whose tasks have all completed, from pool 20/08/08 15:33:52.562 dag-scheduler-event-loop DEBUG DAGScheduler: ShuffleMapTask finished on driver 20/08/08 15:33:52.562 dag-scheduler-event-loop INFO DAGScheduler: ShuffleMapStage 2 (map at ALS.scala:1599) finished in 0.200 s 20/08/08 15:33:52.563 dag-scheduler-event-loop INFO DAGScheduler: looking for newly runnable stages 20/08/08 15:33:52.563 dag-scheduler-event-loop INFO DAGScheduler: running: Set() 20/08/08 15:33:52.563 dag-scheduler-event-loop INFO DAGScheduler: waiting: Set(ResultStage 3) 20/08/08 15:33:52.563 dag-scheduler-event-loop INFO DAGScheduler: failed: Set() 20/08/08 15:33:52.563 dag-scheduler-event-loop DEBUG MapOutputTrackerMaster: Increasing epoch to 2 20/08/08 15:33:52.563 dag-scheduler-event-loop DEBUG DAGScheduler: submitStage(ResultStage 3 (name=count at ALS.scala:960;jobs=1)) 20/08/08 15:33:52.564 dag-scheduler-event-loop DEBUG DAGScheduler: missing: List() 20/08/08 15:33:52.564 dag-scheduler-event-loop INFO DAGScheduler: Submitting ResultStage 3 (userOutBlocks MapPartitionsRDD[20] at mapValues at ALS.scala:1636), which has no missing parents 20/08/08 15:33:52.565 dag-scheduler-event-loop DEBUG DAGScheduler: submitMissingTasks(ResultStage 3) 20/08/08 15:33:52.580 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_3 stored as values in memory (estimated size 33.6 KiB, free 366.2 MiB) 20/08/08 15:33:52.580 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_3 locally took 1 ms 20/08/08 15:33:52.580 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_3 without replication took 1 ms 20/08/08 15:33:52.582 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_3_piece0 stored as bytes in memory (estimated size 15.8 KiB, free 366.2 MiB) 20/08/08 15:33:52.582 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master broadcast_3_piece0 for BlockManagerId(driver, 192.168.1.225, 38159, None) 20/08/08 15:33:52.583 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added broadcast_3_piece0 in memory on 192.168.1.225:38159 (size: 15.8 KiB, free: 366.3 MiB) 20/08/08 15:33:52.583 dag-scheduler-event-loop DEBUG BlockManagerMaster: Updated info of block broadcast_3_piece0 20/08/08 15:33:52.583 dag-scheduler-event-loop DEBUG BlockManager: Told master about block broadcast_3_piece0 20/08/08 15:33:52.583 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_3_piece0 locally took 1 ms 20/08/08 15:33:52.583 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_3_piece0 without replication took 1 ms 20/08/08 15:33:52.583 dag-scheduler-event-loop INFO SparkContext: Created broadcast 3 from broadcast at DAGScheduler.scala:1348 20/08/08 15:33:52.585 dag-scheduler-event-loop INFO DAGScheduler: Submitting 10 missing tasks from ResultStage 3 (userOutBlocks MapPartitionsRDD[20] at mapValues at ALS.scala:1636) (first 15 tasks are for partitions Vector(0, 1, 2, 3, 4, 5, 6, 7, 8, 9)) 20/08/08 15:33:52.585 dag-scheduler-event-loop INFO TaskSchedulerImpl: Adding task set 3.0 with 10 tasks resource profile 0 20/08/08 15:33:52.585 dag-scheduler-event-loop DEBUG TaskSetManager: Epoch for TaskSet 3.0: 2 20/08/08 15:33:52.585 dag-scheduler-event-loop DEBUG TaskSetManager: Adding pending tasks took 0 ms 20/08/08 15:33:52.586 dag-scheduler-event-loop DEBUG TaskSetManager: Valid locality levels for TaskSet 3.0: NODE_LOCAL, NO_PREF, ANY 20/08/08 15:33:52.586 dispatcher-event-loop-1 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_3.0, runningTasks: 0 20/08/08 15:33:52.587 dispatcher-event-loop-1 INFO TaskSetManager: Starting task 0.0 in stage 3.0 (TID 9, 192.168.1.225, executor driver, partition 0, NODE_LOCAL, 7143 bytes) taskResourceAssignments Map() 20/08/08 15:33:52.587 dispatcher-event-loop-1 INFO TaskSetManager: Starting task 1.0 in stage 3.0 (TID 10, 192.168.1.225, executor driver, partition 1, NODE_LOCAL, 7143 bytes) taskResourceAssignments Map() 20/08/08 15:33:52.587 dispatcher-event-loop-1 INFO TaskSetManager: Starting task 2.0 in stage 3.0 (TID 11, 192.168.1.225, executor driver, partition 2, NODE_LOCAL, 7143 bytes) taskResourceAssignments Map() 20/08/08 15:33:52.588 dispatcher-event-loop-1 DEBUG TaskSetManager: No tasks for locality level NODE_LOCAL, so moving to locality level NO_PREF 20/08/08 15:33:52.588 dispatcher-event-loop-1 DEBUG TaskSetManager: Moving to ANY after waiting for 0ms 20/08/08 15:33:52.589 dispatcher-event-loop-1 INFO TaskSetManager: Starting task 3.0 in stage 3.0 (TID 12, 192.168.1.225, executor driver, partition 3, PROCESS_LOCAL, 7143 bytes) taskResourceAssignments Map() 20/08/08 15:33:52.589 Executor task launch worker for task 9 INFO Executor: Running task 0.0 in stage 3.0 (TID 9) 20/08/08 15:33:52.589 Executor task launch worker for task 11 INFO Executor: Running task 2.0 in stage 3.0 (TID 11) 20/08/08 15:33:52.589 Executor task launch worker for task 12 INFO Executor: Running task 3.0 in stage 3.0 (TID 12) 20/08/08 15:33:52.590 Executor task launch worker for task 9 DEBUG ExecutorMetricsPoller: stageTCMP: (3, 0) -> 1 20/08/08 15:33:52.590 Executor task launch worker for task 11 DEBUG ExecutorMetricsPoller: stageTCMP: (3, 0) -> 2 20/08/08 15:33:52.590 Executor task launch worker for task 12 DEBUG ExecutorMetricsPoller: stageTCMP: (3, 0) -> 3 20/08/08 15:33:52.591 Executor task launch worker for task 9 DEBUG BlockManager: Getting local block broadcast_3 20/08/08 15:33:52.591 Executor task launch worker for task 9 DEBUG BlockManager: Level for block broadcast_3 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:52.591 Executor task launch worker for task 10 INFO Executor: Running task 1.0 in stage 3.0 (TID 10) 20/08/08 15:33:52.592 Executor task launch worker for task 10 DEBUG ExecutorMetricsPoller: stageTCMP: (3, 0) -> 4 20/08/08 15:33:52.605 Executor task launch worker for task 10 DEBUG BlockManager: Getting local block rdd_20_1 20/08/08 15:33:52.605 Executor task launch worker for task 10 DEBUG BlockManager: Block rdd_20_1 was not found 20/08/08 15:33:52.605 Executor task launch worker for task 10 DEBUG BlockManager: Getting remote block rdd_20_1 20/08/08 15:33:52.602 Executor task launch worker for task 3 DEBUG GenerateSafeProjection: code for createexternalrow(input[0, double, true], newInstance(class org.apache.spark.ml.linalg.VectorUDT).deserialize, input[2, double, false], StructField(prediction,DoubleType,true), StructField(normalizedFeatures,org.apache.spark.ml.linalg.VectorUDT@3bfc3ba7,true), StructField(weightCol,DoubleType,false)): /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificSafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificSafeProjection extends org.apache.spark.sql.catalyst.expressions.codegen.BaseProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private InternalRow mutableRow; /* 009 */ private boolean resultIsNull_0; /* 010 */ private InternalRow[] mutableStateArray_0 = new InternalRow[1]; /* 011 */ /* 012 */ public SpecificSafeProjection(Object[] references) { /* 013 */ this.references = references; /* 014 */ mutableRow = (InternalRow) references[references.length - 1]; /* 015 */ /* 016 */ /* 017 */ } /* 018 */ /* 019 */ public void initialize(int partitionIndex) { /* 020 */ /* 021 */ } /* 022 */ /* 023 */ public java.lang.Object apply(java.lang.Object _i) { /* 024 */ InternalRow i = (InternalRow) _i; /* 025 */ Object[] values_0 = new Object[3]; /* 026 */ createExternalRow_0_0(i, values_0); /* 027 */ createExternalRow_0_1(i, values_0); /* 028 */ final org.apache.spark.sql.Row value_0 = new org.apache.spark.sql.catalyst.expressions.GenericRowWithSchema(values_0, ((org.apache.spark.sql.types.StructType) references[0] /* schema */)); /* 029 */ if (false) { /* 030 */ mutableRow.setNullAt(0); /* 031 */ } else { /* 032 */ /* 033 */ mutableRow.update(0, value_0); /* 034 */ } /* 035 */ /* 036 */ return mutableRow; /* 037 */ } /* 038 */ /* 039 */ /* 040 */ private void createExternalRow_0_1(InternalRow i, Object[] values_0) { /* 041 */ /* 042 */ double value_5 = i.getDouble(2); /* 043 */ if (false) { /* 044 */ values_0[2] = null; /* 045 */ } else { /* 046 */ values_0[2] = value_5; /* 047 */ } /* 048 */ /* 049 */ } /* 050 */ /* 051 */ /* 052 */ private void createExternalRow_0_0(InternalRow i, Object[] values_0) { /* 053 */ /* 054 */ boolean isNull_1 = i.isNullAt(0); /* 055 */ double value_1 = isNull_1 ? /* 056 */ -1.0 : (i.getDouble(0)); /* 057 */ if (isNull_1) { /* 058 */ values_0[0] = null; /* 059 */ } else { /* 060 */ values_0[0] = value_1; /* 061 */ } /* 062 */ /* 063 */ final org.apache.spark.ml.linalg.VectorUDT value_3 = false ? /* 064 */ null : new org.apache.spark.ml.linalg.VectorUDT(); /* 065 */ boolean isNull_2 = true; /* 066 */ org.apache.spark.ml.linalg.Vector value_2 = null; /* 067 */ if (!false) { /* 068 */ resultIsNull_0 = false; /* 069 */ if (!resultIsNull_0) { /* 070 */ boolean isNull_4 = i.isNullAt(1); /* 071 */ InternalRow value_4 = isNull_4 ? /* 072 */ null : (i.getStruct(1, 4)); /* 073 */ resultIsNull_0 = isNull_4; /* 074 */ mutableStateArray_0[0] = value_4; /* 075 */ } /* 076 */ /* 077 */ isNull_2 = resultIsNull_0; /* 078 */ if (!isNull_2) { /* 079 */ /* 080 */ Object funcResult_0 = null; /* 081 */ funcResult_0 = value_3.deserialize(mutableStateArray_0[0]); /* 082 */ /* 083 */ if (funcResult_0 != null) { /* 084 */ value_2 = (org.apache.spark.ml.linalg.Vector) funcResult_0; /* 085 */ } else { /* 086 */ isNull_2 = true; /* 087 */ } /* 088 */ /* 089 */ /* 090 */ } /* 091 */ } /* 092 */ if (isNull_2) { /* 093 */ values_0[1] = null; /* 094 */ } else { /* 095 */ values_0[1] = value_2; /* 096 */ } /* 097 */ /* 098 */ } /* 099 */ /* 100 */ } 20/08/08 15:33:52.606 Executor task launch worker for task 10 DEBUG BlockManager: Block rdd_20_1 is unknown by block manager master 20/08/08 15:33:52.606 Executor task launch worker for task 10 DEBUG BlockManager: Getting local block rdd_19_1 20/08/08 15:33:52.606 Executor task launch worker for task 11 DEBUG BlockManager: Getting local block rdd_20_2 20/08/08 15:33:52.607 Executor task launch worker for task 11 DEBUG BlockManager: Block rdd_20_2 was not found 20/08/08 15:33:52.607 Executor task launch worker for task 11 DEBUG BlockManager: Getting remote block rdd_20_2 20/08/08 15:33:52.606 Executor task launch worker for task 10 DEBUG BlockManager: Block rdd_19_1 was not found 20/08/08 15:33:52.607 Executor task launch worker for task 10 DEBUG BlockManager: Getting remote block rdd_19_1 20/08/08 15:33:52.608 Executor task launch worker for task 11 DEBUG BlockManager: Block rdd_20_2 is unknown by block manager master 20/08/08 15:33:52.608 Executor task launch worker for task 11 DEBUG BlockManager: Getting local block rdd_19_2 20/08/08 15:33:52.608 Executor task launch worker for task 11 DEBUG BlockManager: Block rdd_19_2 was not found 20/08/08 15:33:52.609 Executor task launch worker for task 11 DEBUG BlockManager: Getting remote block rdd_19_2 20/08/08 15:33:52.609 Executor task launch worker for task 11 DEBUG BlockManager: Block rdd_19_2 is unknown by block manager master 20/08/08 15:33:52.609 Executor task launch worker for task 11 DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 0 20/08/08 15:33:52.609 Executor task launch worker for task 11 DEBUG MapOutputTrackerMaster: Convert map statuses for shuffle 0, mappers 0-4, partitions 2-3 20/08/08 15:33:52.610 Executor task launch worker for task 11 DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 20/08/08 15:33:52.610 Executor task launch worker for task 11 INFO ShuffleBlockFetcherIterator: Getting 2 (527.0 B) non-empty blocks including 2 (527.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 20/08/08 15:33:52.610 Executor task launch worker for task 11 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 20/08/08 15:33:52.610 Executor task launch worker for task 11 DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_0_5_2,0), (shuffle_0_8_2,3) 20/08/08 15:33:52.610 Executor task launch worker for task 11 INFO BlockManager: Getting local shuffle block shuffle_0_5_2 20/08/08 15:33:52.611 Executor task launch worker for task 11 INFO BlockManager: Getting local shuffle block shuffle_0_8_2 20/08/08 15:33:52.611 Executor task launch worker for task 11 DEBUG ShuffleBlockFetcherIterator: Got local blocks in 1 ms 20/08/08 15:33:52.611 Executor task launch worker for task 12 DEBUG BlockManager: Getting local block rdd_20_3 20/08/08 15:33:52.612 Executor task launch worker for task 12 DEBUG BlockManager: Block rdd_20_3 was not found 20/08/08 15:33:52.612 Executor task launch worker for task 12 DEBUG BlockManager: Getting remote block rdd_20_3 20/08/08 15:33:52.612 Executor task launch worker for task 12 DEBUG BlockManager: Block rdd_20_3 is unknown by block manager master 20/08/08 15:33:52.612 Executor task launch worker for task 9 DEBUG BlockManager: Getting local block rdd_20_0 20/08/08 15:33:52.612 Executor task launch worker for task 12 DEBUG BlockManager: Getting local block rdd_19_3 20/08/08 15:33:52.612 Executor task launch worker for task 9 DEBUG BlockManager: Block rdd_20_0 was not found 20/08/08 15:33:52.612 Executor task launch worker for task 9 DEBUG BlockManager: Getting remote block rdd_20_0 20/08/08 15:33:52.612 Executor task launch worker for task 12 DEBUG BlockManager: Block rdd_19_3 was not found 20/08/08 15:33:52.612 Executor task launch worker for task 9 DEBUG BlockManager: Block rdd_20_0 is unknown by block manager master 20/08/08 15:33:52.612 Executor task launch worker for task 12 DEBUG BlockManager: Getting remote block rdd_19_3 20/08/08 15:33:52.612 Executor task launch worker for task 9 DEBUG BlockManager: Getting local block rdd_19_0 20/08/08 15:33:52.613 Executor task launch worker for task 9 DEBUG BlockManager: Block rdd_19_0 was not found 20/08/08 15:33:52.613 Executor task launch worker for task 9 DEBUG BlockManager: Getting remote block rdd_19_0 20/08/08 15:33:52.613 Executor task launch worker for task 12 DEBUG BlockManager: Block rdd_19_3 is unknown by block manager master 20/08/08 15:33:52.613 Executor task launch worker for task 9 DEBUG BlockManager: Block rdd_19_0 is unknown by block manager master 20/08/08 15:33:52.613 Executor task launch worker for task 10 DEBUG BlockManager: Block rdd_19_1 is unknown by block manager master 20/08/08 15:33:52.613 Executor task launch worker for task 12 DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 0 20/08/08 15:33:52.613 Executor task launch worker for task 9 DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 0 20/08/08 15:33:52.613 Executor task launch worker for task 10 DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 0 20/08/08 15:33:52.613 Executor task launch worker for task 10 DEBUG MapOutputTrackerMaster: Convert map statuses for shuffle 0, mappers 0-4, partitions 1-2 20/08/08 15:33:52.613 Executor task launch worker for task 12 DEBUG MapOutputTrackerMaster: Convert map statuses for shuffle 0, mappers 0-4, partitions 3-4 20/08/08 15:33:52.614 Executor task launch worker for task 10 DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 20/08/08 15:33:52.613 Executor task launch worker for task 9 DEBUG MapOutputTrackerMaster: Convert map statuses for shuffle 0, mappers 0-4, partitions 0-1 20/08/08 15:33:52.615 Executor task launch worker for task 10 INFO ShuffleBlockFetcherIterator: Getting 2 (502.0 B) non-empty blocks including 2 (502.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 20/08/08 15:33:52.615 Executor task launch worker for task 10 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 20/08/08 15:33:52.615 Executor task launch worker for task 10 DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_0_7_1,2), (shuffle_0_8_1,3) 20/08/08 15:33:52.615 Executor task launch worker for task 10 INFO BlockManager: Getting local shuffle block shuffle_0_7_1 20/08/08 15:33:52.615 Executor task launch worker for task 9 DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 20/08/08 15:33:52.615 Executor task launch worker for task 10 INFO BlockManager: Getting local shuffle block shuffle_0_8_1 20/08/08 15:33:52.615 Executor task launch worker for task 10 DEBUG ShuffleBlockFetcherIterator: Got local blocks in 1 ms 20/08/08 15:33:52.616 Executor task launch worker for task 9 INFO ShuffleBlockFetcherIterator: Getting 2 (502.0 B) non-empty blocks including 2 (502.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 20/08/08 15:33:52.616 Executor task launch worker for task 9 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 20/08/08 15:33:52.616 Executor task launch worker for task 9 DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_0_6_0,1), (shuffle_0_7_0,2) 20/08/08 15:33:52.616 Executor task launch worker for task 9 INFO BlockManager: Getting local shuffle block shuffle_0_6_0 20/08/08 15:33:52.616 Executor task launch worker for task 9 INFO BlockManager: Getting local shuffle block shuffle_0_7_0 20/08/08 15:33:52.616 Executor task launch worker for task 9 DEBUG ShuffleBlockFetcherIterator: Got local blocks in 1 ms 20/08/08 15:33:52.615 Executor task launch worker for task 12 DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 20/08/08 15:33:52.619 Executor task launch worker for task 12 INFO ShuffleBlockFetcherIterator: Getting 0 (0.0 B) non-empty blocks including 0 (0.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 20/08/08 15:33:52.619 Executor task launch worker for task 12 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 4 ms 20/08/08 15:33:52.619 Executor task launch worker for task 12 DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: 20/08/08 15:33:52.619 Executor task launch worker for task 12 DEBUG ShuffleBlockFetcherIterator: Got local blocks in 4 ms 20/08/08 15:33:52.620 Executor task launch worker for task 12 DEBUG TaskMemoryManager: Task 12 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@1060a40e 20/08/08 15:33:52.620 Executor task launch worker for task 12 INFO MemoryStore: Block rdd_19_3 stored as values in memory (estimated size 16.0 B, free 366.2 MiB) 20/08/08 15:33:52.620 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_19_3 for BlockManagerId(driver, 192.168.1.225, 38159, None) 20/08/08 15:33:52.621 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_19_3 in memory on 192.168.1.225:38159 (size: 16.0 B, free: 366.3 MiB) 20/08/08 15:33:52.608 Executor task launch worker for task 3 DEBUG CodeGenerator: /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificSafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificSafeProjection extends org.apache.spark.sql.catalyst.expressions.codegen.BaseProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private InternalRow mutableRow; /* 009 */ private boolean resultIsNull_0; /* 010 */ private InternalRow[] mutableStateArray_0 = new InternalRow[1]; /* 011 */ /* 012 */ public SpecificSafeProjection(Object[] references) { /* 013 */ this.references = references; /* 014 */ mutableRow = (InternalRow) references[references.length - 1]; /* 015 */ /* 016 */ /* 017 */ } /* 018 */ /* 019 */ public void initialize(int partitionIndex) { /* 020 */ /* 021 */ } /* 022 */ /* 023 */ public java.lang.Object apply(java.lang.Object _i) { /* 024 */ InternalRow i = (InternalRow) _i; /* 025 */ Object[] values_0 = new Object[3]; /* 026 */ createExternalRow_0_0(i, values_0); /* 027 */ createExternalRow_0_1(i, values_0); /* 028 */ final org.apache.spark.sql.Row value_0 = new org.apache.spark.sql.catalyst.expressions.GenericRowWithSchema(values_0, ((org.apache.spark.sql.types.StructType) references[0] /* schema */)); /* 029 */ if (false) { /* 030 */ mutableRow.setNullAt(0); /* 031 */ } else { /* 032 */ /* 033 */ mutableRow.update(0, value_0); /* 034 */ } /* 035 */ /* 036 */ return mutableRow; /* 037 */ } /* 038 */ /* 039 */ /* 040 */ private void createExternalRow_0_1(InternalRow i, Object[] values_0) { /* 041 */ /* 042 */ double value_5 = i.getDouble(2); /* 043 */ if (false) { /* 044 */ values_0[2] = null; /* 045 */ } else { /* 046 */ values_0[2] = value_5; /* 047 */ } /* 048 */ /* 049 */ } /* 050 */ /* 051 */ /* 052 */ private void createExternalRow_0_0(InternalRow i, Object[] values_0) { /* 053 */ /* 054 */ boolean isNull_1 = i.isNullAt(0); /* 055 */ double value_1 = isNull_1 ? /* 056 */ -1.0 : (i.getDouble(0)); /* 057 */ if (isNull_1) { /* 058 */ values_0[0] = null; /* 059 */ } else { /* 060 */ values_0[0] = value_1; /* 061 */ } /* 062 */ /* 063 */ final org.apache.spark.ml.linalg.VectorUDT value_3 = false ? /* 064 */ null : new org.apache.spark.ml.linalg.VectorUDT(); /* 065 */ boolean isNull_2 = true; /* 066 */ org.apache.spark.ml.linalg.Vector value_2 = null; /* 067 */ if (!false) { /* 068 */ resultIsNull_0 = false; /* 069 */ if (!resultIsNull_0) { /* 070 */ boolean isNull_4 = i.isNullAt(1); /* 071 */ InternalRow value_4 = isNull_4 ? /* 072 */ null : (i.getStruct(1, 4)); /* 073 */ resultIsNull_0 = isNull_4; /* 074 */ mutableStateArray_0[0] = value_4; /* 075 */ } /* 076 */ /* 077 */ isNull_2 = resultIsNull_0; /* 078 */ if (!isNull_2) { /* 079 */ /* 080 */ Object funcResult_0 = null; /* 081 */ funcResult_0 = value_3.deserialize(mutableStateArray_0[0]); /* 082 */ /* 083 */ if (funcResult_0 != null) { /* 084 */ value_2 = (org.apache.spark.ml.linalg.Vector) funcResult_0; /* 085 */ } else { /* 086 */ isNull_2 = true; /* 087 */ } /* 088 */ /* 089 */ /* 090 */ } /* 091 */ } /* 092 */ if (isNull_2) { /* 093 */ values_0[1] = null; /* 094 */ } else { /* 095 */ values_0[1] = value_2; /* 096 */ } /* 097 */ /* 098 */ } /* 099 */ /* 100 */ } 20/08/08 15:33:52.629 Executor task launch worker for task 12 DEBUG BlockManagerMaster: Updated info of block rdd_19_3 20/08/08 15:33:52.629 Executor task launch worker for task 12 DEBUG BlockManager: Told master about block rdd_19_3 20/08/08 15:33:52.629 Executor task launch worker for task 12 DEBUG BlockManager: Put block rdd_19_3 locally took 16 ms 20/08/08 15:33:52.630 Executor task launch worker for task 12 DEBUG BlockManager: Putting block rdd_19_3 without replication took 16 ms 20/08/08 15:33:52.630 Executor task launch worker for task 12 DEBUG BlockManager: Getting local block rdd_19_3 20/08/08 15:33:52.630 Executor task launch worker for task 12 DEBUG BlockManager: Level for block rdd_19_3 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:52.631 Executor task launch worker for task 12 INFO MemoryStore: Block rdd_20_3 stored as values in memory (estimated size 16.0 B, free 366.2 MiB) 20/08/08 15:33:52.633 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_20_3 for BlockManagerId(driver, 192.168.1.225, 38159, None) 20/08/08 15:33:52.633 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_20_3 in memory on 192.168.1.225:38159 (size: 16.0 B, free: 366.3 MiB) 20/08/08 15:33:52.634 Executor task launch worker for task 12 DEBUG BlockManagerMaster: Updated info of block rdd_20_3 20/08/08 15:33:52.634 Executor task launch worker for task 12 DEBUG BlockManager: Told master about block rdd_20_3 20/08/08 15:33:52.634 Executor task launch worker for task 12 DEBUG BlockManager: Put block rdd_20_3 locally took 21 ms 20/08/08 15:33:52.634 Executor task launch worker for task 12 DEBUG BlockManager: Putting block rdd_20_3 without replication took 21 ms 20/08/08 15:33:52.634 Executor task launch worker for task 12 DEBUG BlockManager: Getting local block rdd_20_3 20/08/08 15:33:52.634 Executor task launch worker for task 12 DEBUG BlockManager: Level for block rdd_20_3 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:52.635 Executor task launch worker for task 12 INFO Executor: Finished task 3.0 in stage 3.0 (TID 12). 2170 bytes result sent to driver 20/08/08 15:33:52.636 dispatcher-event-loop-0 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_3.0, runningTasks: 3 20/08/08 15:33:52.636 dispatcher-event-loop-0 DEBUG TaskSetManager: No tasks for locality level NODE_LOCAL, so moving to locality level NO_PREF 20/08/08 15:33:52.636 dispatcher-event-loop-0 DEBUG TaskSetManager: Moving to ANY after waiting for 0ms 20/08/08 15:33:52.637 dispatcher-event-loop-0 INFO TaskSetManager: Starting task 4.0 in stage 3.0 (TID 13, 192.168.1.225, executor driver, partition 4, PROCESS_LOCAL, 7143 bytes) taskResourceAssignments Map() 20/08/08 15:33:52.639 Executor task launch worker for task 12 DEBUG ExecutorMetricsPoller: stageTCMP: (3, 0) -> 3 20/08/08 15:33:52.626 Executor task launch worker for task 4 DEBUG GenerateSafeProjection: code for createexternalrow(input[0, double, true], newInstance(class org.apache.spark.ml.linalg.VectorUDT).deserialize, input[2, double, false], StructField(prediction,DoubleType,true), StructField(normalizedFeatures,org.apache.spark.ml.linalg.VectorUDT@3bfc3ba7,true), StructField(weightCol,DoubleType,false)): /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificSafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificSafeProjection extends org.apache.spark.sql.catalyst.expressions.codegen.BaseProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private InternalRow mutableRow; /* 009 */ private boolean resultIsNull_0; /* 010 */ private InternalRow[] mutableStateArray_0 = new InternalRow[1]; /* 011 */ /* 012 */ public SpecificSafeProjection(Object[] references) { /* 013 */ this.references = references; /* 014 */ mutableRow = (InternalRow) references[references.length - 1]; /* 015 */ /* 016 */ /* 017 */ } /* 018 */ /* 019 */ public void initialize(int partitionIndex) { /* 020 */ /* 021 */ } /* 022 */ /* 023 */ public java.lang.Object apply(java.lang.Object _i) { /* 024 */ InternalRow i = (InternalRow) _i; /* 025 */ Object[] values_0 = new Object[3]; /* 026 */ createExternalRow_0_0(i, values_0); /* 027 */ createExternalRow_0_1(i, values_0); /* 028 */ final org.apache.spark.sql.Row value_0 = new org.apache.spark.sql.catalyst.expressions.GenericRowWithSchema(values_0, ((org.apache.spark.sql.types.StructType) references[0] /* schema */)); /* 029 */ if (false) { /* 030 */ mutableRow.setNullAt(0); /* 031 */ } else { /* 032 */ /* 033 */ mutableRow.update(0, value_0); /* 034 */ } /* 035 */ /* 036 */ return mutableRow; /* 037 */ } /* 038 */ /* 039 */ /* 040 */ private void createExternalRow_0_1(InternalRow i, Object[] values_0) { /* 041 */ /* 042 */ double value_5 = i.getDouble(2); /* 043 */ if (false) { /* 044 */ values_0[2] = null; /* 045 */ } else { /* 046 */ values_0[2] = value_5; /* 047 */ } /* 048 */ /* 049 */ } /* 050 */ /* 051 */ /* 052 */ private void createExternalRow_0_0(InternalRow i, Object[] values_0) { /* 053 */ /* 054 */ boolean isNull_1 = i.isNullAt(0); /* 055 */ double value_1 = isNull_1 ? /* 056 */ -1.0 : (i.getDouble(0)); /* 057 */ if (isNull_1) { /* 058 */ values_0[0] = null; /* 059 */ } else { /* 060 */ values_0[0] = value_1; /* 061 */ } /* 062 */ /* 063 */ final org.apache.spark.ml.linalg.VectorUDT value_3 = false ? /* 064 */ null : new org.apache.spark.ml.linalg.VectorUDT(); /* 065 */ boolean isNull_2 = true; /* 066 */ org.apache.spark.ml.linalg.Vector value_2 = null; /* 067 */ if (!false) { /* 068 */ resultIsNull_0 = false; /* 069 */ if (!resultIsNull_0) { /* 070 */ boolean isNull_4 = i.isNullAt(1); /* 071 */ InternalRow value_4 = isNull_4 ? /* 072 */ null : (i.getStruct(1, 4)); /* 073 */ resultIsNull_0 = isNull_4; /* 074 */ mutableStateArray_0[0] = value_4; /* 075 */ } /* 076 */ /* 077 */ isNull_2 = resultIsNull_0; /* 078 */ if (!isNull_2) { /* 079 */ /* 080 */ Object funcResult_0 = null; /* 081 */ funcResult_0 = value_3.deserialize(mutableStateArray_0[0]); /* 082 */ /* 083 */ if (funcResult_0 != null) { /* 084 */ value_2 = (org.apache.spark.ml.linalg.Vector) funcResult_0; /* 085 */ } else { /* 086 */ isNull_2 = true; /* 087 */ } /* 088 */ /* 089 */ /* 090 */ } /* 091 */ } /* 092 */ if (isNull_2) { /* 093 */ values_0[1] = null; /* 094 */ } else { /* 095 */ values_0[1] = value_2; /* 096 */ } /* 097 */ /* 098 */ } /* 099 */ /* 100 */ } 20/08/08 15:33:52.649 Executor task launch worker for task 4 INFO PythonRunner: Times: total = 76, boot = -749, init = 825, finish = 0 20/08/08 15:33:52.651 Executor task launch worker for task 5 DEBUG GenerateSafeProjection: code for mapobjects(lambdavariable(MapObject, StringType, true, -1), lambdavariable(MapObject, StringType, true, -1).toString, input[0, array, true], Some(interface scala.collection.Seq)): /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificSafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificSafeProjection extends org.apache.spark.sql.catalyst.expressions.codegen.BaseProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private InternalRow mutableRow; /* 009 */ private UTF8String value_MapObject_lambda_variable_1; /* 010 */ private boolean isNull_MapObject_lambda_variable_1; /* 011 */ private boolean globalIsNull_0; /* 012 */ /* 013 */ public SpecificSafeProjection(Object[] references) { /* 014 */ this.references = references; /* 015 */ mutableRow = (InternalRow) references[references.length - 1]; /* 016 */ /* 017 */ /* 018 */ } /* 019 */ /* 020 */ public void initialize(int partitionIndex) { /* 021 */ /* 022 */ } /* 023 */ /* 024 */ public java.lang.Object apply(java.lang.Object _i) { /* 025 */ InternalRow i = (InternalRow) _i; /* 026 */ scala.collection.Seq value_3 = MapObjects_0(i); /* 027 */ if (globalIsNull_0) { /* 028 */ mutableRow.setNullAt(0); /* 029 */ } else { /* 030 */ /* 031 */ mutableRow.update(0, value_3); /* 032 */ } /* 033 */ /* 034 */ return mutableRow; /* 035 */ } /* 036 */ /* 037 */ /* 038 */ private scala.collection.Seq MapObjects_0(InternalRow i) { /* 039 */ boolean isNull_1 = i.isNullAt(0); /* 040 */ ArrayData value_1 = isNull_1 ? /* 041 */ null : (i.getArray(0)); /* 042 */ scala.collection.Seq value_0 = null; /* 043 */ /* 044 */ if (!isNull_1) { /* 045 */ /* 046 */ int dataLength_0 = value_1.numElements(); /* 047 */ /* 048 */ scala.collection.mutable.Builder collectionBuilder_0 = scala.collection.Seq$.MODULE$.newBuilder(); /* 049 */ collectionBuilder_0.sizeHint(dataLength_0); /* 050 */ /* 051 */ /* 052 */ int loopIndex_0 = 0; /* 053 */ /* 054 */ while (loopIndex_0 < dataLength_0) { /* 055 */ value_MapObject_lambda_variable_1 = (UTF8String) (value_1.getUTF8String(loopIndex_0)); /* 056 */ isNull_MapObject_lambda_variable_1 = value_1.isNullAt(loopIndex_0); /* 057 */ /* 058 */ boolean isNull_2 = true; /* 059 */ java.lang.String value_2 = null; /* 060 */ if (!isNull_MapObject_lambda_variable_1) { /* 061 */ /* 062 */ isNull_2 = false; /* 063 */ if (!isNull_2) { /* 064 */ /* 065 */ Object funcResult_0 = null; /* 066 */ funcResult_0 = value_MapObject_lambda_variable_1.toString(); /* 067 */ value_2 = (java.lang.String) funcResult_0; /* 068 */ /* 069 */ } /* 070 */ } /* 071 */ if (isNull_2) { /* 072 */ collectionBuilder_0.$plus$eq(null); /* 073 */ } else { /* 074 */ collectionBuilder_0.$plus$eq(value_2); /* 075 */ } /* 076 */ /* 077 */ loopIndex_0 += 1; /* 078 */ } /* 079 */ /* 080 */ value_0 = (scala.collection.Seq) collectionBuilder_0.result(); /* 081 */ } /* 082 */ globalIsNull_0 = isNull_1; /* 083 */ return value_0; /* 084 */ } /* 085 */ /* 086 */ } 20/08/08 15:33:52.652 Executor task launch worker for task 4 INFO Executor: Finished task 0.0 in stage 1.0 (TID 4). 1999 bytes result sent to driver 20/08/08 15:33:52.652 Executor task launch worker for task 4 DEBUG ExecutorMetricsPoller: stageTCMP: (1, 0) -> 3 20/08/08 15:33:52.654 Executor task launch worker for task 10 DEBUG ALS: Start sorting an uncompressed in-block of size 2. (sortId = 1118344319) 20/08/08 15:33:52.634 Executor task launch worker for task 2 DEBUG GenerateSafeProjection: code for createexternalrow(input[0, double, true], newInstance(class org.apache.spark.ml.linalg.VectorUDT).deserialize, input[2, double, false], StructField(prediction,DoubleType,true), StructField(normalizedFeatures,org.apache.spark.ml.linalg.VectorUDT@3bfc3ba7,true), StructField(weightCol,DoubleType,false)): /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificSafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificSafeProjection extends org.apache.spark.sql.catalyst.expressions.codegen.BaseProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private InternalRow mutableRow; /* 009 */ private boolean resultIsNull_0; /* 010 */ private InternalRow[] mutableStateArray_0 = new InternalRow[1]; /* 011 */ /* 012 */ public SpecificSafeProjection(Object[] references) { /* 013 */ this.references = references; /* 014 */ mutableRow = (InternalRow) references[references.length - 1]; /* 015 */ /* 016 */ /* 017 */ } /* 018 */ /* 019 */ public void initialize(int partitionIndex) { /* 020 */ /* 021 */ } /* 022 */ /* 023 */ public java.lang.Object apply(java.lang.Object _i) { /* 024 */ InternalRow i = (InternalRow) _i; /* 025 */ Object[] values_0 = new Object[3]; /* 026 */ createExternalRow_0_0(i, values_0); /* 027 */ createExternalRow_0_1(i, values_0); /* 028 */ final org.apache.spark.sql.Row value_0 = new org.apache.spark.sql.catalyst.expressions.GenericRowWithSchema(values_0, ((org.apache.spark.sql.types.StructType) references[0] /* schema */)); /* 029 */ if (false) { /* 030 */ mutableRow.setNullAt(0); /* 031 */ } else { /* 032 */ /* 033 */ mutableRow.update(0, value_0); /* 034 */ } /* 035 */ /* 036 */ return mutableRow; /* 037 */ } /* 038 */ /* 039 */ /* 040 */ private void createExternalRow_0_1(InternalRow i, Object[] values_0) { /* 041 */ /* 042 */ double value_5 = i.getDouble(2); /* 043 */ if (false) { /* 044 */ values_0[2] = null; /* 045 */ } else { /* 046 */ values_0[2] = value_5; /* 047 */ } /* 048 */ /* 049 */ } /* 050 */ /* 051 */ /* 052 */ private void createExternalRow_0_0(InternalRow i, Object[] values_0) { /* 053 */ /* 054 */ boolean isNull_1 = i.isNullAt(0); /* 055 */ double value_1 = isNull_1 ? /* 056 */ -1.0 : (i.getDouble(0)); /* 057 */ if (isNull_1) { /* 058 */ values_0[0] = null; /* 059 */ } else { /* 060 */ values_0[0] = value_1; /* 061 */ } /* 062 */ /* 063 */ final org.apache.spark.ml.linalg.VectorUDT value_3 = false ? /* 064 */ null : new org.apache.spark.ml.linalg.VectorUDT(); /* 065 */ boolean isNull_2 = true; /* 066 */ org.apache.spark.ml.linalg.Vector value_2 = null; /* 067 */ if (!false) { /* 068 */ resultIsNull_0 = false; /* 069 */ if (!resultIsNull_0) { /* 070 */ boolean isNull_4 = i.isNullAt(1); /* 071 */ InternalRow value_4 = isNull_4 ? /* 072 */ null : (i.getStruct(1, 4)); /* 073 */ resultIsNull_0 = isNull_4; /* 074 */ mutableStateArray_0[0] = value_4; /* 075 */ } /* 076 */ /* 077 */ isNull_2 = resultIsNull_0; /* 078 */ if (!isNull_2) { /* 079 */ /* 080 */ Object funcResult_0 = null; /* 081 */ funcResult_0 = value_3.deserialize(mutableStateArray_0[0]); /* 082 */ /* 083 */ if (funcResult_0 != null) { /* 084 */ value_2 = (org.apache.spark.ml.linalg.Vector) funcResult_0; /* 085 */ } else { /* 086 */ isNull_2 = true; /* 087 */ } /* 088 */ /* 089 */ /* 090 */ } /* 091 */ } /* 092 */ if (isNull_2) { /* 093 */ values_0[1] = null; /* 094 */ } else { /* 095 */ values_0[1] = value_2; /* 096 */ } /* 097 */ /* 098 */ } /* 099 */ /* 100 */ } 20/08/08 15:33:52.655 Executor task launch worker for task 1 DEBUG GenerateSafeProjection: code for createexternalrow(input[0, double, true], newInstance(class org.apache.spark.ml.linalg.VectorUDT).deserialize, input[2, double, false], StructField(prediction,DoubleType,true), StructField(normalizedFeatures,org.apache.spark.ml.linalg.VectorUDT@3bfc3ba7,true), StructField(weightCol,DoubleType,false)): /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificSafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificSafeProjection extends org.apache.spark.sql.catalyst.expressions.codegen.BaseProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private InternalRow mutableRow; /* 009 */ private boolean resultIsNull_0; /* 010 */ private InternalRow[] mutableStateArray_0 = new InternalRow[1]; /* 011 */ /* 012 */ public SpecificSafeProjection(Object[] references) { /* 013 */ this.references = references; /* 014 */ mutableRow = (InternalRow) references[references.length - 1]; /* 015 */ /* 016 */ /* 017 */ } /* 018 */ /* 019 */ public void initialize(int partitionIndex) { /* 020 */ /* 021 */ } /* 022 */ /* 023 */ public java.lang.Object apply(java.lang.Object _i) { /* 024 */ InternalRow i = (InternalRow) _i; /* 025 */ Object[] values_0 = new Object[3]; /* 026 */ createExternalRow_0_0(i, values_0); /* 027 */ createExternalRow_0_1(i, values_0); /* 028 */ final org.apache.spark.sql.Row value_0 = new org.apache.spark.sql.catalyst.expressions.GenericRowWithSchema(values_0, ((org.apache.spark.sql.types.StructType) references[0] /* schema */)); /* 029 */ if (false) { /* 030 */ mutableRow.setNullAt(0); /* 031 */ } else { /* 032 */ /* 033 */ mutableRow.update(0, value_0); /* 034 */ } /* 035 */ /* 036 */ return mutableRow; /* 037 */ } /* 038 */ /* 039 */ /* 040 */ private void createExternalRow_0_1(InternalRow i, Object[] values_0) { /* 041 */ /* 042 */ double value_5 = i.getDouble(2); /* 043 */ if (false) { /* 044 */ values_0[2] = null; /* 045 */ } else { /* 046 */ values_0[2] = value_5; /* 047 */ } /* 048 */ /* 049 */ } /* 050 */ /* 051 */ /* 052 */ private void createExternalRow_0_0(InternalRow i, Object[] values_0) { /* 053 */ /* 054 */ boolean isNull_1 = i.isNullAt(0); /* 055 */ double value_1 = isNull_1 ? /* 056 */ -1.0 : (i.getDouble(0)); /* 057 */ if (isNull_1) { /* 058 */ values_0[0] = null; /* 059 */ } else { /* 060 */ values_0[0] = value_1; /* 061 */ } /* 062 */ /* 063 */ final org.apache.spark.ml.linalg.VectorUDT value_3 = false ? /* 064 */ null : new org.apache.spark.ml.linalg.VectorUDT(); /* 065 */ boolean isNull_2 = true; /* 066 */ org.apache.spark.ml.linalg.Vector value_2 = null; /* 067 */ if (!false) { /* 068 */ resultIsNull_0 = false; /* 069 */ if (!resultIsNull_0) { /* 070 */ boolean isNull_4 = i.isNullAt(1); /* 071 */ InternalRow value_4 = isNull_4 ? /* 072 */ null : (i.getStruct(1, 4)); /* 073 */ resultIsNull_0 = isNull_4; /* 074 */ mutableStateArray_0[0] = value_4; /* 075 */ } /* 076 */ /* 077 */ isNull_2 = resultIsNull_0; /* 078 */ if (!isNull_2) { /* 079 */ /* 080 */ Object funcResult_0 = null; /* 081 */ funcResult_0 = value_3.deserialize(mutableStateArray_0[0]); /* 082 */ /* 083 */ if (funcResult_0 != null) { /* 084 */ value_2 = (org.apache.spark.ml.linalg.Vector) funcResult_0; /* 085 */ } else { /* 086 */ isNull_2 = true; /* 087 */ } /* 088 */ /* 089 */ /* 090 */ } /* 091 */ } /* 092 */ if (isNull_2) { /* 093 */ values_0[1] = null; /* 094 */ } else { /* 095 */ values_0[1] = value_2; /* 096 */ } /* 097 */ /* 098 */ } /* 099 */ /* 100 */ } 20/08/08 15:33:52.662 Executor task launch worker for task 10 DEBUG ALS: Sorting took 0.002097773 seconds. (sortId = 1118344319) 20/08/08 15:33:52.674 Executor task launch worker for task 13 INFO Executor: Running task 4.0 in stage 3.0 (TID 13) 20/08/08 15:33:52.675 Executor task launch worker for task 9 DEBUG ALS: Start sorting an uncompressed in-block of size 2. (sortId = -728070258) 20/08/08 15:33:52.675 Executor task launch worker for task 9 DEBUG ALS: Sorting took 4.194E-5 seconds. (sortId = -728070258) 20/08/08 15:33:52.676 Executor task launch worker for task 13 DEBUG ExecutorMetricsPoller: stageTCMP: (3, 0) -> 4 20/08/08 15:33:52.676 Executor task launch worker for task 10 DEBUG TaskMemoryManager: Task 10 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@54483c74 20/08/08 15:33:52.676 Executor task launch worker for task 9 DEBUG TaskMemoryManager: Task 9 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@64802f21 20/08/08 15:33:52.676 Executor task launch worker for task 10 INFO MemoryStore: Block rdd_19_1 stored as values in memory (estimated size 192.0 B, free 366.2 MiB) 20/08/08 15:33:52.676 Executor task launch worker for task 9 INFO MemoryStore: Block rdd_19_0 stored as values in memory (estimated size 192.0 B, free 366.2 MiB) 20/08/08 15:33:52.677 Executor task launch worker for task 11 DEBUG ALS: Start sorting an uncompressed in-block of size 2. (sortId = 1321296193) 20/08/08 15:33:52.677 Executor task launch worker for task 11 DEBUG ALS: Sorting took 3.3781E-5 seconds. (sortId = 1321296193) 20/08/08 15:33:52.678 Executor task launch worker for task 11 DEBUG TaskMemoryManager: Task 11 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@a80d6b8 20/08/08 15:33:52.678 Executor task launch worker for task 11 INFO MemoryStore: Block rdd_19_2 stored as values in memory (estimated size 192.0 B, free 366.2 MiB) 20/08/08 15:33:52.652 Executor task launch worker for task 5 DEBUG BlockManager: Getting local block broadcast_2 20/08/08 15:33:52.679 Executor task launch worker for task 5 DEBUG BlockManager: Level for block broadcast_2 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:52.652 dispatcher-event-loop-2 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_1.0, runningTasks: 3 20/08/08 15:33:52.679 dispatcher-event-loop-2 DEBUG TaskSetManager: No tasks for locality level NO_PREF, so moving to locality level ANY 20/08/08 15:33:52.680 task-result-getter-0 INFO TaskSetManager: Finished task 0.0 in stage 1.0 (TID 4) in 142 ms on 192.168.1.225 (executor driver) (1/4) 20/08/08 15:33:52.681 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_19_1 for BlockManagerId(driver, 192.168.1.225, 38159, None) 20/08/08 15:33:52.682 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_19_1 in memory on 192.168.1.225:38159 (size: 192.0 B, free: 366.3 MiB) 20/08/08 15:33:52.682 Executor task launch worker for task 10 DEBUG BlockManagerMaster: Updated info of block rdd_19_1 20/08/08 15:33:52.682 Executor task launch worker for task 10 DEBUG BlockManager: Told master about block rdd_19_1 20/08/08 15:33:52.682 Executor task launch worker for task 10 DEBUG BlockManager: Put block rdd_19_1 locally took 68 ms 20/08/08 15:33:52.682 Executor task launch worker for task 10 DEBUG BlockManager: Putting block rdd_19_1 without replication took 69 ms 20/08/08 15:33:52.682 Executor task launch worker for task 10 DEBUG BlockManager: Getting local block rdd_19_1 20/08/08 15:33:52.682 Executor task launch worker for task 10 DEBUG BlockManager: Level for block rdd_19_1 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:52.677 task-result-getter-2 INFO TaskSetManager: Finished task 3.0 in stage 3.0 (TID 12) in 89 ms on 192.168.1.225 (executor driver) (1/10) 20/08/08 15:33:52.682 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_19_0 for BlockManagerId(driver, 192.168.1.225, 38159, None) 20/08/08 15:33:52.683 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_19_0 in memory on 192.168.1.225:38159 (size: 192.0 B, free: 366.3 MiB) 20/08/08 15:33:52.683 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_19_2 for BlockManagerId(driver, 192.168.1.225, 38159, None) 20/08/08 15:33:52.683 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_19_2 in memory on 192.168.1.225:38159 (size: 192.0 B, free: 366.3 MiB) 20/08/08 15:33:52.684 Executor task launch worker for task 9 DEBUG BlockManagerMaster: Updated info of block rdd_19_0 20/08/08 15:33:52.684 Executor task launch worker for task 9 DEBUG BlockManager: Told master about block rdd_19_0 20/08/08 15:33:52.684 Executor task launch worker for task 9 DEBUG BlockManager: Put block rdd_19_0 locally took 70 ms 20/08/08 15:33:52.684 Executor task launch worker for task 9 DEBUG BlockManager: Putting block rdd_19_0 without replication took 70 ms 20/08/08 15:33:52.684 Executor task launch worker for task 9 DEBUG BlockManager: Getting local block rdd_19_0 20/08/08 15:33:52.684 Executor task launch worker for task 9 DEBUG BlockManager: Level for block rdd_19_0 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:52.684 Executor task launch worker for task 10 INFO MemoryStore: Block rdd_20_1 stored as values in memory (estimated size 296.0 B, free 366.2 MiB) 20/08/08 15:33:52.685 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_20_1 for BlockManagerId(driver, 192.168.1.225, 38159, None) 20/08/08 15:33:52.685 Executor task launch worker for task 11 DEBUG BlockManagerMaster: Updated info of block rdd_19_2 20/08/08 15:33:52.685 Executor task launch worker for task 11 DEBUG BlockManager: Told master about block rdd_19_2 20/08/08 15:33:52.685 Executor task launch worker for task 9 INFO MemoryStore: Block rdd_20_0 stored as values in memory (estimated size 296.0 B, free 366.2 MiB) 20/08/08 15:33:52.685 Executor task launch worker for task 11 DEBUG BlockManager: Put block rdd_19_2 locally took 75 ms 20/08/08 15:33:52.685 Executor task launch worker for task 11 DEBUG BlockManager: Putting block rdd_19_2 without replication took 76 ms 20/08/08 15:33:52.685 Executor task launch worker for task 11 DEBUG BlockManager: Getting local block rdd_19_2 20/08/08 15:33:52.685 Executor task launch worker for task 11 DEBUG BlockManager: Level for block rdd_19_2 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:52.686 Executor task launch worker for task 7 DEBUG GenerateSafeProjection: code for mapobjects(lambdavariable(MapObject, StringType, true, -1), lambdavariable(MapObject, StringType, true, -1).toString, input[0, array, true], Some(interface scala.collection.Seq)): /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificSafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificSafeProjection extends org.apache.spark.sql.catalyst.expressions.codegen.BaseProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private InternalRow mutableRow; /* 009 */ private UTF8String value_MapObject_lambda_variable_1; /* 010 */ private boolean isNull_MapObject_lambda_variable_1; /* 011 */ private boolean globalIsNull_0; /* 012 */ /* 013 */ public SpecificSafeProjection(Object[] references) { /* 014 */ this.references = references; /* 015 */ mutableRow = (InternalRow) references[references.length - 1]; /* 016 */ /* 017 */ /* 018 */ } /* 019 */ /* 020 */ public void initialize(int partitionIndex) { /* 021 */ /* 022 */ } /* 023 */ /* 024 */ public java.lang.Object apply(java.lang.Object _i) { /* 025 */ InternalRow i = (InternalRow) _i; /* 026 */ scala.collection.Seq value_3 = MapObjects_0(i); /* 027 */ if (globalIsNull_0) { /* 028 */ mutableRow.setNullAt(0); /* 029 */ } else { /* 030 */ /* 031 */ mutableRow.update(0, value_3); /* 032 */ } /* 033 */ /* 034 */ return mutableRow; /* 035 */ } /* 036 */ /* 037 */ /* 038 */ private scala.collection.Seq MapObjects_0(InternalRow i) { /* 039 */ boolean isNull_1 = i.isNullAt(0); /* 040 */ ArrayData value_1 = isNull_1 ? /* 041 */ null : (i.getArray(0)); /* 042 */ scala.collection.Seq value_0 = null; /* 043 */ /* 044 */ if (!isNull_1) { /* 045 */ /* 046 */ int dataLength_0 = value_1.numElements(); /* 047 */ /* 048 */ scala.collection.mutable.Builder collectionBuilder_0 = scala.collection.Seq$.MODULE$.newBuilder(); /* 049 */ collectionBuilder_0.sizeHint(dataLength_0); /* 050 */ /* 051 */ /* 052 */ int loopIndex_0 = 0; /* 053 */ /* 054 */ while (loopIndex_0 < dataLength_0) { /* 055 */ value_MapObject_lambda_variable_1 = (UTF8String) (value_1.getUTF8String(loopIndex_0)); /* 056 */ isNull_MapObject_lambda_variable_1 = value_1.isNullAt(loopIndex_0); /* 057 */ /* 058 */ boolean isNull_2 = true; /* 059 */ java.lang.String value_2 = null; /* 060 */ if (!isNull_MapObject_lambda_variable_1) { /* 061 */ /* 062 */ isNull_2 = false; /* 063 */ if (!isNull_2) { /* 064 */ /* 065 */ Object funcResult_0 = null; /* 066 */ funcResult_0 = value_MapObject_lambda_variable_1.toString(); /* 067 */ value_2 = (java.lang.String) funcResult_0; /* 068 */ /* 069 */ } /* 070 */ } /* 071 */ if (isNull_2) { /* 072 */ collectionBuilder_0.$plus$eq(null); /* 073 */ } else { /* 074 */ collectionBuilder_0.$plus$eq(value_2); /* 075 */ } /* 076 */ /* 077 */ loopIndex_0 += 1; /* 078 */ } /* 079 */ /* 080 */ value_0 = (scala.collection.Seq) collectionBuilder_0.result(); /* 081 */ } /* 082 */ globalIsNull_0 = isNull_1; /* 083 */ return value_0; /* 084 */ } /* 085 */ /* 086 */ } 20/08/08 15:33:52.686 Executor task launch worker for task 11 INFO MemoryStore: Block rdd_20_2 stored as values in memory (estimated size 296.0 B, free 366.2 MiB) 20/08/08 15:33:52.688 Executor task launch worker for task 13 DEBUG BlockManager: Getting local block rdd_20_4 20/08/08 15:33:52.688 Executor task launch worker for task 13 DEBUG BlockManager: Block rdd_20_4 was not found 20/08/08 15:33:52.688 Executor task launch worker for task 13 DEBUG BlockManager: Getting remote block rdd_20_4 20/08/08 15:33:52.692 Executor task launch worker for task 7 DEBUG GenerateUnsafeProjection: code for newInstance(class org.apache.spark.ml.linalg.VectorUDT).serialize: /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificUnsafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificUnsafeProjection extends org.apache.spark.sql.catalyst.expressions.UnsafeProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private boolean resultIsNull_0; /* 009 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[] mutableStateArray_1 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[2]; /* 010 */ private org.apache.spark.ml.linalg.Vector[] mutableStateArray_0 = new org.apache.spark.ml.linalg.Vector[1]; /* 011 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[] mutableStateArray_2 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[2]; /* 012 */ /* 013 */ public SpecificUnsafeProjection(Object[] references) { /* 014 */ this.references = references; /* 015 */ /* 016 */ mutableStateArray_1[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(1, 32); /* 017 */ mutableStateArray_1[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(mutableStateArray_1[0], 4); /* 018 */ mutableStateArray_2[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(mutableStateArray_1[1], 4); /* 019 */ mutableStateArray_2[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(mutableStateArray_1[1], 8); /* 020 */ /* 021 */ } /* 022 */ /* 023 */ public void initialize(int partitionIndex) { /* 024 */ /* 025 */ } /* 026 */ /* 027 */ // Scala.Function1 need this /* 028 */ public java.lang.Object apply(java.lang.Object row) { /* 029 */ return apply((InternalRow) row); /* 030 */ } /* 031 */ /* 032 */ public UnsafeRow apply(InternalRow i) { /* 033 */ mutableStateArray_1[0].reset(); /* 034 */ /* 035 */ /* 036 */ mutableStateArray_1[0].zeroOutNullBytes(); /* 037 */ /* 038 */ final org.apache.spark.ml.linalg.VectorUDT value_1 = false ? /* 039 */ null : new org.apache.spark.ml.linalg.VectorUDT(); /* 040 */ boolean isNull_0 = true; /* 041 */ InternalRow value_0 = null; /* 042 */ if (!false) { /* 043 */ resultIsNull_0 = false; /* 044 */ if (!resultIsNull_0) { /* 045 */ boolean isNull_2 = i.isNullAt(0); /* 046 */ org.apache.spark.ml.linalg.Vector value_2 = isNull_2 ? /* 047 */ null : ((org.apache.spark.ml.linalg.Vector)i.get(0, null)); /* 048 */ resultIsNull_0 = isNull_2; /* 049 */ mutableStateArray_0[0] = value_2; /* 050 */ } /* 051 */ /* 052 */ isNull_0 = resultIsNull_0; /* 053 */ if (!isNull_0) { /* 054 */ /* 055 */ Object funcResult_0 = null; /* 056 */ funcResult_0 = value_1.serialize(mutableStateArray_0[0]); /* 057 */ /* 058 */ if (funcResult_0 != null) { /* 059 */ value_0 = (InternalRow) funcResult_0; /* 060 */ } else { /* 061 */ isNull_0 = true; /* 062 */ } /* 063 */ /* 064 */ /* 065 */ } /* 066 */ } /* 067 */ if (isNull_0) { /* 068 */ mutableStateArray_1[0].setNullAt(0); /* 069 */ } else { /* 070 */ final InternalRow tmpInput_0 = value_0; /* 071 */ if (tmpInput_0 instanceof UnsafeRow) { /* 072 */ mutableStateArray_1[0].write(0, (UnsafeRow) tmpInput_0); /* 073 */ } else { /* 074 */ // Remember the current cursor so that we can calculate how many bytes are /* 075 */ // written later. /* 076 */ final int previousCursor_0 = mutableStateArray_1[0].cursor(); /* 077 */ /* 078 */ mutableStateArray_1[1].resetRowWriter(); /* 079 */ /* 080 */ /* 081 */ mutableStateArray_1[1].write(0, (tmpInput_0.getByte(0))); /* 082 */ /* 083 */ /* 084 */ if ((tmpInput_0.isNullAt(1))) { /* 085 */ mutableStateArray_1[1].setNullAt(1); /* 086 */ } else { /* 087 */ mutableStateArray_1[1].write(1, (tmpInput_0.getInt(1))); /* 088 */ } /* 089 */ /* 090 */ /* 091 */ if ((tmpInput_0.isNullAt(2))) { /* 092 */ mutableStateArray_1[1].setNullAt(2); /* 093 */ } else { /* 094 */ // Remember the current cursor so that we can calculate how many bytes are /* 095 */ // written later. /* 096 */ final int previousCursor_1 = mutableStateArray_1[1].cursor(); /* 097 */ /* 098 */ final ArrayData tmpInput_1 = (tmpInput_0.getArray(2)); /* 099 */ if (tmpInput_1 instanceof UnsafeArrayData) { /* 100 */ mutableStateArray_1[1].write((UnsafeArrayData) tmpInput_1); /* 101 */ } else { /* 102 */ final int numElements_0 = tmpInput_1.numElements(); /* 103 */ mutableStateArray_2[0].initialize(numElements_0); /* 104 */ /* 105 */ for (int index_0 = 0; index_0 < numElements_0; index_0++) { /* 106 */ mutableStateArray_2[0].write(index_0, tmpInput_1.getInt(index_0)); /* 107 */ } /* 108 */ } /* 109 */ /* 110 */ mutableStateArray_1[1].setOffsetAndSizeFromPreviousCursor(2, previousCursor_1); /* 111 */ } /* 112 */ /* 113 */ /* 114 */ if ((tmpInput_0.isNullAt(3))) { /* 115 */ mutableStateArray_1[1].setNullAt(3); /* 116 */ } else { /* 117 */ // Remember the current cursor so that we can calculate how many bytes are /* 118 */ // written later. /* 119 */ final int previousCursor_2 = mutableStateArray_1[1].cursor(); /* 120 */ /* 121 */ final ArrayData tmpInput_2 = (tmpInput_0.getArray(3)); /* 122 */ if (tmpInput_2 instanceof UnsafeArrayData) { /* 123 */ mutableStateArray_1[1].write((UnsafeArrayData) tmpInput_2); /* 124 */ } else { /* 125 */ final int numElements_1 = tmpInput_2.numElements(); /* 126 */ mutableStateArray_2[1].initialize(numElements_1); /* 127 */ /* 128 */ for (int index_1 = 0; index_1 < numElements_1; index_1++) { /* 129 */ mutableStateArray_2[1].write(index_1, tmpInput_2.getDouble(index_1)); /* 130 */ } /* 131 */ } /* 132 */ /* 133 */ mutableStateArray_1[1].setOffsetAndSizeFromPreviousCursor(3, previousCursor_2); /* 134 */ } /* 135 */ /* 136 */ /* 137 */ mutableStateArray_1[0].setOffsetAndSizeFromPreviousCursor(0, previousCursor_0); /* 138 */ } /* 139 */ } /* 140 */ return (mutableStateArray_1[0].getRow()); /* 141 */ } /* 142 */ /* 143 */ /* 144 */ } 20/08/08 15:33:52.693 Executor task launch worker for task 7 INFO PythonRunner: Times: total = 105, boot = -724, init = 829, finish = 0 20/08/08 15:33:52.685 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_20_1 in memory on 192.168.1.225:38159 (size: 296.0 B, free: 366.3 MiB) 20/08/08 15:33:52.693 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_20_0 for BlockManagerId(driver, 192.168.1.225, 38159, None) 20/08/08 15:33:52.694 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_20_0 in memory on 192.168.1.225:38159 (size: 296.0 B, free: 366.3 MiB) 20/08/08 15:33:52.694 Executor task launch worker for task 10 DEBUG BlockManagerMaster: Updated info of block rdd_20_1 20/08/08 15:33:52.694 Executor task launch worker for task 9 DEBUG BlockManagerMaster: Updated info of block rdd_20_0 20/08/08 15:33:52.694 Executor task launch worker for task 9 DEBUG BlockManager: Told master about block rdd_20_0 20/08/08 15:33:52.694 Executor task launch worker for task 9 DEBUG BlockManager: Put block rdd_20_0 locally took 81 ms 20/08/08 15:33:52.694 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_20_2 for BlockManagerId(driver, 192.168.1.225, 38159, None) 20/08/08 15:33:52.694 Executor task launch worker for task 10 DEBUG BlockManager: Told master about block rdd_20_1 20/08/08 15:33:52.694 Executor task launch worker for task 9 DEBUG BlockManager: Putting block rdd_20_0 without replication took 81 ms 20/08/08 15:33:52.694 Executor task launch worker for task 10 DEBUG BlockManager: Put block rdd_20_1 locally took 88 ms 20/08/08 15:33:52.694 Executor task launch worker for task 9 DEBUG BlockManager: Getting local block rdd_20_0 20/08/08 15:33:52.694 Executor task launch worker for task 10 DEBUG BlockManager: Putting block rdd_20_1 without replication took 88 ms 20/08/08 15:33:52.694 Executor task launch worker for task 10 DEBUG BlockManager: Getting local block rdd_20_1 20/08/08 15:33:52.694 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_20_2 in memory on 192.168.1.225:38159 (size: 296.0 B, free: 366.3 MiB) 20/08/08 15:33:52.695 Executor task launch worker for task 10 DEBUG BlockManager: Level for block rdd_20_1 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:52.694 Executor task launch worker for task 9 DEBUG BlockManager: Level for block rdd_20_0 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:52.695 Executor task launch worker for task 13 DEBUG BlockManager: Block rdd_20_4 is unknown by block manager master 20/08/08 15:33:52.695 Executor task launch worker for task 13 DEBUG BlockManager: Getting local block rdd_19_4 20/08/08 15:33:52.695 Executor task launch worker for task 13 DEBUG BlockManager: Block rdd_19_4 was not found 20/08/08 15:33:52.695 Executor task launch worker for task 13 DEBUG BlockManager: Getting remote block rdd_19_4 20/08/08 15:33:52.695 Executor task launch worker for task 11 DEBUG BlockManagerMaster: Updated info of block rdd_20_2 20/08/08 15:33:52.695 Executor task launch worker for task 11 DEBUG BlockManager: Told master about block rdd_20_2 20/08/08 15:33:52.695 Executor task launch worker for task 11 DEBUG BlockManager: Put block rdd_20_2 locally took 87 ms 20/08/08 15:33:52.696 Executor task launch worker for task 11 DEBUG BlockManager: Putting block rdd_20_2 without replication took 87 ms 20/08/08 15:33:52.696 Executor task launch worker for task 13 DEBUG BlockManager: Block rdd_19_4 is unknown by block manager master 20/08/08 15:33:52.696 Executor task launch worker for task 11 DEBUG BlockManager: Getting local block rdd_20_2 20/08/08 15:33:52.696 Executor task launch worker for task 13 DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 0 20/08/08 15:33:52.696 Executor task launch worker for task 11 DEBUG BlockManager: Level for block rdd_20_2 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:52.696 Executor task launch worker for task 13 DEBUG MapOutputTrackerMaster: Convert map statuses for shuffle 0, mappers 0-4, partitions 4-5 20/08/08 15:33:52.696 Executor task launch worker for task 9 INFO Executor: Finished task 0.0 in stage 3.0 (TID 9). 2170 bytes result sent to driver 20/08/08 15:33:52.696 Executor task launch worker for task 9 DEBUG ExecutorMetricsPoller: stageTCMP: (3, 0) -> 3 20/08/08 15:33:52.696 Executor task launch worker for task 13 DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 20/08/08 15:33:52.697 dispatcher-event-loop-3 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_3.0, runningTasks: 3 20/08/08 15:33:52.697 Executor task launch worker for task 13 INFO ShuffleBlockFetcherIterator: Getting 0 (0.0 B) non-empty blocks including 0 (0.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 20/08/08 15:33:52.697 Executor task launch worker for task 13 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 20/08/08 15:33:52.697 Executor task launch worker for task 13 DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: 20/08/08 15:33:52.697 Executor task launch worker for task 13 DEBUG ShuffleBlockFetcherIterator: Got local blocks in 0 ms 20/08/08 15:33:52.697 Executor task launch worker for task 11 INFO Executor: Finished task 2.0 in stage 3.0 (TID 11). 2170 bytes result sent to driver 20/08/08 15:33:52.697 Executor task launch worker for task 11 DEBUG ExecutorMetricsPoller: stageTCMP: (3, 0) -> 2 20/08/08 15:33:52.698 Executor task launch worker for task 13 DEBUG TaskMemoryManager: Task 13 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@e004cb8 20/08/08 15:33:52.697 Executor task launch worker for task 5 DEBUG GenerateUnsafeProjection: code for newInstance(class org.apache.spark.ml.linalg.VectorUDT).serialize: /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificUnsafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificUnsafeProjection extends org.apache.spark.sql.catalyst.expressions.UnsafeProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private boolean resultIsNull_0; /* 009 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[] mutableStateArray_1 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[2]; /* 010 */ private org.apache.spark.ml.linalg.Vector[] mutableStateArray_0 = new org.apache.spark.ml.linalg.Vector[1]; /* 011 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[] mutableStateArray_2 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[2]; /* 012 */ /* 013 */ public SpecificUnsafeProjection(Object[] references) { /* 014 */ this.references = references; /* 015 */ /* 016 */ mutableStateArray_1[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(1, 32); /* 017 */ mutableStateArray_1[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(mutableStateArray_1[0], 4); /* 018 */ mutableStateArray_2[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(mutableStateArray_1[1], 4); /* 019 */ mutableStateArray_2[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(mutableStateArray_1[1], 8); /* 020 */ /* 021 */ } /* 022 */ /* 023 */ public void initialize(int partitionIndex) { /* 024 */ /* 025 */ } /* 026 */ /* 027 */ // Scala.Function1 need this /* 028 */ public java.lang.Object apply(java.lang.Object row) { /* 029 */ return apply((InternalRow) row); /* 030 */ } /* 031 */ /* 032 */ public UnsafeRow apply(InternalRow i) { /* 033 */ mutableStateArray_1[0].reset(); /* 034 */ /* 035 */ /* 036 */ mutableStateArray_1[0].zeroOutNullBytes(); /* 037 */ /* 038 */ final org.apache.spark.ml.linalg.VectorUDT value_1 = false ? /* 039 */ null : new org.apache.spark.ml.linalg.VectorUDT(); /* 040 */ boolean isNull_0 = true; /* 041 */ InternalRow value_0 = null; /* 042 */ if (!false) { /* 043 */ resultIsNull_0 = false; /* 044 */ if (!resultIsNull_0) { /* 045 */ boolean isNull_2 = i.isNullAt(0); /* 046 */ org.apache.spark.ml.linalg.Vector value_2 = isNull_2 ? /* 047 */ null : ((org.apache.spark.ml.linalg.Vector)i.get(0, null)); /* 048 */ resultIsNull_0 = isNull_2; /* 049 */ mutableStateArray_0[0] = value_2; /* 050 */ } /* 051 */ /* 052 */ isNull_0 = resultIsNull_0; /* 053 */ if (!isNull_0) { /* 054 */ /* 055 */ Object funcResult_0 = null; /* 056 */ funcResult_0 = value_1.serialize(mutableStateArray_0[0]); /* 057 */ /* 058 */ if (funcResult_0 != null) { /* 059 */ value_0 = (InternalRow) funcResult_0; /* 060 */ } else { /* 061 */ isNull_0 = true; /* 062 */ } /* 063 */ /* 064 */ /* 065 */ } /* 066 */ } /* 067 */ if (isNull_0) { /* 068 */ mutableStateArray_1[0].setNullAt(0); /* 069 */ } else { /* 070 */ final InternalRow tmpInput_0 = value_0; /* 071 */ if (tmpInput_0 instanceof UnsafeRow) { /* 072 */ mutableStateArray_1[0].write(0, (UnsafeRow) tmpInput_0); /* 073 */ } else { /* 074 */ // Remember the current cursor so that we can calculate how many bytes are /* 075 */ // written later. /* 076 */ final int previousCursor_0 = mutableStateArray_1[0].cursor(); /* 077 */ /* 078 */ mutableStateArray_1[1].resetRowWriter(); /* 079 */ /* 080 */ /* 081 */ mutableStateArray_1[1].write(0, (tmpInput_0.getByte(0))); /* 082 */ /* 083 */ /* 084 */ if ((tmpInput_0.isNullAt(1))) { /* 085 */ mutableStateArray_1[1].setNullAt(1); /* 086 */ } else { /* 087 */ mutableStateArray_1[1].write(1, (tmpInput_0.getInt(1))); /* 088 */ } /* 089 */ /* 090 */ /* 091 */ if ((tmpInput_0.isNullAt(2))) { /* 092 */ mutableStateArray_1[1].setNullAt(2); /* 093 */ } else { /* 094 */ // Remember the current cursor so that we can calculate how many bytes are /* 095 */ // written later. /* 096 */ final int previousCursor_1 = mutableStateArray_1[1].cursor(); /* 097 */ /* 098 */ final ArrayData tmpInput_1 = (tmpInput_0.getArray(2)); /* 099 */ if (tmpInput_1 instanceof UnsafeArrayData) { /* 100 */ mutableStateArray_1[1].write((UnsafeArrayData) tmpInput_1); /* 101 */ } else { /* 102 */ final int numElements_0 = tmpInput_1.numElements(); /* 103 */ mutableStateArray_2[0].initialize(numElements_0); /* 104 */ /* 105 */ for (int index_0 = 0; index_0 < numElements_0; index_0++) { /* 106 */ mutableStateArray_2[0].write(index_0, tmpInput_1.getInt(index_0)); /* 107 */ } /* 108 */ } /* 109 */ /* 110 */ mutableStateArray_1[1].setOffsetAndSizeFromPreviousCursor(2, previousCursor_1); /* 111 */ } /* 112 */ /* 113 */ /* 114 */ if ((tmpInput_0.isNullAt(3))) { /* 115 */ mutableStateArray_1[1].setNullAt(3); /* 116 */ } else { /* 117 */ // Remember the current cursor so that we can calculate how many bytes are /* 118 */ // written later. /* 119 */ final int previousCursor_2 = mutableStateArray_1[1].cursor(); /* 120 */ /* 121 */ final ArrayData tmpInput_2 = (tmpInput_0.getArray(3)); /* 122 */ if (tmpInput_2 instanceof UnsafeArrayData) { /* 123 */ mutableStateArray_1[1].write((UnsafeArrayData) tmpInput_2); /* 124 */ } else { /* 125 */ final int numElements_1 = tmpInput_2.numElements(); /* 126 */ mutableStateArray_2[1].initialize(numElements_1); /* 127 */ /* 128 */ for (int index_1 = 0; index_1 < numElements_1; index_1++) { /* 129 */ mutableStateArray_2[1].write(index_1, tmpInput_2.getDouble(index_1)); /* 130 */ } /* 131 */ } /* 132 */ /* 133 */ mutableStateArray_1[1].setOffsetAndSizeFromPreviousCursor(3, previousCursor_2); /* 134 */ } /* 135 */ /* 136 */ /* 137 */ mutableStateArray_1[0].setOffsetAndSizeFromPreviousCursor(0, previousCursor_0); /* 138 */ } /* 139 */ } /* 140 */ return (mutableStateArray_1[0].getRow()); /* 141 */ } /* 142 */ /* 143 */ /* 144 */ } 20/08/08 15:33:52.698 Executor task launch worker for task 13 INFO MemoryStore: Block rdd_19_4 stored as values in memory (estimated size 16.0 B, free 366.2 MiB) 20/08/08 15:33:52.698 Executor task launch worker for task 5 INFO PythonRunner: Times: total = 54, boot = -752, init = 806, finish = 0 20/08/08 15:33:52.699 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_19_4 for BlockManagerId(driver, 192.168.1.225, 38159, None) 20/08/08 15:33:52.699 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_19_4 in memory on 192.168.1.225:38159 (size: 16.0 B, free: 366.3 MiB) 20/08/08 15:33:52.699 Executor task launch worker for task 13 DEBUG BlockManagerMaster: Updated info of block rdd_19_4 20/08/08 15:33:52.699 Executor task launch worker for task 13 DEBUG BlockManager: Told master about block rdd_19_4 20/08/08 15:33:52.699 Executor task launch worker for task 13 DEBUG BlockManager: Put block rdd_19_4 locally took 3 ms 20/08/08 15:33:52.699 Executor task launch worker for task 13 DEBUG BlockManager: Putting block rdd_19_4 without replication took 3 ms 20/08/08 15:33:52.699 Executor task launch worker for task 13 DEBUG BlockManager: Getting local block rdd_19_4 20/08/08 15:33:52.699 Executor task launch worker for task 13 DEBUG BlockManager: Level for block rdd_19_4 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:52.700 Executor task launch worker for task 13 INFO MemoryStore: Block rdd_20_4 stored as values in memory (estimated size 16.0 B, free 366.2 MiB) 20/08/08 15:33:52.701 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_20_4 for BlockManagerId(driver, 192.168.1.225, 38159, None) 20/08/08 15:33:52.701 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_20_4 in memory on 192.168.1.225:38159 (size: 16.0 B, free: 366.3 MiB) 20/08/08 15:33:52.701 Executor task launch worker for task 13 DEBUG BlockManagerMaster: Updated info of block rdd_20_4 20/08/08 15:33:52.701 Executor task launch worker for task 13 DEBUG BlockManager: Told master about block rdd_20_4 20/08/08 15:33:52.701 Executor task launch worker for task 13 DEBUG BlockManager: Put block rdd_20_4 locally took 6 ms 20/08/08 15:33:52.702 Executor task launch worker for task 13 DEBUG BlockManager: Putting block rdd_20_4 without replication took 6 ms 20/08/08 15:33:52.702 Executor task launch worker for task 13 DEBUG BlockManager: Getting local block rdd_20_4 20/08/08 15:33:52.702 Executor task launch worker for task 13 DEBUG BlockManager: Level for block rdd_20_4 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:52.697 dispatcher-event-loop-3 DEBUG TaskSetManager: No tasks for locality level NODE_LOCAL, so moving to locality level NO_PREF 20/08/08 15:33:52.703 dispatcher-event-loop-3 DEBUG TaskSetManager: Moving to ANY after waiting for 0ms 20/08/08 15:33:52.703 dispatcher-event-loop-3 INFO TaskSetManager: Starting task 5.0 in stage 3.0 (TID 14, 192.168.1.225, executor driver, partition 5, PROCESS_LOCAL, 7143 bytes) taskResourceAssignments Map() 20/08/08 15:33:52.704 Executor task launch worker for task 14 INFO Executor: Running task 5.0 in stage 3.0 (TID 14) 20/08/08 15:33:52.704 task-result-getter-0 INFO TaskSetManager: Finished task 0.0 in stage 3.0 (TID 9) in 118 ms on 192.168.1.225 (executor driver) (2/10) 20/08/08 15:33:52.704 Executor task launch worker for task 13 INFO Executor: Finished task 4.0 in stage 3.0 (TID 13). 2170 bytes result sent to driver 20/08/08 15:33:52.704 Executor task launch worker for task 13 DEBUG ExecutorMetricsPoller: stageTCMP: (3, 0) -> 1 20/08/08 15:33:52.704 Executor task launch worker for task 14 DEBUG ExecutorMetricsPoller: stageTCMP: (3, 0) -> 2 20/08/08 15:33:52.704 task-result-getter-3 INFO TaskSetManager: Finished task 2.0 in stage 3.0 (TID 11) in 117 ms on 192.168.1.225 (executor driver) (3/10) 20/08/08 15:33:52.705 dispatcher-event-loop-3 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_3.0, runningTasks: 3 20/08/08 15:33:52.705 dispatcher-event-loop-3 DEBUG TaskSetManager: No tasks for locality level NODE_LOCAL, so moving to locality level NO_PREF 20/08/08 15:33:52.705 dispatcher-event-loop-3 DEBUG TaskSetManager: Moving to ANY after waiting for 0ms 20/08/08 15:33:52.706 dispatcher-event-loop-3 INFO TaskSetManager: Starting task 6.0 in stage 3.0 (TID 15, 192.168.1.225, executor driver, partition 6, PROCESS_LOCAL, 7143 bytes) taskResourceAssignments Map() 20/08/08 15:33:52.706 dispatcher-event-loop-3 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_3.0, runningTasks: 3 20/08/08 15:33:52.707 dispatcher-event-loop-3 DEBUG TaskSetManager: No tasks for locality level NODE_LOCAL, so moving to locality level NO_PREF 20/08/08 15:33:52.707 dispatcher-event-loop-3 DEBUG TaskSetManager: Moving to ANY after waiting for 0ms 20/08/08 15:33:52.707 dispatcher-event-loop-3 INFO TaskSetManager: Starting task 7.0 in stage 3.0 (TID 16, 192.168.1.225, executor driver, partition 7, PROCESS_LOCAL, 7143 bytes) taskResourceAssignments Map() 20/08/08 15:33:52.708 Executor task launch worker for task 16 INFO Executor: Running task 7.0 in stage 3.0 (TID 16) 20/08/08 15:33:52.708 Executor task launch worker for task 16 DEBUG ExecutorMetricsPoller: stageTCMP: (3, 0) -> 3 20/08/08 15:33:52.708 Executor task launch worker for task 15 INFO Executor: Running task 6.0 in stage 3.0 (TID 15) 20/08/08 15:33:52.708 task-result-getter-1 INFO TaskSetManager: Finished task 4.0 in stage 3.0 (TID 13) in 71 ms on 192.168.1.225 (executor driver) (4/10) 20/08/08 15:33:52.709 Executor task launch worker for task 15 DEBUG ExecutorMetricsPoller: stageTCMP: (3, 0) -> 4 20/08/08 15:33:52.713 Executor task launch worker for task 14 DEBUG BlockManager: Getting local block rdd_20_5 20/08/08 15:33:52.713 Executor task launch worker for task 14 DEBUG BlockManager: Block rdd_20_5 was not found 20/08/08 15:33:52.713 Executor task launch worker for task 14 DEBUG BlockManager: Getting remote block rdd_20_5 20/08/08 15:33:52.713 Executor task launch worker for task 14 DEBUG BlockManager: Block rdd_20_5 is unknown by block manager master 20/08/08 15:33:52.714 Executor task launch worker for task 14 DEBUG BlockManager: Getting local block rdd_19_5 20/08/08 15:33:52.714 Executor task launch worker for task 14 DEBUG BlockManager: Block rdd_19_5 was not found 20/08/08 15:33:52.714 Executor task launch worker for task 14 DEBUG BlockManager: Getting remote block rdd_19_5 20/08/08 15:33:52.714 Executor task launch worker for task 14 DEBUG BlockManager: Block rdd_19_5 is unknown by block manager master 20/08/08 15:33:52.714 Executor task launch worker for task 14 DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 0 20/08/08 15:33:52.714 Executor task launch worker for task 14 DEBUG MapOutputTrackerMaster: Convert map statuses for shuffle 0, mappers 0-4, partitions 5-6 20/08/08 15:33:52.714 Executor task launch worker for task 14 DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 20/08/08 15:33:52.715 Executor task launch worker for task 14 INFO ShuffleBlockFetcherIterator: Getting 0 (0.0 B) non-empty blocks including 0 (0.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 20/08/08 15:33:52.715 Executor task launch worker for task 14 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 20/08/08 15:33:52.715 Executor task launch worker for task 14 DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: 20/08/08 15:33:52.715 Executor task launch worker for task 14 DEBUG ShuffleBlockFetcherIterator: Got local blocks in 0 ms 20/08/08 15:33:52.716 Executor task launch worker for task 14 DEBUG TaskMemoryManager: Task 14 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@6510d4b 20/08/08 15:33:52.716 Executor task launch worker for task 14 INFO MemoryStore: Block rdd_19_5 stored as values in memory (estimated size 16.0 B, free 366.2 MiB) 20/08/08 15:33:52.716 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_19_5 for BlockManagerId(driver, 192.168.1.225, 38159, None) 20/08/08 15:33:52.717 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_19_5 in memory on 192.168.1.225:38159 (size: 16.0 B, free: 366.3 MiB) 20/08/08 15:33:52.699 Executor task launch worker for task 7 INFO Executor: Finished task 3.0 in stage 1.0 (TID 7). 2063 bytes result sent to driver 20/08/08 15:33:52.708 Executor task launch worker for task 5 INFO Executor: Finished task 1.0 in stage 1.0 (TID 5). 2062 bytes result sent to driver 20/08/08 15:33:52.718 dispatcher-event-loop-3 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_1.0, runningTasks: 2 20/08/08 15:33:52.718 Executor task launch worker for task 10 INFO Executor: Finished task 1.0 in stage 3.0 (TID 10). 2170 bytes result sent to driver 20/08/08 15:33:52.718 Executor task launch worker for task 10 DEBUG ExecutorMetricsPoller: stageTCMP: (3, 0) -> 3 20/08/08 15:33:52.718 dispatcher-event-loop-3 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_1.0, runningTasks: 1 20/08/08 15:33:52.718 dispatcher-event-loop-2 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_3.0, runningTasks: 3 20/08/08 15:33:52.719 task-result-getter-1 INFO TaskSetManager: Finished task 3.0 in stage 1.0 (TID 7) in 179 ms on 192.168.1.225 (executor driver) (2/4) 20/08/08 15:33:52.720 dispatcher-event-loop-2 DEBUG TaskSetManager: No tasks for locality level NODE_LOCAL, so moving to locality level NO_PREF 20/08/08 15:33:52.720 dispatcher-event-loop-2 DEBUG TaskSetManager: Moving to ANY after waiting for 0ms 20/08/08 15:33:52.720 dispatcher-event-loop-2 INFO TaskSetManager: Starting task 8.0 in stage 3.0 (TID 17, 192.168.1.225, executor driver, partition 8, PROCESS_LOCAL, 7143 bytes) taskResourceAssignments Map() 20/08/08 15:33:52.721 task-result-getter-2 INFO TaskSetManager: Finished task 1.0 in stage 3.0 (TID 10) in 134 ms on 192.168.1.225 (executor driver) (5/10) 20/08/08 15:33:52.721 task-result-getter-2 INFO TaskSetManager: Finished task 1.0 in stage 1.0 (TID 5) in 182 ms on 192.168.1.225 (executor driver) (3/4) 20/08/08 15:33:52.721 Executor task launch worker for task 17 INFO Executor: Running task 8.0 in stage 3.0 (TID 17) 20/08/08 15:33:52.718 Executor task launch worker for task 14 DEBUG BlockManagerMaster: Updated info of block rdd_19_5 20/08/08 15:33:52.722 Executor task launch worker for task 17 DEBUG ExecutorMetricsPoller: stageTCMP: (3, 0) -> 4 20/08/08 15:33:52.722 Executor task launch worker for task 14 DEBUG BlockManager: Told master about block rdd_19_5 20/08/08 15:33:52.722 Executor task launch worker for task 14 DEBUG BlockManager: Put block rdd_19_5 locally took 7 ms 20/08/08 15:33:52.722 Executor task launch worker for task 14 DEBUG BlockManager: Putting block rdd_19_5 without replication took 7 ms 20/08/08 15:33:52.722 Executor task launch worker for task 14 DEBUG BlockManager: Getting local block rdd_19_5 20/08/08 15:33:52.722 Executor task launch worker for task 14 DEBUG BlockManager: Level for block rdd_19_5 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:52.723 Executor task launch worker for task 14 INFO MemoryStore: Block rdd_20_5 stored as values in memory (estimated size 16.0 B, free 366.2 MiB) 20/08/08 15:33:52.723 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_20_5 for BlockManagerId(driver, 192.168.1.225, 38159, None) 20/08/08 15:33:52.724 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_20_5 in memory on 192.168.1.225:38159 (size: 16.0 B, free: 366.3 MiB) 20/08/08 15:33:52.724 Executor task launch worker for task 14 DEBUG BlockManagerMaster: Updated info of block rdd_20_5 20/08/08 15:33:52.724 Executor task launch worker for task 14 DEBUG BlockManager: Told master about block rdd_20_5 20/08/08 15:33:52.724 Executor task launch worker for task 14 DEBUG BlockManager: Put block rdd_20_5 locally took 10 ms 20/08/08 15:33:52.724 Executor task launch worker for task 14 DEBUG BlockManager: Putting block rdd_20_5 without replication took 10 ms 20/08/08 15:33:52.724 Executor task launch worker for task 14 DEBUG BlockManager: Getting local block rdd_20_5 20/08/08 15:33:52.724 Executor task launch worker for task 14 DEBUG BlockManager: Level for block rdd_20_5 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:52.725 Executor task launch worker for task 16 DEBUG BlockManager: Getting local block rdd_20_7 20/08/08 15:33:52.725 Executor task launch worker for task 16 DEBUG BlockManager: Block rdd_20_7 was not found 20/08/08 15:33:52.725 Executor task launch worker for task 16 DEBUG BlockManager: Getting remote block rdd_20_7 20/08/08 15:33:52.729 Executor task launch worker for task 14 INFO Executor: Finished task 5.0 in stage 3.0 (TID 14). 2170 bytes result sent to driver 20/08/08 15:33:52.730 Executor task launch worker for task 16 DEBUG BlockManager: Block rdd_20_7 is unknown by block manager master 20/08/08 15:33:52.730 dispatcher-event-loop-1 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_3.0, runningTasks: 3 20/08/08 15:33:52.730 dispatcher-event-loop-1 DEBUG TaskSetManager: No tasks for locality level NODE_LOCAL, so moving to locality level NO_PREF 20/08/08 15:33:52.730 dispatcher-event-loop-1 DEBUG TaskSetManager: Moving to ANY after waiting for 0ms 20/08/08 15:33:52.730 dispatcher-event-loop-1 INFO TaskSetManager: Starting task 9.0 in stage 3.0 (TID 18, 192.168.1.225, executor driver, partition 9, PROCESS_LOCAL, 7143 bytes) taskResourceAssignments Map() 20/08/08 15:33:52.731 Executor task launch worker for task 18 INFO Executor: Running task 9.0 in stage 3.0 (TID 18) 20/08/08 15:33:52.731 task-result-getter-0 INFO TaskSetManager: Finished task 5.0 in stage 3.0 (TID 14) in 28 ms on 192.168.1.225 (executor driver) (6/10) 20/08/08 15:33:52.731 Executor task launch worker for task 16 DEBUG BlockManager: Getting local block rdd_19_7 20/08/08 15:33:52.731 Executor task launch worker for task 16 DEBUG BlockManager: Block rdd_19_7 was not found 20/08/08 15:33:52.731 Executor task launch worker for task 16 DEBUG BlockManager: Getting remote block rdd_19_7 20/08/08 15:33:52.731 Executor task launch worker for task 14 DEBUG ExecutorMetricsPoller: stageTCMP: (3, 0) -> 3 20/08/08 15:33:52.731 Executor task launch worker for task 18 DEBUG ExecutorMetricsPoller: stageTCMP: (3, 0) -> 4 20/08/08 15:33:52.731 Executor task launch worker for task 17 DEBUG BlockManager: Getting local block rdd_20_8 20/08/08 15:33:52.732 Executor task launch worker for task 17 DEBUG BlockManager: Block rdd_20_8 was not found 20/08/08 15:33:52.732 Executor task launch worker for task 17 DEBUG BlockManager: Getting remote block rdd_20_8 20/08/08 15:33:52.726 Executor task launch worker for task 3 INFO CodeGenerator: Code generated in 120.100248 ms 20/08/08 15:33:52.717 Executor task launch worker for task 7 DEBUG ExecutorMetricsPoller: stageTCMP: (1, 0) -> 2 20/08/08 15:33:52.734 Executor task launch worker for task 5 DEBUG ExecutorMetricsPoller: stageTCMP: (1, 0) -> 1 20/08/08 15:33:52.739 Executor task launch worker for task 18 DEBUG BlockManager: Getting local block rdd_20_9 20/08/08 15:33:52.739 Executor task launch worker for task 18 DEBUG BlockManager: Block rdd_20_9 was not found 20/08/08 15:33:52.739 Executor task launch worker for task 18 DEBUG BlockManager: Getting remote block rdd_20_9 20/08/08 15:33:52.750 Executor task launch worker for task 18 DEBUG BlockManager: Block rdd_20_9 is unknown by block manager master 20/08/08 15:33:52.750 Executor task launch worker for task 17 DEBUG BlockManager: Block rdd_20_8 is unknown by block manager master 20/08/08 15:33:52.750 Executor task launch worker for task 18 DEBUG BlockManager: Getting local block rdd_19_9 20/08/08 15:33:52.750 Executor task launch worker for task 18 DEBUG BlockManager: Block rdd_19_9 was not found 20/08/08 15:33:52.750 Executor task launch worker for task 17 DEBUG BlockManager: Getting local block rdd_19_8 20/08/08 15:33:52.750 Executor task launch worker for task 18 DEBUG BlockManager: Getting remote block rdd_19_9 20/08/08 15:33:52.750 Executor task launch worker for task 17 DEBUG BlockManager: Block rdd_19_8 was not found 20/08/08 15:33:52.750 Executor task launch worker for task 17 DEBUG BlockManager: Getting remote block rdd_19_8 20/08/08 15:33:52.751 Executor task launch worker for task 17 DEBUG BlockManager: Block rdd_19_8 is unknown by block manager master 20/08/08 15:33:52.751 Executor task launch worker for task 18 DEBUG BlockManager: Block rdd_19_9 is unknown by block manager master 20/08/08 15:33:52.751 Executor task launch worker for task 17 DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 0 20/08/08 15:33:52.751 Executor task launch worker for task 18 DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 0 20/08/08 15:33:52.751 Executor task launch worker for task 17 DEBUG MapOutputTrackerMaster: Convert map statuses for shuffle 0, mappers 0-4, partitions 8-9 20/08/08 15:33:52.751 Executor task launch worker for task 18 DEBUG MapOutputTrackerMaster: Convert map statuses for shuffle 0, mappers 0-4, partitions 9-10 20/08/08 15:33:52.751 Executor task launch worker for task 18 DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 20/08/08 15:33:52.751 Executor task launch worker for task 17 DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 20/08/08 15:33:52.752 Executor task launch worker for task 18 INFO ShuffleBlockFetcherIterator: Getting 0 (0.0 B) non-empty blocks including 0 (0.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 20/08/08 15:33:52.752 Executor task launch worker for task 17 INFO ShuffleBlockFetcherIterator: Getting 0 (0.0 B) non-empty blocks including 0 (0.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 20/08/08 15:33:52.752 Executor task launch worker for task 18 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 20/08/08 15:33:52.752 Executor task launch worker for task 17 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 20/08/08 15:33:52.752 Executor task launch worker for task 18 DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: 20/08/08 15:33:52.752 Executor task launch worker for task 17 DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: 20/08/08 15:33:52.752 Executor task launch worker for task 18 DEBUG ShuffleBlockFetcherIterator: Got local blocks in 0 ms 20/08/08 15:33:52.752 Executor task launch worker for task 17 DEBUG ShuffleBlockFetcherIterator: Got local blocks in 0 ms 20/08/08 15:33:52.753 Executor task launch worker for task 18 DEBUG TaskMemoryManager: Task 18 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@23fd1f66 20/08/08 15:33:52.753 Executor task launch worker for task 17 DEBUG TaskMemoryManager: Task 17 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@7eb2e557 20/08/08 15:33:52.754 Executor task launch worker for task 18 INFO MemoryStore: Block rdd_19_9 stored as values in memory (estimated size 16.0 B, free 366.2 MiB) 20/08/08 15:33:52.754 Executor task launch worker for task 17 INFO MemoryStore: Block rdd_19_8 stored as values in memory (estimated size 16.0 B, free 366.2 MiB) 20/08/08 15:33:52.754 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_19_9 for BlockManagerId(driver, 192.168.1.225, 38159, None) 20/08/08 15:33:52.754 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_19_9 in memory on 192.168.1.225:38159 (size: 16.0 B, free: 366.3 MiB) 20/08/08 15:33:52.754 Executor task launch worker for task 18 DEBUG BlockManagerMaster: Updated info of block rdd_19_9 20/08/08 15:33:52.754 Executor task launch worker for task 18 DEBUG BlockManager: Told master about block rdd_19_9 20/08/08 15:33:52.755 Executor task launch worker for task 18 DEBUG BlockManager: Put block rdd_19_9 locally took 3 ms 20/08/08 15:33:52.755 Executor task launch worker for task 18 DEBUG BlockManager: Putting block rdd_19_9 without replication took 3 ms 20/08/08 15:33:52.755 Executor task launch worker for task 18 DEBUG BlockManager: Getting local block rdd_19_9 20/08/08 15:33:52.755 Executor task launch worker for task 18 DEBUG BlockManager: Level for block rdd_19_9 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:52.755 Executor task launch worker for task 18 INFO MemoryStore: Block rdd_20_9 stored as values in memory (estimated size 16.0 B, free 366.2 MiB) 20/08/08 15:33:52.756 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_20_9 for BlockManagerId(driver, 192.168.1.225, 38159, None) 20/08/08 15:33:52.756 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_20_9 in memory on 192.168.1.225:38159 (size: 16.0 B, free: 366.3 MiB) 20/08/08 15:33:52.756 Executor task launch worker for task 18 DEBUG BlockManagerMaster: Updated info of block rdd_20_9 20/08/08 15:33:52.756 Executor task launch worker for task 18 DEBUG BlockManager: Told master about block rdd_20_9 20/08/08 15:33:52.756 Executor task launch worker for task 18 DEBUG BlockManager: Put block rdd_20_9 locally took 6 ms 20/08/08 15:33:52.756 Executor task launch worker for task 18 DEBUG BlockManager: Putting block rdd_20_9 without replication took 6 ms 20/08/08 15:33:52.756 Executor task launch worker for task 18 DEBUG BlockManager: Getting local block rdd_20_9 20/08/08 15:33:52.756 Executor task launch worker for task 18 DEBUG BlockManager: Level for block rdd_20_9 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:52.757 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_19_8 for BlockManagerId(driver, 192.168.1.225, 38159, None) 20/08/08 15:33:52.757 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_19_8 in memory on 192.168.1.225:38159 (size: 16.0 B, free: 366.3 MiB) 20/08/08 15:33:52.757 Executor task launch worker for task 17 DEBUG BlockManagerMaster: Updated info of block rdd_19_8 20/08/08 15:33:52.758 Executor task launch worker for task 17 DEBUG BlockManager: Told master about block rdd_19_8 20/08/08 15:33:52.758 Executor task launch worker for task 17 DEBUG BlockManager: Put block rdd_19_8 locally took 6 ms 20/08/08 15:33:52.758 Executor task launch worker for task 17 DEBUG BlockManager: Putting block rdd_19_8 without replication took 7 ms 20/08/08 15:33:52.758 Executor task launch worker for task 17 DEBUG BlockManager: Getting local block rdd_19_8 20/08/08 15:33:52.758 Executor task launch worker for task 17 DEBUG BlockManager: Level for block rdd_19_8 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:52.758 Executor task launch worker for task 15 DEBUG BlockManager: Getting local block rdd_20_6 20/08/08 15:33:52.758 Executor task launch worker for task 6 DEBUG GenerateSafeProjection: code for mapobjects(lambdavariable(MapObject, StringType, true, -1), lambdavariable(MapObject, StringType, true, -1).toString, input[0, array, true], Some(interface scala.collection.Seq)): /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificSafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificSafeProjection extends org.apache.spark.sql.catalyst.expressions.codegen.BaseProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private InternalRow mutableRow; /* 009 */ private UTF8String value_MapObject_lambda_variable_1; /* 010 */ private boolean isNull_MapObject_lambda_variable_1; /* 011 */ private boolean globalIsNull_0; /* 012 */ /* 013 */ public SpecificSafeProjection(Object[] references) { /* 014 */ this.references = references; /* 015 */ mutableRow = (InternalRow) references[references.length - 1]; /* 016 */ /* 017 */ /* 018 */ } /* 019 */ /* 020 */ public void initialize(int partitionIndex) { /* 021 */ /* 022 */ } /* 023 */ /* 024 */ public java.lang.Object apply(java.lang.Object _i) { /* 025 */ InternalRow i = (InternalRow) _i; /* 026 */ scala.collection.Seq value_3 = MapObjects_0(i); /* 027 */ if (globalIsNull_0) { /* 028 */ mutableRow.setNullAt(0); /* 029 */ } else { /* 030 */ /* 031 */ mutableRow.update(0, value_3); /* 032 */ } /* 033 */ /* 034 */ return mutableRow; /* 035 */ } /* 036 */ /* 037 */ /* 038 */ private scala.collection.Seq MapObjects_0(InternalRow i) { /* 039 */ boolean isNull_1 = i.isNullAt(0); /* 040 */ ArrayData value_1 = isNull_1 ? /* 041 */ null : (i.getArray(0)); /* 042 */ scala.collection.Seq value_0 = null; /* 043 */ /* 044 */ if (!isNull_1) { /* 045 */ /* 046 */ int dataLength_0 = value_1.numElements(); /* 047 */ /* 048 */ scala.collection.mutable.Builder collectionBuilder_0 = scala.collection.Seq$.MODULE$.newBuilder(); /* 049 */ collectionBuilder_0.sizeHint(dataLength_0); /* 050 */ /* 051 */ /* 052 */ int loopIndex_0 = 0; /* 053 */ /* 054 */ while (loopIndex_0 < dataLength_0) { /* 055 */ value_MapObject_lambda_variable_1 = (UTF8String) (value_1.getUTF8String(loopIndex_0)); /* 056 */ isNull_MapObject_lambda_variable_1 = value_1.isNullAt(loopIndex_0); /* 057 */ /* 058 */ boolean isNull_2 = true; /* 059 */ java.lang.String value_2 = null; /* 060 */ if (!isNull_MapObject_lambda_variable_1) { /* 061 */ /* 062 */ isNull_2 = false; /* 063 */ if (!isNull_2) { /* 064 */ /* 065 */ Object funcResult_0 = null; /* 066 */ funcResult_0 = value_MapObject_lambda_variable_1.toString(); /* 067 */ value_2 = (java.lang.String) funcResult_0; /* 068 */ /* 069 */ } /* 070 */ } /* 071 */ if (isNull_2) { /* 072 */ collectionBuilder_0.$plus$eq(null); /* 073 */ } else { /* 074 */ collectionBuilder_0.$plus$eq(value_2); /* 075 */ } /* 076 */ /* 077 */ loopIndex_0 += 1; /* 078 */ } /* 079 */ /* 080 */ value_0 = (scala.collection.Seq) collectionBuilder_0.result(); /* 081 */ } /* 082 */ globalIsNull_0 = isNull_1; /* 083 */ return value_0; /* 084 */ } /* 085 */ /* 086 */ } 20/08/08 15:33:52.758 Executor task launch worker for task 15 DEBUG BlockManager: Block rdd_20_6 was not found 20/08/08 15:33:52.758 Executor task launch worker for task 15 DEBUG BlockManager: Getting remote block rdd_20_6 20/08/08 15:33:52.758 Executor task launch worker for task 16 DEBUG BlockManager: Block rdd_19_7 is unknown by block manager master 20/08/08 15:33:52.758 Executor task launch worker for task 18 INFO Executor: Finished task 9.0 in stage 3.0 (TID 18). 2170 bytes result sent to driver 20/08/08 15:33:52.758 Executor task launch worker for task 16 DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 0 20/08/08 15:33:52.758 Executor task launch worker for task 16 DEBUG MapOutputTrackerMaster: Convert map statuses for shuffle 0, mappers 0-4, partitions 7-8 20/08/08 15:33:52.759 Executor task launch worker for task 17 INFO MemoryStore: Block rdd_20_8 stored as values in memory (estimated size 16.0 B, free 366.2 MiB) 20/08/08 15:33:52.763 Executor task launch worker for task 16 DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 20/08/08 15:33:52.764 Executor task launch worker for task 16 INFO ShuffleBlockFetcherIterator: Getting 0 (0.0 B) non-empty blocks including 0 (0.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 20/08/08 15:33:52.764 Executor task launch worker for task 16 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 20/08/08 15:33:52.764 Executor task launch worker for task 16 DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: 20/08/08 15:33:52.764 Executor task launch worker for task 16 DEBUG ShuffleBlockFetcherIterator: Got local blocks in 0 ms 20/08/08 15:33:52.764 Executor task launch worker for task 6 DEBUG GenerateUnsafeProjection: code for newInstance(class org.apache.spark.ml.linalg.VectorUDT).serialize: /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificUnsafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificUnsafeProjection extends org.apache.spark.sql.catalyst.expressions.UnsafeProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private boolean resultIsNull_0; /* 009 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[] mutableStateArray_1 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[2]; /* 010 */ private org.apache.spark.ml.linalg.Vector[] mutableStateArray_0 = new org.apache.spark.ml.linalg.Vector[1]; /* 011 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[] mutableStateArray_2 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[2]; /* 012 */ /* 013 */ public SpecificUnsafeProjection(Object[] references) { /* 014 */ this.references = references; /* 015 */ /* 016 */ mutableStateArray_1[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(1, 32); /* 017 */ mutableStateArray_1[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(mutableStateArray_1[0], 4); /* 018 */ mutableStateArray_2[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(mutableStateArray_1[1], 4); /* 019 */ mutableStateArray_2[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(mutableStateArray_1[1], 8); /* 020 */ /* 021 */ } /* 022 */ /* 023 */ public void initialize(int partitionIndex) { /* 024 */ /* 025 */ } /* 026 */ /* 027 */ // Scala.Function1 need this /* 028 */ public java.lang.Object apply(java.lang.Object row) { /* 029 */ return apply((InternalRow) row); /* 030 */ } /* 031 */ /* 032 */ public UnsafeRow apply(InternalRow i) { /* 033 */ mutableStateArray_1[0].reset(); /* 034 */ /* 035 */ /* 036 */ mutableStateArray_1[0].zeroOutNullBytes(); /* 037 */ /* 038 */ final org.apache.spark.ml.linalg.VectorUDT value_1 = false ? /* 039 */ null : new org.apache.spark.ml.linalg.VectorUDT(); /* 040 */ boolean isNull_0 = true; /* 041 */ InternalRow value_0 = null; /* 042 */ if (!false) { /* 043 */ resultIsNull_0 = false; /* 044 */ if (!resultIsNull_0) { /* 045 */ boolean isNull_2 = i.isNullAt(0); /* 046 */ org.apache.spark.ml.linalg.Vector value_2 = isNull_2 ? /* 047 */ null : ((org.apache.spark.ml.linalg.Vector)i.get(0, null)); /* 048 */ resultIsNull_0 = isNull_2; /* 049 */ mutableStateArray_0[0] = value_2; /* 050 */ } /* 051 */ /* 052 */ isNull_0 = resultIsNull_0; /* 053 */ if (!isNull_0) { /* 054 */ /* 055 */ Object funcResult_0 = null; /* 056 */ funcResult_0 = value_1.serialize(mutableStateArray_0[0]); /* 057 */ /* 058 */ if (funcResult_0 != null) { /* 059 */ value_0 = (InternalRow) funcResult_0; /* 060 */ } else { /* 061 */ isNull_0 = true; /* 062 */ } /* 063 */ /* 064 */ /* 065 */ } /* 066 */ } /* 067 */ if (isNull_0) { /* 068 */ mutableStateArray_1[0].setNullAt(0); /* 069 */ } else { /* 070 */ final InternalRow tmpInput_0 = value_0; /* 071 */ if (tmpInput_0 instanceof UnsafeRow) { /* 072 */ mutableStateArray_1[0].write(0, (UnsafeRow) tmpInput_0); /* 073 */ } else { /* 074 */ // Remember the current cursor so that we can calculate how many bytes are /* 075 */ // written later. /* 076 */ final int previousCursor_0 = mutableStateArray_1[0].cursor(); /* 077 */ /* 078 */ mutableStateArray_1[1].resetRowWriter(); /* 079 */ /* 080 */ /* 081 */ mutableStateArray_1[1].write(0, (tmpInput_0.getByte(0))); /* 082 */ /* 083 */ /* 084 */ if ((tmpInput_0.isNullAt(1))) { /* 085 */ mutableStateArray_1[1].setNullAt(1); /* 086 */ } else { /* 087 */ mutableStateArray_1[1].write(1, (tmpInput_0.getInt(1))); /* 088 */ } /* 089 */ /* 090 */ /* 091 */ if ((tmpInput_0.isNullAt(2))) { /* 092 */ mutableStateArray_1[1].setNullAt(2); /* 093 */ } else { /* 094 */ // Remember the current cursor so that we can calculate how many bytes are /* 095 */ // written later. /* 096 */ final int previousCursor_1 = mutableStateArray_1[1].cursor(); /* 097 */ /* 098 */ final ArrayData tmpInput_1 = (tmpInput_0.getArray(2)); /* 099 */ if (tmpInput_1 instanceof UnsafeArrayData) { /* 100 */ mutableStateArray_1[1].write((UnsafeArrayData) tmpInput_1); /* 101 */ } else { /* 102 */ final int numElements_0 = tmpInput_1.numElements(); /* 103 */ mutableStateArray_2[0].initialize(numElements_0); /* 104 */ /* 105 */ for (int index_0 = 0; index_0 < numElements_0; index_0++) { /* 106 */ mutableStateArray_2[0].write(index_0, tmpInput_1.getInt(index_0)); /* 107 */ } /* 108 */ } /* 109 */ /* 110 */ mutableStateArray_1[1].setOffsetAndSizeFromPreviousCursor(2, previousCursor_1); /* 111 */ } /* 112 */ /* 113 */ /* 114 */ if ((tmpInput_0.isNullAt(3))) { /* 115 */ mutableStateArray_1[1].setNullAt(3); /* 116 */ } else { /* 117 */ // Remember the current cursor so that we can calculate how many bytes are /* 118 */ // written later. /* 119 */ final int previousCursor_2 = mutableStateArray_1[1].cursor(); /* 120 */ /* 121 */ final ArrayData tmpInput_2 = (tmpInput_0.getArray(3)); /* 122 */ if (tmpInput_2 instanceof UnsafeArrayData) { /* 123 */ mutableStateArray_1[1].write((UnsafeArrayData) tmpInput_2); /* 124 */ } else { /* 125 */ final int numElements_1 = tmpInput_2.numElements(); /* 126 */ mutableStateArray_2[1].initialize(numElements_1); /* 127 */ /* 128 */ for (int index_1 = 0; index_1 < numElements_1; index_1++) { /* 129 */ mutableStateArray_2[1].write(index_1, tmpInput_2.getDouble(index_1)); /* 130 */ } /* 131 */ } /* 132 */ /* 133 */ mutableStateArray_1[1].setOffsetAndSizeFromPreviousCursor(3, previousCursor_2); /* 134 */ } /* 135 */ /* 136 */ /* 137 */ mutableStateArray_1[0].setOffsetAndSizeFromPreviousCursor(0, previousCursor_0); /* 138 */ } /* 139 */ } /* 140 */ return (mutableStateArray_1[0].getRow()); /* 141 */ } /* 142 */ /* 143 */ /* 144 */ } 20/08/08 15:33:52.765 Executor task launch worker for task 6 INFO PythonRunner: Times: total = 88, boot = -820, init = 908, finish = 0 20/08/08 15:33:52.765 Executor task launch worker for task 16 DEBUG TaskMemoryManager: Task 16 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@19268e70 20/08/08 15:33:52.763 Executor task launch worker for task 1 DEBUG GenerateSafeProjection: code for newInstance(class org.apache.spark.ml.linalg.VectorUDT).deserialize: /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificSafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificSafeProjection extends org.apache.spark.sql.catalyst.expressions.codegen.BaseProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private InternalRow mutableRow; /* 009 */ private boolean resultIsNull_0; /* 010 */ private InternalRow[] mutableStateArray_0 = new InternalRow[1]; /* 011 */ /* 012 */ public SpecificSafeProjection(Object[] references) { /* 013 */ this.references = references; /* 014 */ mutableRow = (InternalRow) references[references.length - 1]; /* 015 */ /* 016 */ /* 017 */ } /* 018 */ /* 019 */ public void initialize(int partitionIndex) { /* 020 */ /* 021 */ } /* 022 */ /* 023 */ public java.lang.Object apply(java.lang.Object _i) { /* 024 */ InternalRow i = (InternalRow) _i; /* 025 */ final org.apache.spark.ml.linalg.VectorUDT value_1 = false ? /* 026 */ null : new org.apache.spark.ml.linalg.VectorUDT(); /* 027 */ boolean isNull_0 = true; /* 028 */ org.apache.spark.ml.linalg.Vector value_0 = null; /* 029 */ if (!false) { /* 030 */ resultIsNull_0 = false; /* 031 */ if (!resultIsNull_0) { /* 032 */ boolean isNull_2 = i.isNullAt(0); /* 033 */ InternalRow value_2 = isNull_2 ? /* 034 */ null : (i.getStruct(0, 4)); /* 035 */ resultIsNull_0 = isNull_2; /* 036 */ mutableStateArray_0[0] = value_2; /* 037 */ } /* 038 */ /* 039 */ isNull_0 = resultIsNull_0; /* 040 */ if (!isNull_0) { /* 041 */ /* 042 */ Object funcResult_0 = null; /* 043 */ funcResult_0 = value_1.deserialize(mutableStateArray_0[0]); /* 044 */ /* 045 */ if (funcResult_0 != null) { /* 046 */ value_0 = (org.apache.spark.ml.linalg.Vector) funcResult_0; /* 047 */ } else { /* 048 */ isNull_0 = true; /* 049 */ } /* 050 */ /* 051 */ /* 052 */ } /* 053 */ } /* 054 */ if (isNull_0) { /* 055 */ mutableRow.setNullAt(0); /* 056 */ } else { /* 057 */ /* 058 */ mutableRow.update(0, value_0); /* 059 */ } /* 060 */ /* 061 */ return mutableRow; /* 062 */ } /* 063 */ /* 064 */ /* 065 */ } 20/08/08 15:33:52.766 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_20_8 for BlockManagerId(driver, 192.168.1.225, 38159, None) 20/08/08 15:33:52.766 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_20_8 in memory on 192.168.1.225:38159 (size: 16.0 B, free: 366.3 MiB) 20/08/08 15:33:52.766 Executor task launch worker for task 17 DEBUG BlockManagerMaster: Updated info of block rdd_20_8 20/08/08 15:33:52.767 Executor task launch worker for task 17 DEBUG BlockManager: Told master about block rdd_20_8 20/08/08 15:33:52.767 Executor task launch worker for task 17 DEBUG BlockManager: Put block rdd_20_8 locally took 16 ms 20/08/08 15:33:52.767 Executor task launch worker for task 15 DEBUG BlockManager: Block rdd_20_6 is unknown by block manager master 20/08/08 15:33:52.767 Executor task launch worker for task 17 DEBUG BlockManager: Putting block rdd_20_8 without replication took 16 ms 20/08/08 15:33:52.767 Executor task launch worker for task 17 DEBUG BlockManager: Getting local block rdd_20_8 20/08/08 15:33:52.767 Executor task launch worker for task 15 DEBUG BlockManager: Getting local block rdd_19_6 20/08/08 15:33:52.767 Executor task launch worker for task 17 DEBUG BlockManager: Level for block rdd_20_8 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:52.768 Executor task launch worker for task 17 INFO Executor: Finished task 8.0 in stage 3.0 (TID 17). 2170 bytes result sent to driver 20/08/08 15:33:52.768 Executor task launch worker for task 17 DEBUG ExecutorMetricsPoller: stageTCMP: (3, 0) -> 3 20/08/08 15:33:52.768 Executor task launch worker for task 18 DEBUG ExecutorMetricsPoller: stageTCMP: (3, 0) -> 2 20/08/08 15:33:52.769 dispatcher-event-loop-2 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_3.0, runningTasks: 3 20/08/08 15:33:52.769 dispatcher-event-loop-2 DEBUG TaskSetManager: No tasks for locality level NODE_LOCAL, so moving to locality level NO_PREF 20/08/08 15:33:52.769 dispatcher-event-loop-2 DEBUG TaskSetManager: No tasks for locality level NO_PREF, so moving to locality level ANY 20/08/08 15:33:52.771 dispatcher-event-loop-2 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_3.0, runningTasks: 2 20/08/08 15:33:52.772 task-result-getter-1 INFO TaskSetManager: Finished task 9.0 in stage 3.0 (TID 18) in 42 ms on 192.168.1.225 (executor driver) (7/10) 20/08/08 15:33:52.772 Executor task launch worker for task 6 INFO Executor: Finished task 2.0 in stage 1.0 (TID 6). 2047 bytes result sent to driver 20/08/08 15:33:52.773 task-result-getter-3 INFO TaskSetManager: Finished task 8.0 in stage 3.0 (TID 17) in 52 ms on 192.168.1.225 (executor driver) (8/10) 20/08/08 15:33:52.773 Executor task launch worker for task 6 DEBUG ExecutorMetricsPoller: removing (1, 0) from stageTCMP 20/08/08 15:33:52.773 Executor task launch worker for task 16 INFO MemoryStore: Block rdd_19_7 stored as values in memory (estimated size 16.0 B, free 366.2 MiB) 20/08/08 15:33:52.767 Executor task launch worker for task 15 DEBUG BlockManager: Block rdd_19_6 was not found 20/08/08 15:33:52.773 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_19_7 for BlockManagerId(driver, 192.168.1.225, 38159, None) 20/08/08 15:33:52.773 Executor task launch worker for task 15 DEBUG BlockManager: Getting remote block rdd_19_6 20/08/08 15:33:52.774 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_19_7 in memory on 192.168.1.225:38159 (size: 16.0 B, free: 366.3 MiB) 20/08/08 15:33:52.773 dispatcher-event-loop-0 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_1.0, runningTasks: 0 20/08/08 15:33:52.774 Executor task launch worker for task 15 DEBUG BlockManager: Block rdd_19_6 is unknown by block manager master 20/08/08 15:33:52.774 Executor task launch worker for task 15 DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 0 20/08/08 15:33:52.774 Executor task launch worker for task 15 DEBUG MapOutputTrackerMaster: Convert map statuses for shuffle 0, mappers 0-4, partitions 6-7 20/08/08 15:33:52.776 task-result-getter-3 INFO TaskSetManager: Finished task 2.0 in stage 1.0 (TID 6) in 237 ms on 192.168.1.225 (executor driver) (4/4) 20/08/08 15:33:52.776 task-result-getter-3 INFO TaskSchedulerImpl: Removed TaskSet 1.0, whose tasks have all completed, from pool 20/08/08 15:33:52.775 Executor task launch worker for task 15 DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 20/08/08 15:33:52.774 Executor task launch worker for task 16 DEBUG BlockManagerMaster: Updated info of block rdd_19_7 20/08/08 15:33:52.777 dag-scheduler-event-loop INFO DAGScheduler: ResultStage 1 ( at /usr/lib/python3.6/runpy.py:85) finished in 0.264 s 20/08/08 15:33:52.777 dag-scheduler-event-loop DEBUG DAGScheduler: After removal of stage 1, remaining stages = 0 20/08/08 15:33:52.778 dag-scheduler-event-loop INFO DAGScheduler: Job 1 is finished. Cancelling potential speculative or zombie tasks for this job 20/08/08 15:33:52.778 dag-scheduler-event-loop INFO TaskSchedulerImpl: Killing all running tasks in stage 1: Stage finished 20/08/08 15:33:52.776 Executor task launch worker for task 16 DEBUG BlockManager: Told master about block rdd_19_7 20/08/08 15:33:52.778 Executor task launch worker for task 16 DEBUG BlockManager: Put block rdd_19_7 locally took 19 ms 20/08/08 15:33:52.778 Thread-4 INFO DAGScheduler: Job 1 finished: at /usr/lib/python3.6/runpy.py:85, took 0.269765 s 20/08/08 15:33:52.778 Executor task launch worker for task 16 DEBUG BlockManager: Putting block rdd_19_7 without replication took 19 ms 20/08/08 15:33:52.778 Executor task launch worker for task 16 DEBUG BlockManager: Getting local block rdd_19_7 20/08/08 15:33:52.778 Executor task launch worker for task 16 DEBUG BlockManager: Level for block rdd_19_7 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:52.779 Executor task launch worker for task 16 INFO MemoryStore: Block rdd_20_7 stored as values in memory (estimated size 16.0 B, free 366.2 MiB) 20/08/08 15:33:52.777 Executor task launch worker for task 15 INFO ShuffleBlockFetcherIterator: Getting 0 (0.0 B) non-empty blocks including 0 (0.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 20/08/08 15:33:52.779 Executor task launch worker for task 15 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 3 ms 20/08/08 15:33:52.779 Executor task launch worker for task 15 DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: 20/08/08 15:33:52.779 Executor task launch worker for task 15 DEBUG ShuffleBlockFetcherIterator: Got local blocks in 4 ms 20/08/08 15:33:52.780 Executor task launch worker for task 15 DEBUG TaskMemoryManager: Task 15 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@447c4476 20/08/08 15:33:52.781 Executor task launch worker for task 15 INFO MemoryStore: Block rdd_19_6 stored as values in memory (estimated size 16.0 B, free 366.2 MiB) 20/08/08 15:33:52.781 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_20_7 for BlockManagerId(driver, 192.168.1.225, 38159, None) 20/08/08 15:33:52.781 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_20_7 in memory on 192.168.1.225:38159 (size: 16.0 B, free: 366.3 MiB) 20/08/08 15:33:52.781 Executor task launch worker for task 2 DEBUG GenerateSafeProjection: code for newInstance(class org.apache.spark.ml.linalg.VectorUDT).deserialize: /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificSafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificSafeProjection extends org.apache.spark.sql.catalyst.expressions.codegen.BaseProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private InternalRow mutableRow; /* 009 */ private boolean resultIsNull_0; /* 010 */ private InternalRow[] mutableStateArray_0 = new InternalRow[1]; /* 011 */ /* 012 */ public SpecificSafeProjection(Object[] references) { /* 013 */ this.references = references; /* 014 */ mutableRow = (InternalRow) references[references.length - 1]; /* 015 */ /* 016 */ /* 017 */ } /* 018 */ /* 019 */ public void initialize(int partitionIndex) { /* 020 */ /* 021 */ } /* 022 */ /* 023 */ public java.lang.Object apply(java.lang.Object _i) { /* 024 */ InternalRow i = (InternalRow) _i; /* 025 */ final org.apache.spark.ml.linalg.VectorUDT value_1 = false ? /* 026 */ null : new org.apache.spark.ml.linalg.VectorUDT(); /* 027 */ boolean isNull_0 = true; /* 028 */ org.apache.spark.ml.linalg.Vector value_0 = null; /* 029 */ if (!false) { /* 030 */ resultIsNull_0 = false; /* 031 */ if (!resultIsNull_0) { /* 032 */ boolean isNull_2 = i.isNullAt(0); /* 033 */ InternalRow value_2 = isNull_2 ? /* 034 */ null : (i.getStruct(0, 4)); /* 035 */ resultIsNull_0 = isNull_2; /* 036 */ mutableStateArray_0[0] = value_2; /* 037 */ } /* 038 */ /* 039 */ isNull_0 = resultIsNull_0; /* 040 */ if (!isNull_0) { /* 041 */ /* 042 */ Object funcResult_0 = null; /* 043 */ funcResult_0 = value_1.deserialize(mutableStateArray_0[0]); /* 044 */ /* 045 */ if (funcResult_0 != null) { /* 046 */ value_0 = (org.apache.spark.ml.linalg.Vector) funcResult_0; /* 047 */ } else { /* 048 */ isNull_0 = true; /* 049 */ } /* 050 */ /* 051 */ /* 052 */ } /* 053 */ } /* 054 */ if (isNull_0) { /* 055 */ mutableRow.setNullAt(0); /* 056 */ } else { /* 057 */ /* 058 */ mutableRow.update(0, value_0); /* 059 */ } /* 060 */ /* 061 */ return mutableRow; /* 062 */ } /* 063 */ /* 064 */ /* 065 */ } 20/08/08 15:33:52.782 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_19_6 for BlockManagerId(driver, 192.168.1.225, 38159, None) 20/08/08 15:33:52.782 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_19_6 in memory on 192.168.1.225:38159 (size: 16.0 B, free: 366.3 MiB) 20/08/08 15:33:52.782 Executor task launch worker for task 16 DEBUG BlockManagerMaster: Updated info of block rdd_20_7 20/08/08 15:33:52.782 Executor task launch worker for task 16 DEBUG BlockManager: Told master about block rdd_20_7 20/08/08 15:33:52.782 Executor task launch worker for task 16 DEBUG BlockManager: Put block rdd_20_7 locally took 51 ms 20/08/08 15:33:52.782 Executor task launch worker for task 16 DEBUG BlockManager: Putting block rdd_20_7 without replication took 51 ms 20/08/08 15:33:52.782 Executor task launch worker for task 16 DEBUG BlockManager: Getting local block rdd_20_7 20/08/08 15:33:52.783 Executor task launch worker for task 16 DEBUG BlockManager: Level for block rdd_20_7 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:52.784 Executor task launch worker for task 16 INFO Executor: Finished task 7.0 in stage 3.0 (TID 16). 2170 bytes result sent to driver 20/08/08 15:33:52.784 dispatcher-event-loop-1 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_3.0, runningTasks: 1 20/08/08 15:33:52.785 task-result-getter-2 INFO TaskSetManager: Finished task 7.0 in stage 3.0 (TID 16) in 78 ms on 192.168.1.225 (executor driver) (9/10) 20/08/08 15:33:52.785 Executor task launch worker for task 15 DEBUG BlockManagerMaster: Updated info of block rdd_19_6 20/08/08 15:33:52.785 Executor task launch worker for task 15 DEBUG BlockManager: Told master about block rdd_19_6 20/08/08 15:33:52.785 Executor task launch worker for task 15 DEBUG BlockManager: Put block rdd_19_6 locally took 11 ms 20/08/08 15:33:52.785 Executor task launch worker for task 15 DEBUG BlockManager: Putting block rdd_19_6 without replication took 11 ms 20/08/08 15:33:52.785 Executor task launch worker for task 15 DEBUG BlockManager: Getting local block rdd_19_6 20/08/08 15:33:52.786 Executor task launch worker for task 15 DEBUG BlockManager: Level for block rdd_19_6 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:52.786 Executor task launch worker for task 4 DEBUG GenerateSafeProjection: code for newInstance(class org.apache.spark.ml.linalg.VectorUDT).deserialize: /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificSafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificSafeProjection extends org.apache.spark.sql.catalyst.expressions.codegen.BaseProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private InternalRow mutableRow; /* 009 */ private boolean resultIsNull_0; /* 010 */ private InternalRow[] mutableStateArray_0 = new InternalRow[1]; /* 011 */ /* 012 */ public SpecificSafeProjection(Object[] references) { /* 013 */ this.references = references; /* 014 */ mutableRow = (InternalRow) references[references.length - 1]; /* 015 */ /* 016 */ /* 017 */ } /* 018 */ /* 019 */ public void initialize(int partitionIndex) { /* 020 */ /* 021 */ } /* 022 */ /* 023 */ public java.lang.Object apply(java.lang.Object _i) { /* 024 */ InternalRow i = (InternalRow) _i; /* 025 */ final org.apache.spark.ml.linalg.VectorUDT value_1 = false ? /* 026 */ null : new org.apache.spark.ml.linalg.VectorUDT(); /* 027 */ boolean isNull_0 = true; /* 028 */ org.apache.spark.ml.linalg.Vector value_0 = null; /* 029 */ if (!false) { /* 030 */ resultIsNull_0 = false; /* 031 */ if (!resultIsNull_0) { /* 032 */ boolean isNull_2 = i.isNullAt(0); /* 033 */ InternalRow value_2 = isNull_2 ? /* 034 */ null : (i.getStruct(0, 4)); /* 035 */ resultIsNull_0 = isNull_2; /* 036 */ mutableStateArray_0[0] = value_2; /* 037 */ } /* 038 */ /* 039 */ isNull_0 = resultIsNull_0; /* 040 */ if (!isNull_0) { /* 041 */ /* 042 */ Object funcResult_0 = null; /* 043 */ funcResult_0 = value_1.deserialize(mutableStateArray_0[0]); /* 044 */ /* 045 */ if (funcResult_0 != null) { /* 046 */ value_0 = (org.apache.spark.ml.linalg.Vector) funcResult_0; /* 047 */ } else { /* 048 */ isNull_0 = true; /* 049 */ } /* 050 */ /* 051 */ /* 052 */ } /* 053 */ } /* 054 */ if (isNull_0) { /* 055 */ mutableRow.setNullAt(0); /* 056 */ } else { /* 057 */ /* 058 */ mutableRow.update(0, value_0); /* 059 */ } /* 060 */ /* 061 */ return mutableRow; /* 062 */ } /* 063 */ /* 064 */ /* 065 */ } 20/08/08 15:33:52.786 Executor task launch worker for task 15 INFO MemoryStore: Block rdd_20_6 stored as values in memory (estimated size 16.0 B, free 366.2 MiB) 20/08/08 15:33:52.787 Executor task launch worker for task 16 DEBUG ExecutorMetricsPoller: stageTCMP: (3, 0) -> 1 20/08/08 15:33:52.787 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_20_6 for BlockManagerId(driver, 192.168.1.225, 38159, None) 20/08/08 15:33:52.787 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_20_6 in memory on 192.168.1.225:38159 (size: 16.0 B, free: 366.3 MiB) 20/08/08 15:33:52.787 Executor task launch worker for task 3 DEBUG GenerateSafeProjection: code for newInstance(class org.apache.spark.ml.linalg.VectorUDT).deserialize: /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificSafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificSafeProjection extends org.apache.spark.sql.catalyst.expressions.codegen.BaseProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private InternalRow mutableRow; /* 009 */ private boolean resultIsNull_0; /* 010 */ private InternalRow[] mutableStateArray_0 = new InternalRow[1]; /* 011 */ /* 012 */ public SpecificSafeProjection(Object[] references) { /* 013 */ this.references = references; /* 014 */ mutableRow = (InternalRow) references[references.length - 1]; /* 015 */ /* 016 */ /* 017 */ } /* 018 */ /* 019 */ public void initialize(int partitionIndex) { /* 020 */ /* 021 */ } /* 022 */ /* 023 */ public java.lang.Object apply(java.lang.Object _i) { /* 024 */ InternalRow i = (InternalRow) _i; /* 025 */ final org.apache.spark.ml.linalg.VectorUDT value_1 = false ? /* 026 */ null : new org.apache.spark.ml.linalg.VectorUDT(); /* 027 */ boolean isNull_0 = true; /* 028 */ org.apache.spark.ml.linalg.Vector value_0 = null; /* 029 */ if (!false) { /* 030 */ resultIsNull_0 = false; /* 031 */ if (!resultIsNull_0) { /* 032 */ boolean isNull_2 = i.isNullAt(0); /* 033 */ InternalRow value_2 = isNull_2 ? /* 034 */ null : (i.getStruct(0, 4)); /* 035 */ resultIsNull_0 = isNull_2; /* 036 */ mutableStateArray_0[0] = value_2; /* 037 */ } /* 038 */ /* 039 */ isNull_0 = resultIsNull_0; /* 040 */ if (!isNull_0) { /* 041 */ /* 042 */ Object funcResult_0 = null; /* 043 */ funcResult_0 = value_1.deserialize(mutableStateArray_0[0]); /* 044 */ /* 045 */ if (funcResult_0 != null) { /* 046 */ value_0 = (org.apache.spark.ml.linalg.Vector) funcResult_0; /* 047 */ } else { /* 048 */ isNull_0 = true; /* 049 */ } /* 050 */ /* 051 */ /* 052 */ } /* 053 */ } /* 054 */ if (isNull_0) { /* 055 */ mutableRow.setNullAt(0); /* 056 */ } else { /* 057 */ /* 058 */ mutableRow.update(0, value_0); /* 059 */ } /* 060 */ /* 061 */ return mutableRow; /* 062 */ } /* 063 */ /* 064 */ /* 065 */ } 20/08/08 15:33:52.788 Executor task launch worker for task 15 DEBUG BlockManagerMaster: Updated info of block rdd_20_6 20/08/08 15:33:52.788 Executor task launch worker for task 15 DEBUG BlockManager: Told master about block rdd_20_6 20/08/08 15:33:52.788 Executor task launch worker for task 15 DEBUG BlockManager: Put block rdd_20_6 locally took 21 ms 20/08/08 15:33:52.788 Executor task launch worker for task 15 DEBUG BlockManager: Putting block rdd_20_6 without replication took 21 ms 20/08/08 15:33:52.788 Executor task launch worker for task 15 DEBUG BlockManager: Getting local block rdd_20_6 20/08/08 15:33:52.788 Executor task launch worker for task 15 DEBUG BlockManager: Level for block rdd_20_6 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:52.790 Executor task launch worker for task 15 INFO Executor: Finished task 6.0 in stage 3.0 (TID 15). 2170 bytes result sent to driver 20/08/08 15:33:52.790 dispatcher-event-loop-0 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_3.0, runningTasks: 0 20/08/08 15:33:52.791 task-result-getter-0 INFO TaskSetManager: Finished task 6.0 in stage 3.0 (TID 15) in 86 ms on 192.168.1.225 (executor driver) (10/10) 20/08/08 15:33:52.792 task-result-getter-0 INFO TaskSchedulerImpl: Removed TaskSet 3.0, whose tasks have all completed, from pool 20/08/08 15:33:52.792 dag-scheduler-event-loop INFO DAGScheduler: ResultStage 3 (count at ALS.scala:960) finished in 0.225 s 20/08/08 15:33:52.793 dag-scheduler-event-loop DEBUG DAGScheduler: After removal of stage 2, remaining stages = 2 20/08/08 15:33:52.793 Executor task launch worker for task 15 DEBUG ExecutorMetricsPoller: removing (3, 0) from stageTCMP 20/08/08 15:33:52.793 dag-scheduler-event-loop DEBUG DAGScheduler: After removal of stage 1, remaining stages = 1 20/08/08 15:33:52.793 dag-scheduler-event-loop DEBUG DAGScheduler: After removal of stage 3, remaining stages = 0 20/08/08 15:33:52.793 dag-scheduler-event-loop INFO DAGScheduler: Job 1 is finished. Cancelling potential speculative or zombie tasks for this job 20/08/08 15:33:52.793 dag-scheduler-event-loop INFO TaskSchedulerImpl: Killing all running tasks in stage 3: Stage finished 20/08/08 15:33:52.793 Thread-4 INFO DAGScheduler: Job 1 finished: count at ALS.scala:960, took 1.111593 s 20/08/08 15:33:52.796 Thread-4 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$train$3 20/08/08 15:33:52.799 Thread-4 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$train$3) is now cleaned +++ 20/08/08 15:33:52.800 Thread-4 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$makeBlocks$1 20/08/08 15:33:52.797 Executor task launch worker for task 1 DEBUG CodeGenerator: /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificSafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificSafeProjection extends org.apache.spark.sql.catalyst.expressions.codegen.BaseProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private InternalRow mutableRow; /* 009 */ private boolean resultIsNull_0; /* 010 */ private InternalRow[] mutableStateArray_0 = new InternalRow[1]; /* 011 */ /* 012 */ public SpecificSafeProjection(Object[] references) { /* 013 */ this.references = references; /* 014 */ mutableRow = (InternalRow) references[references.length - 1]; /* 015 */ /* 016 */ /* 017 */ } /* 018 */ /* 019 */ public void initialize(int partitionIndex) { /* 020 */ /* 021 */ } /* 022 */ /* 023 */ public java.lang.Object apply(java.lang.Object _i) { /* 024 */ InternalRow i = (InternalRow) _i; /* 025 */ final org.apache.spark.ml.linalg.VectorUDT value_1 = false ? /* 026 */ null : new org.apache.spark.ml.linalg.VectorUDT(); /* 027 */ boolean isNull_0 = true; /* 028 */ org.apache.spark.ml.linalg.Vector value_0 = null; /* 029 */ if (!false) { /* 030 */ resultIsNull_0 = false; /* 031 */ if (!resultIsNull_0) { /* 032 */ boolean isNull_2 = i.isNullAt(0); /* 033 */ InternalRow value_2 = isNull_2 ? /* 034 */ null : (i.getStruct(0, 4)); /* 035 */ resultIsNull_0 = isNull_2; /* 036 */ mutableStateArray_0[0] = value_2; /* 037 */ } /* 038 */ /* 039 */ isNull_0 = resultIsNull_0; /* 040 */ if (!isNull_0) { /* 041 */ /* 042 */ Object funcResult_0 = null; /* 043 */ funcResult_0 = value_1.deserialize(mutableStateArray_0[0]); /* 044 */ /* 045 */ if (funcResult_0 != null) { /* 046 */ value_0 = (org.apache.spark.ml.linalg.Vector) funcResult_0; /* 047 */ } else { /* 048 */ isNull_0 = true; /* 049 */ } /* 050 */ /* 051 */ /* 052 */ } /* 053 */ } /* 054 */ if (isNull_0) { /* 055 */ mutableRow.setNullAt(0); /* 056 */ } else { /* 057 */ /* 058 */ mutableRow.update(0, value_0); /* 059 */ } /* 060 */ /* 061 */ return mutableRow; /* 062 */ } /* 063 */ /* 064 */ /* 065 */ } 20/08/08 15:33:52.802 Thread-4 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$makeBlocks$1) is now cleaned +++ 20/08/08 15:33:52.803 Thread-4 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$groupByKey$2 20/08/08 15:33:52.813 Thread-4 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$groupByKey$2) is now cleaned +++ 20/08/08 15:33:52.815 Executor task launch worker for task 1 INFO CodeGenerator: Code generated in 50.206113 ms 20/08/08 15:33:52.819 Thread-4 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$groupByKey$3 20/08/08 15:33:52.822 Thread-4 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$groupByKey$3) is now cleaned +++ 20/08/08 15:33:52.823 Thread-4 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$groupByKey$4 20/08/08 15:33:52.826 Thread-4 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$groupByKey$4) is now cleaned +++ 20/08/08 15:33:52.827 Thread-4 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$makeBlocks$5 20/08/08 15:33:52.829 Thread-4 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$makeBlocks$5) is now cleaned +++ 20/08/08 15:33:52.832 Thread-4 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$makeBlocks$7 20/08/08 15:33:52.828 Executor task launch worker for task 2 WARN BLAS: Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS 20/08/08 15:33:52.834 Executor task launch worker for task 2 WARN BLAS: Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS 20/08/08 15:33:52.834 Thread-4 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$makeBlocks$7) is now cleaned +++ 20/08/08 15:33:52.838 Thread-4 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$count$1$adapted 20/08/08 15:33:52.844 Thread-4 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$count$1$adapted) is now cleaned +++ 20/08/08 15:33:52.844 Thread-4 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$runJob$5 20/08/08 15:33:52.847 Executor task launch worker for task 4 DEBUG GenerateUnsafeProjection: code for newInstance(class org.apache.spark.ml.linalg.VectorUDT).serialize: /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificUnsafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificUnsafeProjection extends org.apache.spark.sql.catalyst.expressions.UnsafeProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private boolean resultIsNull_0; /* 009 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[] mutableStateArray_1 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[2]; /* 010 */ private org.apache.spark.ml.linalg.Vector[] mutableStateArray_0 = new org.apache.spark.ml.linalg.Vector[1]; /* 011 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[] mutableStateArray_2 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[2]; /* 012 */ /* 013 */ public SpecificUnsafeProjection(Object[] references) { /* 014 */ this.references = references; /* 015 */ /* 016 */ mutableStateArray_1[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(1, 32); /* 017 */ mutableStateArray_1[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(mutableStateArray_1[0], 4); /* 018 */ mutableStateArray_2[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(mutableStateArray_1[1], 4); /* 019 */ mutableStateArray_2[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(mutableStateArray_1[1], 8); /* 020 */ /* 021 */ } /* 022 */ /* 023 */ public void initialize(int partitionIndex) { /* 024 */ /* 025 */ } /* 026 */ /* 027 */ // Scala.Function1 need this /* 028 */ public java.lang.Object apply(java.lang.Object row) { /* 029 */ return apply((InternalRow) row); /* 030 */ } /* 031 */ /* 032 */ public UnsafeRow apply(InternalRow i) { /* 033 */ mutableStateArray_1[0].reset(); /* 034 */ /* 035 */ /* 036 */ mutableStateArray_1[0].zeroOutNullBytes(); /* 037 */ /* 038 */ final org.apache.spark.ml.linalg.VectorUDT value_1 = false ? /* 039 */ null : new org.apache.spark.ml.linalg.VectorUDT(); /* 040 */ boolean isNull_0 = true; /* 041 */ InternalRow value_0 = null; /* 042 */ if (!false) { /* 043 */ resultIsNull_0 = false; /* 044 */ if (!resultIsNull_0) { /* 045 */ boolean isNull_2 = i.isNullAt(0); /* 046 */ org.apache.spark.ml.linalg.Vector value_2 = isNull_2 ? /* 047 */ null : ((org.apache.spark.ml.linalg.Vector)i.get(0, null)); /* 048 */ resultIsNull_0 = isNull_2; /* 049 */ mutableStateArray_0[0] = value_2; /* 050 */ } /* 051 */ /* 052 */ isNull_0 = resultIsNull_0; /* 053 */ if (!isNull_0) { /* 054 */ /* 055 */ Object funcResult_0 = null; /* 056 */ funcResult_0 = value_1.serialize(mutableStateArray_0[0]); /* 057 */ /* 058 */ if (funcResult_0 != null) { /* 059 */ value_0 = (InternalRow) funcResult_0; /* 060 */ } else { /* 061 */ isNull_0 = true; /* 062 */ } /* 063 */ /* 064 */ /* 065 */ } /* 066 */ } /* 067 */ if (isNull_0) { /* 068 */ mutableStateArray_1[0].setNullAt(0); /* 069 */ } else { /* 070 */ final InternalRow tmpInput_0 = value_0; /* 071 */ if (tmpInput_0 instanceof UnsafeRow) { /* 072 */ mutableStateArray_1[0].write(0, (UnsafeRow) tmpInput_0); /* 073 */ } else { /* 074 */ // Remember the current cursor so that we can calculate how many bytes are /* 075 */ // written later. /* 076 */ final int previousCursor_0 = mutableStateArray_1[0].cursor(); /* 077 */ /* 078 */ mutableStateArray_1[1].resetRowWriter(); /* 079 */ /* 080 */ /* 081 */ mutableStateArray_1[1].write(0, (tmpInput_0.getByte(0))); /* 082 */ /* 083 */ /* 084 */ if ((tmpInput_0.isNullAt(1))) { /* 085 */ mutableStateArray_1[1].setNullAt(1); /* 086 */ } else { /* 087 */ mutableStateArray_1[1].write(1, (tmpInput_0.getInt(1))); /* 088 */ } /* 089 */ /* 090 */ /* 091 */ if ((tmpInput_0.isNullAt(2))) { /* 092 */ mutableStateArray_1[1].setNullAt(2); /* 093 */ } else { /* 094 */ // Remember the current cursor so that we can calculate how many bytes are /* 095 */ // written later. /* 096 */ final int previousCursor_1 = mutableStateArray_1[1].cursor(); /* 097 */ /* 098 */ final ArrayData tmpInput_1 = (tmpInput_0.getArray(2)); /* 099 */ if (tmpInput_1 instanceof UnsafeArrayData) { /* 100 */ mutableStateArray_1[1].write((UnsafeArrayData) tmpInput_1); /* 101 */ } else { /* 102 */ final int numElements_0 = tmpInput_1.numElements(); /* 103 */ mutableStateArray_2[0].initialize(numElements_0); /* 104 */ /* 105 */ for (int index_0 = 0; index_0 < numElements_0; index_0++) { /* 106 */ mutableStateArray_2[0].write(index_0, tmpInput_1.getInt(index_0)); /* 107 */ } /* 108 */ } /* 109 */ /* 110 */ mutableStateArray_1[1].setOffsetAndSizeFromPreviousCursor(2, previousCursor_1); /* 111 */ } /* 112 */ /* 113 */ /* 114 */ if ((tmpInput_0.isNullAt(3))) { /* 115 */ mutableStateArray_1[1].setNullAt(3); /* 116 */ } else { /* 117 */ // Remember the current cursor so that we can calculate how many bytes are /* 118 */ // written later. /* 119 */ final int previousCursor_2 = mutableStateArray_1[1].cursor(); /* 120 */ /* 121 */ final ArrayData tmpInput_2 = (tmpInput_0.getArray(3)); /* 122 */ if (tmpInput_2 instanceof UnsafeArrayData) { /* 123 */ mutableStateArray_1[1].write((UnsafeArrayData) tmpInput_2); /* 124 */ } else { /* 125 */ final int numElements_1 = tmpInput_2.numElements(); /* 126 */ mutableStateArray_2[1].initialize(numElements_1); /* 127 */ /* 128 */ for (int index_1 = 0; index_1 < numElements_1; index_1++) { /* 129 */ mutableStateArray_2[1].write(index_1, tmpInput_2.getDouble(index_1)); /* 130 */ } /* 131 */ } /* 132 */ /* 133 */ mutableStateArray_1[1].setOffsetAndSizeFromPreviousCursor(3, previousCursor_2); /* 134 */ } /* 135 */ /* 136 */ /* 137 */ mutableStateArray_1[0].setOffsetAndSizeFromPreviousCursor(0, previousCursor_0); /* 138 */ } /* 139 */ } /* 140 */ return (mutableStateArray_1[0].getRow()); /* 141 */ } /* 142 */ /* 143 */ /* 144 */ } 20/08/08 15:33:52.849 Thread-4 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$runJob$5) is now cleaned +++ 20/08/08 15:33:52.849 Thread-4 INFO SparkContext: Starting job: count at ALS.scala:967 20/08/08 15:33:52.849 Executor task launch worker for task 4 DEBUG CodeGenerator: /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificUnsafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificUnsafeProjection extends org.apache.spark.sql.catalyst.expressions.UnsafeProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private boolean resultIsNull_0; /* 009 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[] mutableStateArray_1 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[2]; /* 010 */ private org.apache.spark.ml.linalg.Vector[] mutableStateArray_0 = new org.apache.spark.ml.linalg.Vector[1]; /* 011 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[] mutableStateArray_2 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[2]; /* 012 */ /* 013 */ public SpecificUnsafeProjection(Object[] references) { /* 014 */ this.references = references; /* 015 */ /* 016 */ mutableStateArray_1[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(1, 32); /* 017 */ mutableStateArray_1[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(mutableStateArray_1[0], 4); /* 018 */ mutableStateArray_2[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(mutableStateArray_1[1], 4); /* 019 */ mutableStateArray_2[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(mutableStateArray_1[1], 8); /* 020 */ /* 021 */ } /* 022 */ /* 023 */ public void initialize(int partitionIndex) { /* 024 */ /* 025 */ } /* 026 */ /* 027 */ // Scala.Function1 need this /* 028 */ public java.lang.Object apply(java.lang.Object row) { /* 029 */ return apply((InternalRow) row); /* 030 */ } /* 031 */ /* 032 */ public UnsafeRow apply(InternalRow i) { /* 033 */ mutableStateArray_1[0].reset(); /* 034 */ /* 035 */ /* 036 */ mutableStateArray_1[0].zeroOutNullBytes(); /* 037 */ /* 038 */ final org.apache.spark.ml.linalg.VectorUDT value_1 = false ? /* 039 */ null : new org.apache.spark.ml.linalg.VectorUDT(); /* 040 */ boolean isNull_0 = true; /* 041 */ InternalRow value_0 = null; /* 042 */ if (!false) { /* 043 */ resultIsNull_0 = false; /* 044 */ if (!resultIsNull_0) { /* 045 */ boolean isNull_2 = i.isNullAt(0); /* 046 */ org.apache.spark.ml.linalg.Vector value_2 = isNull_2 ? /* 047 */ null : ((org.apache.spark.ml.linalg.Vector)i.get(0, null)); /* 048 */ resultIsNull_0 = isNull_2; /* 049 */ mutableStateArray_0[0] = value_2; /* 050 */ } /* 051 */ /* 052 */ isNull_0 = resultIsNull_0; /* 053 */ if (!isNull_0) { /* 054 */ /* 055 */ Object funcResult_0 = null; /* 056 */ funcResult_0 = value_1.serialize(mutableStateArray_0[0]); /* 057 */ /* 058 */ if (funcResult_0 != null) { /* 059 */ value_0 = (InternalRow) funcResult_0; /* 060 */ } else { /* 061 */ isNull_0 = true; /* 062 */ } /* 063 */ /* 064 */ /* 065 */ } /* 066 */ } /* 067 */ if (isNull_0) { /* 068 */ mutableStateArray_1[0].setNullAt(0); /* 069 */ } else { /* 070 */ final InternalRow tmpInput_0 = value_0; /* 071 */ if (tmpInput_0 instanceof UnsafeRow) { /* 072 */ mutableStateArray_1[0].write(0, (UnsafeRow) tmpInput_0); /* 073 */ } else { /* 074 */ // Remember the current cursor so that we can calculate how many bytes are /* 075 */ // written later. /* 076 */ final int previousCursor_0 = mutableStateArray_1[0].cursor(); /* 077 */ /* 078 */ mutableStateArray_1[1].resetRowWriter(); /* 079 */ /* 080 */ /* 081 */ mutableStateArray_1[1].write(0, (tmpInput_0.getByte(0))); /* 082 */ /* 083 */ /* 084 */ if ((tmpInput_0.isNullAt(1))) { /* 085 */ mutableStateArray_1[1].setNullAt(1); /* 086 */ } else { /* 087 */ mutableStateArray_1[1].write(1, (tmpInput_0.getInt(1))); /* 088 */ } /* 089 */ /* 090 */ /* 091 */ if ((tmpInput_0.isNullAt(2))) { /* 092 */ mutableStateArray_1[1].setNullAt(2); /* 093 */ } else { /* 094 */ // Remember the current cursor so that we can calculate how many bytes are /* 095 */ // written later. /* 096 */ final int previousCursor_1 = mutableStateArray_1[1].cursor(); /* 097 */ /* 098 */ final ArrayData tmpInput_1 = (tmpInput_0.getArray(2)); /* 099 */ if (tmpInput_1 instanceof UnsafeArrayData) { /* 100 */ mutableStateArray_1[1].write((UnsafeArrayData) tmpInput_1); /* 101 */ } else { /* 102 */ final int numElements_0 = tmpInput_1.numElements(); /* 103 */ mutableStateArray_2[0].initialize(numElements_0); /* 104 */ /* 105 */ for (int index_0 = 0; index_0 < numElements_0; index_0++) { /* 106 */ mutableStateArray_2[0].write(index_0, tmpInput_1.getInt(index_0)); /* 107 */ } /* 108 */ } /* 109 */ /* 110 */ mutableStateArray_1[1].setOffsetAndSizeFromPreviousCursor(2, previousCursor_1); /* 111 */ } /* 112 */ /* 113 */ /* 114 */ if ((tmpInput_0.isNullAt(3))) { /* 115 */ mutableStateArray_1[1].setNullAt(3); /* 116 */ } else { /* 117 */ // Remember the current cursor so that we can calculate how many bytes are /* 118 */ // written later. /* 119 */ final int previousCursor_2 = mutableStateArray_1[1].cursor(); /* 120 */ /* 121 */ final ArrayData tmpInput_2 = (tmpInput_0.getArray(3)); /* 122 */ if (tmpInput_2 instanceof UnsafeArrayData) { /* 123 */ mutableStateArray_1[1].write((UnsafeArrayData) tmpInput_2); /* 124 */ } else { /* 125 */ final int numElements_1 = tmpInput_2.numElements(); /* 126 */ mutableStateArray_2[1].initialize(numElements_1); /* 127 */ /* 128 */ for (int index_1 = 0; index_1 < numElements_1; index_1++) { /* 129 */ mutableStateArray_2[1].write(index_1, tmpInput_2.getDouble(index_1)); /* 130 */ } /* 131 */ } /* 132 */ /* 133 */ mutableStateArray_1[1].setOffsetAndSizeFromPreviousCursor(3, previousCursor_2); /* 134 */ } /* 135 */ /* 136 */ /* 137 */ mutableStateArray_1[0].setOffsetAndSizeFromPreviousCursor(0, previousCursor_0); /* 138 */ } /* 139 */ } /* 140 */ return (mutableStateArray_1[0].getRow()); /* 141 */ } /* 142 */ /* 143 */ /* 144 */ } 20/08/08 15:33:52.850 dag-scheduler-event-loop DEBUG DAGScheduler: Merging stage rdd profiles: Set() 20/08/08 15:33:52.850 dag-scheduler-event-loop DEBUG DAGScheduler: Merging stage rdd profiles: Set() 20/08/08 15:33:52.851 dag-scheduler-event-loop DEBUG DAGScheduler: Merging stage rdd profiles: Set() 20/08/08 15:33:52.852 dag-scheduler-event-loop INFO DAGScheduler: Registering RDD 22 (map at ALS.scala:1599) as input to shuffle 2 20/08/08 15:33:52.852 dag-scheduler-event-loop INFO DAGScheduler: Got job 2 (count at ALS.scala:967) with 10 output partitions 20/08/08 15:33:52.852 dag-scheduler-event-loop INFO DAGScheduler: Final stage: ResultStage 6 (count at ALS.scala:967) 20/08/08 15:33:52.852 dag-scheduler-event-loop INFO DAGScheduler: Parents of final stage: List(ShuffleMapStage 5) 20/08/08 15:33:52.853 dag-scheduler-event-loop INFO DAGScheduler: Missing parents: List(ShuffleMapStage 5) 20/08/08 15:33:52.854 dag-scheduler-event-loop DEBUG DAGScheduler: submitStage(ResultStage 6 (name=count at ALS.scala:967;jobs=2)) 20/08/08 15:33:52.854 dag-scheduler-event-loop DEBUG DAGScheduler: missing: List(ShuffleMapStage 5) 20/08/08 15:33:52.854 dag-scheduler-event-loop DEBUG DAGScheduler: submitStage(ShuffleMapStage 5 (name=map at ALS.scala:1599;jobs=2)) 20/08/08 15:33:52.858 dag-scheduler-event-loop DEBUG DAGScheduler: missing: List() 20/08/08 15:33:52.858 dag-scheduler-event-loop INFO DAGScheduler: Submitting ShuffleMapStage 5 (MapPartitionsRDD[22] at map at ALS.scala:1599), which has no missing parents 20/08/08 15:33:52.858 dag-scheduler-event-loop DEBUG DAGScheduler: submitMissingTasks(ShuffleMapStage 5) 20/08/08 15:33:52.863 Thread-4 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$pythonToJava$1 20/08/08 15:33:52.863 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_4 stored as values in memory (estimated size 32.5 KiB, free 366.1 MiB) 20/08/08 15:33:52.863 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_4 locally took 1 ms 20/08/08 15:33:52.863 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_4 without replication took 1 ms 20/08/08 15:33:52.863 Thread-4 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$pythonToJava$1) is now cleaned +++ 20/08/08 15:33:52.865 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_4_piece0 stored as bytes in memory (estimated size 15.4 KiB, free 366.1 MiB) 20/08/08 15:33:52.865 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master broadcast_4_piece0 for BlockManagerId(driver, 192.168.1.225, 38159, None) 20/08/08 15:33:52.865 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added broadcast_4_piece0 in memory on 192.168.1.225:38159 (size: 15.4 KiB, free: 366.2 MiB) 20/08/08 15:33:52.866 dag-scheduler-event-loop DEBUG BlockManagerMaster: Updated info of block broadcast_4_piece0 20/08/08 15:33:52.866 dag-scheduler-event-loop DEBUG BlockManager: Told master about block broadcast_4_piece0 20/08/08 15:33:52.866 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_4_piece0 locally took 1 ms 20/08/08 15:33:52.866 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_4_piece0 without replication took 1 ms 20/08/08 15:33:52.866 dag-scheduler-event-loop INFO SparkContext: Created broadcast 4 from broadcast at DAGScheduler.scala:1348 20/08/08 15:33:52.867 dag-scheduler-event-loop INFO DAGScheduler: Submitting 4 missing tasks from ShuffleMapStage 5 (MapPartitionsRDD[22] at map at ALS.scala:1599) (first 15 tasks are for partitions Vector(0, 1, 2, 3)) 20/08/08 15:33:52.867 dag-scheduler-event-loop INFO TaskSchedulerImpl: Adding task set 5.0 with 4 tasks resource profile 0 20/08/08 15:33:52.868 dag-scheduler-event-loop DEBUG TaskSetManager: Epoch for TaskSet 5.0: 2 20/08/08 15:33:52.858 Executor task launch worker for task 3 DEBUG GenerateUnsafeProjection: code for newInstance(class org.apache.spark.ml.linalg.VectorUDT).serialize: /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificUnsafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificUnsafeProjection extends org.apache.spark.sql.catalyst.expressions.UnsafeProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private boolean resultIsNull_0; /* 009 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[] mutableStateArray_1 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[2]; /* 010 */ private org.apache.spark.ml.linalg.Vector[] mutableStateArray_0 = new org.apache.spark.ml.linalg.Vector[1]; /* 011 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[] mutableStateArray_2 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[2]; /* 012 */ /* 013 */ public SpecificUnsafeProjection(Object[] references) { /* 014 */ this.references = references; /* 015 */ /* 016 */ mutableStateArray_1[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(1, 32); /* 017 */ mutableStateArray_1[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(mutableStateArray_1[0], 4); /* 018 */ mutableStateArray_2[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(mutableStateArray_1[1], 4); /* 019 */ mutableStateArray_2[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(mutableStateArray_1[1], 8); /* 020 */ /* 021 */ } /* 022 */ /* 023 */ public void initialize(int partitionIndex) { /* 024 */ /* 025 */ } /* 026 */ /* 027 */ // Scala.Function1 need this /* 028 */ public java.lang.Object apply(java.lang.Object row) { /* 029 */ return apply((InternalRow) row); /* 030 */ } /* 031 */ /* 032 */ public UnsafeRow apply(InternalRow i) { /* 033 */ mutableStateArray_1[0].reset(); /* 034 */ /* 035 */ /* 036 */ mutableStateArray_1[0].zeroOutNullBytes(); /* 037 */ /* 038 */ final org.apache.spark.ml.linalg.VectorUDT value_1 = false ? /* 039 */ null : new org.apache.spark.ml.linalg.VectorUDT(); /* 040 */ boolean isNull_0 = true; /* 041 */ InternalRow value_0 = null; /* 042 */ if (!false) { /* 043 */ resultIsNull_0 = false; /* 044 */ if (!resultIsNull_0) { /* 045 */ boolean isNull_2 = i.isNullAt(0); /* 046 */ org.apache.spark.ml.linalg.Vector value_2 = isNull_2 ? /* 047 */ null : ((org.apache.spark.ml.linalg.Vector)i.get(0, null)); /* 048 */ resultIsNull_0 = isNull_2; /* 049 */ mutableStateArray_0[0] = value_2; /* 050 */ } /* 051 */ /* 052 */ isNull_0 = resultIsNull_0; /* 053 */ if (!isNull_0) { /* 054 */ /* 055 */ Object funcResult_0 = null; /* 056 */ funcResult_0 = value_1.serialize(mutableStateArray_0[0]); /* 057 */ /* 058 */ if (funcResult_0 != null) { /* 059 */ value_0 = (InternalRow) funcResult_0; /* 060 */ } else { /* 061 */ isNull_0 = true; /* 062 */ } /* 063 */ /* 064 */ /* 065 */ } /* 066 */ } /* 067 */ if (isNull_0) { /* 068 */ mutableStateArray_1[0].setNullAt(0); /* 069 */ } else { /* 070 */ final InternalRow tmpInput_0 = value_0; /* 071 */ if (tmpInput_0 instanceof UnsafeRow) { /* 072 */ mutableStateArray_1[0].write(0, (UnsafeRow) tmpInput_0); /* 073 */ } else { /* 074 */ // Remember the current cursor so that we can calculate how many bytes are /* 075 */ // written later. /* 076 */ final int previousCursor_0 = mutableStateArray_1[0].cursor(); /* 077 */ /* 078 */ mutableStateArray_1[1].resetRowWriter(); /* 079 */ /* 080 */ /* 081 */ mutableStateArray_1[1].write(0, (tmpInput_0.getByte(0))); /* 082 */ /* 083 */ /* 084 */ if ((tmpInput_0.isNullAt(1))) { /* 085 */ mutableStateArray_1[1].setNullAt(1); /* 086 */ } else { /* 087 */ mutableStateArray_1[1].write(1, (tmpInput_0.getInt(1))); /* 088 */ } /* 089 */ /* 090 */ /* 091 */ if ((tmpInput_0.isNullAt(2))) { /* 092 */ mutableStateArray_1[1].setNullAt(2); /* 093 */ } else { /* 094 */ // Remember the current cursor so that we can calculate how many bytes are /* 095 */ // written later. /* 096 */ final int previousCursor_1 = mutableStateArray_1[1].cursor(); /* 097 */ /* 098 */ final ArrayData tmpInput_1 = (tmpInput_0.getArray(2)); /* 099 */ if (tmpInput_1 instanceof UnsafeArrayData) { /* 100 */ mutableStateArray_1[1].write((UnsafeArrayData) tmpInput_1); /* 101 */ } else { /* 102 */ final int numElements_0 = tmpInput_1.numElements(); /* 103 */ mutableStateArray_2[0].initialize(numElements_0); /* 104 */ /* 105 */ for (int index_0 = 0; index_0 < numElements_0; index_0++) { /* 106 */ mutableStateArray_2[0].write(index_0, tmpInput_1.getInt(index_0)); /* 107 */ } /* 108 */ } /* 109 */ /* 110 */ mutableStateArray_1[1].setOffsetAndSizeFromPreviousCursor(2, previousCursor_1); /* 111 */ } /* 112 */ /* 113 */ /* 114 */ if ((tmpInput_0.isNullAt(3))) { /* 115 */ mutableStateArray_1[1].setNullAt(3); /* 116 */ } else { /* 117 */ // Remember the current cursor so that we can calculate how many bytes are /* 118 */ // written later. /* 119 */ final int previousCursor_2 = mutableStateArray_1[1].cursor(); /* 120 */ /* 121 */ final ArrayData tmpInput_2 = (tmpInput_0.getArray(3)); /* 122 */ if (tmpInput_2 instanceof UnsafeArrayData) { /* 123 */ mutableStateArray_1[1].write((UnsafeArrayData) tmpInput_2); /* 124 */ } else { /* 125 */ final int numElements_1 = tmpInput_2.numElements(); /* 126 */ mutableStateArray_2[1].initialize(numElements_1); /* 127 */ /* 128 */ for (int index_1 = 0; index_1 < numElements_1; index_1++) { /* 129 */ mutableStateArray_2[1].write(index_1, tmpInput_2.getDouble(index_1)); /* 130 */ } /* 131 */ } /* 132 */ /* 133 */ mutableStateArray_1[1].setOffsetAndSizeFromPreviousCursor(3, previousCursor_2); /* 134 */ } /* 135 */ /* 136 */ /* 137 */ mutableStateArray_1[0].setOffsetAndSizeFromPreviousCursor(0, previousCursor_0); /* 138 */ } /* 139 */ } /* 140 */ return (mutableStateArray_1[0].getRow()); /* 141 */ } /* 142 */ /* 143 */ /* 144 */ } 20/08/08 15:33:52.868 dag-scheduler-event-loop DEBUG TaskSetManager: Adding pending tasks took 0 ms 20/08/08 15:33:52.869 dag-scheduler-event-loop DEBUG TaskSetManager: Valid locality levels for TaskSet 5.0: PROCESS_LOCAL, NODE_LOCAL, ANY 20/08/08 15:33:52.870 Thread-4 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$toJavaArray$1 20/08/08 15:33:52.870 dispatcher-event-loop-3 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_5.0, runningTasks: 0 20/08/08 15:33:52.870 dispatcher-event-loop-3 INFO TaskSetManager: Starting task 0.0 in stage 5.0 (TID 19, 192.168.1.225, executor driver, partition 0, PROCESS_LOCAL, 7132 bytes) taskResourceAssignments Map() 20/08/08 15:33:52.870 Thread-4 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$toJavaArray$1) is now cleaned +++ 20/08/08 15:33:52.870 dispatcher-event-loop-3 INFO TaskSetManager: Starting task 1.0 in stage 5.0 (TID 20, 192.168.1.225, executor driver, partition 1, PROCESS_LOCAL, 7132 bytes) taskResourceAssignments Map() 20/08/08 15:33:52.871 dispatcher-event-loop-3 INFO TaskSetManager: Starting task 2.0 in stage 5.0 (TID 21, 192.168.1.225, executor driver, partition 2, PROCESS_LOCAL, 7132 bytes) taskResourceAssignments Map() 20/08/08 15:33:52.871 dispatcher-event-loop-3 INFO TaskSetManager: Starting task 3.0 in stage 5.0 (TID 22, 192.168.1.225, executor driver, partition 3, PROCESS_LOCAL, 7132 bytes) taskResourceAssignments Map() 20/08/08 15:33:52.872 Executor task launch worker for task 21 INFO Executor: Running task 2.0 in stage 5.0 (TID 21) 20/08/08 15:33:52.872 Executor task launch worker for task 22 INFO Executor: Running task 3.0 in stage 5.0 (TID 22) 20/08/08 15:33:52.872 Executor task launch worker for task 20 INFO Executor: Running task 1.0 in stage 5.0 (TID 20) 20/08/08 15:33:52.872 Executor task launch worker for task 19 INFO Executor: Running task 0.0 in stage 5.0 (TID 19) 20/08/08 15:33:52.874 Thread-4 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$applySchemaToPythonRDD$1 20/08/08 15:33:52.875 Thread-4 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$applySchemaToPythonRDD$1) is now cleaned +++ 20/08/08 15:33:52.872 Executor task launch worker for task 22 DEBUG ExecutorMetricsPoller: stageTCMP: (5, 0) -> 1 20/08/08 15:33:52.883 Executor task launch worker for task 22 DEBUG BlockManager: Getting local block broadcast_4 20/08/08 15:33:52.883 Executor task launch worker for task 22 DEBUG BlockManager: Level for block broadcast_4 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:52.882 Executor task launch worker for task 21 DEBUG ExecutorMetricsPoller: stageTCMP: (5, 0) -> 4 20/08/08 15:33:52.878 Executor task launch worker for task 1 DEBUG GenerateUnsafeProjection: code for newInstance(class org.apache.spark.ml.linalg.VectorUDT).serialize: /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificUnsafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificUnsafeProjection extends org.apache.spark.sql.catalyst.expressions.UnsafeProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private boolean resultIsNull_0; /* 009 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[] mutableStateArray_1 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[2]; /* 010 */ private org.apache.spark.ml.linalg.Vector[] mutableStateArray_0 = new org.apache.spark.ml.linalg.Vector[1]; /* 011 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[] mutableStateArray_2 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[2]; /* 012 */ /* 013 */ public SpecificUnsafeProjection(Object[] references) { /* 014 */ this.references = references; /* 015 */ /* 016 */ mutableStateArray_1[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(1, 32); /* 017 */ mutableStateArray_1[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(mutableStateArray_1[0], 4); /* 018 */ mutableStateArray_2[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(mutableStateArray_1[1], 4); /* 019 */ mutableStateArray_2[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(mutableStateArray_1[1], 8); /* 020 */ /* 021 */ } /* 022 */ /* 023 */ public void initialize(int partitionIndex) { /* 024 */ /* 025 */ } /* 026 */ /* 027 */ // Scala.Function1 need this /* 028 */ public java.lang.Object apply(java.lang.Object row) { /* 029 */ return apply((InternalRow) row); /* 030 */ } /* 031 */ /* 032 */ public UnsafeRow apply(InternalRow i) { /* 033 */ mutableStateArray_1[0].reset(); /* 034 */ /* 035 */ /* 036 */ mutableStateArray_1[0].zeroOutNullBytes(); /* 037 */ /* 038 */ final org.apache.spark.ml.linalg.VectorUDT value_1 = false ? /* 039 */ null : new org.apache.spark.ml.linalg.VectorUDT(); /* 040 */ boolean isNull_0 = true; /* 041 */ InternalRow value_0 = null; /* 042 */ if (!false) { /* 043 */ resultIsNull_0 = false; /* 044 */ if (!resultIsNull_0) { /* 045 */ boolean isNull_2 = i.isNullAt(0); /* 046 */ org.apache.spark.ml.linalg.Vector value_2 = isNull_2 ? /* 047 */ null : ((org.apache.spark.ml.linalg.Vector)i.get(0, null)); /* 048 */ resultIsNull_0 = isNull_2; /* 049 */ mutableStateArray_0[0] = value_2; /* 050 */ } /* 051 */ /* 052 */ isNull_0 = resultIsNull_0; /* 053 */ if (!isNull_0) { /* 054 */ /* 055 */ Object funcResult_0 = null; /* 056 */ funcResult_0 = value_1.serialize(mutableStateArray_0[0]); /* 057 */ /* 058 */ if (funcResult_0 != null) { /* 059 */ value_0 = (InternalRow) funcResult_0; /* 060 */ } else { /* 061 */ isNull_0 = true; /* 062 */ } /* 063 */ /* 064 */ /* 065 */ } /* 066 */ } /* 067 */ if (isNull_0) { /* 068 */ mutableStateArray_1[0].setNullAt(0); /* 069 */ } else { /* 070 */ final InternalRow tmpInput_0 = value_0; /* 071 */ if (tmpInput_0 instanceof UnsafeRow) { /* 072 */ mutableStateArray_1[0].write(0, (UnsafeRow) tmpInput_0); /* 073 */ } else { /* 074 */ // Remember the current cursor so that we can calculate how many bytes are /* 075 */ // written later. /* 076 */ final int previousCursor_0 = mutableStateArray_1[0].cursor(); /* 077 */ /* 078 */ mutableStateArray_1[1].resetRowWriter(); /* 079 */ /* 080 */ /* 081 */ mutableStateArray_1[1].write(0, (tmpInput_0.getByte(0))); /* 082 */ /* 083 */ /* 084 */ if ((tmpInput_0.isNullAt(1))) { /* 085 */ mutableStateArray_1[1].setNullAt(1); /* 086 */ } else { /* 087 */ mutableStateArray_1[1].write(1, (tmpInput_0.getInt(1))); /* 088 */ } /* 089 */ /* 090 */ /* 091 */ if ((tmpInput_0.isNullAt(2))) { /* 092 */ mutableStateArray_1[1].setNullAt(2); /* 093 */ } else { /* 094 */ // Remember the current cursor so that we can calculate how many bytes are /* 095 */ // written later. /* 096 */ final int previousCursor_1 = mutableStateArray_1[1].cursor(); /* 097 */ /* 098 */ final ArrayData tmpInput_1 = (tmpInput_0.getArray(2)); /* 099 */ if (tmpInput_1 instanceof UnsafeArrayData) { /* 100 */ mutableStateArray_1[1].write((UnsafeArrayData) tmpInput_1); /* 101 */ } else { /* 102 */ final int numElements_0 = tmpInput_1.numElements(); /* 103 */ mutableStateArray_2[0].initialize(numElements_0); /* 104 */ /* 105 */ for (int index_0 = 0; index_0 < numElements_0; index_0++) { /* 106 */ mutableStateArray_2[0].write(index_0, tmpInput_1.getInt(index_0)); /* 107 */ } /* 108 */ } /* 109 */ /* 110 */ mutableStateArray_1[1].setOffsetAndSizeFromPreviousCursor(2, previousCursor_1); /* 111 */ } /* 112 */ /* 113 */ /* 114 */ if ((tmpInput_0.isNullAt(3))) { /* 115 */ mutableStateArray_1[1].setNullAt(3); /* 116 */ } else { /* 117 */ // Remember the current cursor so that we can calculate how many bytes are /* 118 */ // written later. /* 119 */ final int previousCursor_2 = mutableStateArray_1[1].cursor(); /* 120 */ /* 121 */ final ArrayData tmpInput_2 = (tmpInput_0.getArray(3)); /* 122 */ if (tmpInput_2 instanceof UnsafeArrayData) { /* 123 */ mutableStateArray_1[1].write((UnsafeArrayData) tmpInput_2); /* 124 */ } else { /* 125 */ final int numElements_1 = tmpInput_2.numElements(); /* 126 */ mutableStateArray_2[1].initialize(numElements_1); /* 127 */ /* 128 */ for (int index_1 = 0; index_1 < numElements_1; index_1++) { /* 129 */ mutableStateArray_2[1].write(index_1, tmpInput_2.getDouble(index_1)); /* 130 */ } /* 131 */ } /* 132 */ /* 133 */ mutableStateArray_1[1].setOffsetAndSizeFromPreviousCursor(3, previousCursor_2); /* 134 */ } /* 135 */ /* 136 */ /* 137 */ mutableStateArray_1[0].setOffsetAndSizeFromPreviousCursor(0, previousCursor_0); /* 138 */ } /* 139 */ } /* 140 */ return (mutableStateArray_1[0].getRow()); /* 141 */ } /* 142 */ /* 143 */ /* 144 */ } 20/08/08 15:33:52.881 Executor task launch worker for task 19 DEBUG ExecutorMetricsPoller: stageTCMP: (5, 0) -> 3 20/08/08 15:33:52.892 Executor task launch worker for task 22 DEBUG BlockManager: Getting local block rdd_16_3 20/08/08 15:33:52.892 Executor task launch worker for task 22 DEBUG BlockManager: Level for block rdd_16_3 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:52.892 Executor task launch worker for task 22 INFO BlockManager: Found block rdd_16_3 locally 20/08/08 15:33:52.895 Executor task launch worker for task 22 DEBUG ALS: Converting to local indices took 0.001340588 seconds. 20/08/08 15:33:52.881 Executor task launch worker for task 20 DEBUG ExecutorMetricsPoller: stageTCMP: (5, 0) -> 2 20/08/08 15:33:52.897 Executor task launch worker for task 22 DEBUG ALS: Converting to local indices took 0.001428949 seconds. 20/08/08 15:33:52.879 Executor task launch worker for task 2 DEBUG GenerateUnsafeProjection: code for newInstance(class org.apache.spark.ml.linalg.VectorUDT).serialize: /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificUnsafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificUnsafeProjection extends org.apache.spark.sql.catalyst.expressions.UnsafeProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private boolean resultIsNull_0; /* 009 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[] mutableStateArray_1 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[2]; /* 010 */ private org.apache.spark.ml.linalg.Vector[] mutableStateArray_0 = new org.apache.spark.ml.linalg.Vector[1]; /* 011 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[] mutableStateArray_2 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[2]; /* 012 */ /* 013 */ public SpecificUnsafeProjection(Object[] references) { /* 014 */ this.references = references; /* 015 */ /* 016 */ mutableStateArray_1[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(1, 32); /* 017 */ mutableStateArray_1[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(mutableStateArray_1[0], 4); /* 018 */ mutableStateArray_2[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(mutableStateArray_1[1], 4); /* 019 */ mutableStateArray_2[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(mutableStateArray_1[1], 8); /* 020 */ /* 021 */ } /* 022 */ /* 023 */ public void initialize(int partitionIndex) { /* 024 */ /* 025 */ } /* 026 */ /* 027 */ // Scala.Function1 need this /* 028 */ public java.lang.Object apply(java.lang.Object row) { /* 029 */ return apply((InternalRow) row); /* 030 */ } /* 031 */ /* 032 */ public UnsafeRow apply(InternalRow i) { /* 033 */ mutableStateArray_1[0].reset(); /* 034 */ /* 035 */ /* 036 */ mutableStateArray_1[0].zeroOutNullBytes(); /* 037 */ /* 038 */ final org.apache.spark.ml.linalg.VectorUDT value_1 = false ? /* 039 */ null : new org.apache.spark.ml.linalg.VectorUDT(); /* 040 */ boolean isNull_0 = true; /* 041 */ InternalRow value_0 = null; /* 042 */ if (!false) { /* 043 */ resultIsNull_0 = false; /* 044 */ if (!resultIsNull_0) { /* 045 */ boolean isNull_2 = i.isNullAt(0); /* 046 */ org.apache.spark.ml.linalg.Vector value_2 = isNull_2 ? /* 047 */ null : ((org.apache.spark.ml.linalg.Vector)i.get(0, null)); /* 048 */ resultIsNull_0 = isNull_2; /* 049 */ mutableStateArray_0[0] = value_2; /* 050 */ } /* 051 */ /* 052 */ isNull_0 = resultIsNull_0; /* 053 */ if (!isNull_0) { /* 054 */ /* 055 */ Object funcResult_0 = null; /* 056 */ funcResult_0 = value_1.serialize(mutableStateArray_0[0]); /* 057 */ /* 058 */ if (funcResult_0 != null) { /* 059 */ value_0 = (InternalRow) funcResult_0; /* 060 */ } else { /* 061 */ isNull_0 = true; /* 062 */ } /* 063 */ /* 064 */ /* 065 */ } /* 066 */ } /* 067 */ if (isNull_0) { /* 068 */ mutableStateArray_1[0].setNullAt(0); /* 069 */ } else { /* 070 */ final InternalRow tmpInput_0 = value_0; /* 071 */ if (tmpInput_0 instanceof UnsafeRow) { /* 072 */ mutableStateArray_1[0].write(0, (UnsafeRow) tmpInput_0); /* 073 */ } else { /* 074 */ // Remember the current cursor so that we can calculate how many bytes are /* 075 */ // written later. /* 076 */ final int previousCursor_0 = mutableStateArray_1[0].cursor(); /* 077 */ /* 078 */ mutableStateArray_1[1].resetRowWriter(); /* 079 */ /* 080 */ /* 081 */ mutableStateArray_1[1].write(0, (tmpInput_0.getByte(0))); /* 082 */ /* 083 */ /* 084 */ if ((tmpInput_0.isNullAt(1))) { /* 085 */ mutableStateArray_1[1].setNullAt(1); /* 086 */ } else { /* 087 */ mutableStateArray_1[1].write(1, (tmpInput_0.getInt(1))); /* 088 */ } /* 089 */ /* 090 */ /* 091 */ if ((tmpInput_0.isNullAt(2))) { /* 092 */ mutableStateArray_1[1].setNullAt(2); /* 093 */ } else { /* 094 */ // Remember the current cursor so that we can calculate how many bytes are /* 095 */ // written later. /* 096 */ final int previousCursor_1 = mutableStateArray_1[1].cursor(); /* 097 */ /* 098 */ final ArrayData tmpInput_1 = (tmpInput_0.getArray(2)); /* 099 */ if (tmpInput_1 instanceof UnsafeArrayData) { /* 100 */ mutableStateArray_1[1].write((UnsafeArrayData) tmpInput_1); /* 101 */ } else { /* 102 */ final int numElements_0 = tmpInput_1.numElements(); /* 103 */ mutableStateArray_2[0].initialize(numElements_0); /* 104 */ /* 105 */ for (int index_0 = 0; index_0 < numElements_0; index_0++) { /* 106 */ mutableStateArray_2[0].write(index_0, tmpInput_1.getInt(index_0)); /* 107 */ } /* 108 */ } /* 109 */ /* 110 */ mutableStateArray_1[1].setOffsetAndSizeFromPreviousCursor(2, previousCursor_1); /* 111 */ } /* 112 */ /* 113 */ /* 114 */ if ((tmpInput_0.isNullAt(3))) { /* 115 */ mutableStateArray_1[1].setNullAt(3); /* 116 */ } else { /* 117 */ // Remember the current cursor so that we can calculate how many bytes are /* 118 */ // written later. /* 119 */ final int previousCursor_2 = mutableStateArray_1[1].cursor(); /* 120 */ /* 121 */ final ArrayData tmpInput_2 = (tmpInput_0.getArray(3)); /* 122 */ if (tmpInput_2 instanceof UnsafeArrayData) { /* 123 */ mutableStateArray_1[1].write((UnsafeArrayData) tmpInput_2); /* 124 */ } else { /* 125 */ final int numElements_1 = tmpInput_2.numElements(); /* 126 */ mutableStateArray_2[1].initialize(numElements_1); /* 127 */ /* 128 */ for (int index_1 = 0; index_1 < numElements_1; index_1++) { /* 129 */ mutableStateArray_2[1].write(index_1, tmpInput_2.getDouble(index_1)); /* 130 */ } /* 131 */ } /* 132 */ /* 133 */ mutableStateArray_1[1].setOffsetAndSizeFromPreviousCursor(3, previousCursor_2); /* 134 */ } /* 135 */ /* 136 */ /* 137 */ mutableStateArray_1[0].setOffsetAndSizeFromPreviousCursor(0, previousCursor_0); /* 138 */ } /* 139 */ } /* 140 */ return (mutableStateArray_1[0].getRow()); /* 141 */ } /* 142 */ /* 143 */ /* 144 */ } 20/08/08 15:33:52.907 Executor task launch worker for task 20 DEBUG BlockManager: Getting local block rdd_16_1 20/08/08 15:33:52.907 Executor task launch worker for task 20 DEBUG BlockManager: Level for block rdd_16_1 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:52.907 Executor task launch worker for task 20 INFO BlockManager: Found block rdd_16_1 locally 20/08/08 15:33:52.910 Executor task launch worker for task 22 INFO Executor: Finished task 3.0 in stage 5.0 (TID 22). 1993 bytes result sent to driver 20/08/08 15:33:52.910 Executor task launch worker for task 19 DEBUG BlockManager: Getting local block rdd_16_0 20/08/08 15:33:52.912 Executor task launch worker for task 19 DEBUG BlockManager: Level for block rdd_16_0 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:52.912 Executor task launch worker for task 19 INFO BlockManager: Found block rdd_16_0 locally 20/08/08 15:33:52.912 dispatcher-event-loop-2 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_5.0, runningTasks: 3 20/08/08 15:33:52.913 dispatcher-event-loop-2 DEBUG TaskSetManager: No tasks for locality level PROCESS_LOCAL, so moving to locality level NODE_LOCAL 20/08/08 15:33:52.913 dispatcher-event-loop-2 DEBUG TaskSetManager: No tasks for locality level NODE_LOCAL, so moving to locality level ANY 20/08/08 15:33:52.913 Executor task launch worker for task 22 DEBUG ExecutorMetricsPoller: stageTCMP: (5, 0) -> 3 20/08/08 15:33:52.911 Executor task launch worker for task 20 DEBUG ALS: Converting to local indices took 0.002732307 seconds. 20/08/08 15:33:52.913 task-result-getter-1 INFO TaskSetManager: Finished task 3.0 in stage 5.0 (TID 22) in 42 ms on 192.168.1.225 (executor driver) (1/4) 20/08/08 15:33:52.914 dag-scheduler-event-loop DEBUG DAGScheduler: ShuffleMapTask finished on driver 20/08/08 15:33:52.914 Executor task launch worker for task 19 DEBUG ALS: Converting to local indices took 0.001372198 seconds. 20/08/08 15:33:52.910 Executor task launch worker for task 21 DEBUG BlockManager: Getting local block rdd_16_2 20/08/08 15:33:52.915 Executor task launch worker for task 21 DEBUG BlockManager: Level for block rdd_16_2 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:52.915 Executor task launch worker for task 21 INFO BlockManager: Found block rdd_16_2 locally 20/08/08 15:33:52.917 Executor task launch worker for task 21 DEBUG ALS: Converting to local indices took 0.001314948 seconds. 20/08/08 15:33:52.918 Executor task launch worker for task 21 DEBUG ALS: Converting to local indices took 0.001308928 seconds. 20/08/08 15:33:52.924 Executor task launch worker for task 19 INFO Executor: Finished task 0.0 in stage 5.0 (TID 19). 1993 bytes result sent to driver 20/08/08 15:33:52.924 Executor task launch worker for task 20 INFO Executor: Finished task 1.0 in stage 5.0 (TID 20). 1993 bytes result sent to driver 20/08/08 15:33:52.925 Executor task launch worker for task 19 DEBUG ExecutorMetricsPoller: stageTCMP: (5, 0) -> 2 20/08/08 15:33:52.925 Executor task launch worker for task 20 DEBUG ExecutorMetricsPoller: stageTCMP: (5, 0) -> 1 20/08/08 15:33:52.925 dispatcher-event-loop-1 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_5.0, runningTasks: 2 20/08/08 15:33:52.927 task-result-getter-3 INFO TaskSetManager: Finished task 0.0 in stage 5.0 (TID 19) in 57 ms on 192.168.1.225 (executor driver) (2/4) 20/08/08 15:33:52.927 dispatcher-event-loop-1 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_5.0, runningTasks: 1 20/08/08 15:33:52.928 dag-scheduler-event-loop DEBUG DAGScheduler: ShuffleMapTask finished on driver 20/08/08 15:33:52.928 task-result-getter-2 INFO TaskSetManager: Finished task 1.0 in stage 5.0 (TID 20) in 58 ms on 192.168.1.225 (executor driver) (3/4) 20/08/08 15:33:52.929 Executor task launch worker for task 21 INFO Executor: Finished task 2.0 in stage 5.0 (TID 21). 1993 bytes result sent to driver 20/08/08 15:33:52.929 dag-scheduler-event-loop DEBUG DAGScheduler: ShuffleMapTask finished on driver 20/08/08 15:33:52.929 dispatcher-event-loop-3 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_5.0, runningTasks: 0 20/08/08 15:33:52.930 Executor task launch worker for task 21 DEBUG ExecutorMetricsPoller: removing (5, 0) from stageTCMP 20/08/08 15:33:52.930 task-result-getter-0 INFO TaskSetManager: Finished task 2.0 in stage 5.0 (TID 21) in 59 ms on 192.168.1.225 (executor driver) (4/4) 20/08/08 15:33:52.930 task-result-getter-0 INFO TaskSchedulerImpl: Removed TaskSet 5.0, whose tasks have all completed, from pool 20/08/08 15:33:52.931 dag-scheduler-event-loop DEBUG DAGScheduler: ShuffleMapTask finished on driver 20/08/08 15:33:52.931 dag-scheduler-event-loop INFO DAGScheduler: ShuffleMapStage 5 (map at ALS.scala:1599) finished in 0.072 s 20/08/08 15:33:52.931 dag-scheduler-event-loop INFO DAGScheduler: looking for newly runnable stages 20/08/08 15:33:52.931 dag-scheduler-event-loop INFO DAGScheduler: running: Set() 20/08/08 15:33:52.932 dag-scheduler-event-loop INFO DAGScheduler: waiting: Set(ResultStage 6) 20/08/08 15:33:52.932 dag-scheduler-event-loop INFO DAGScheduler: failed: Set() 20/08/08 15:33:52.932 dag-scheduler-event-loop DEBUG MapOutputTrackerMaster: Increasing epoch to 3 20/08/08 15:33:52.932 dag-scheduler-event-loop DEBUG DAGScheduler: submitStage(ResultStage 6 (name=count at ALS.scala:967;jobs=2)) 20/08/08 15:33:52.934 dag-scheduler-event-loop DEBUG DAGScheduler: missing: List() 20/08/08 15:33:52.934 dag-scheduler-event-loop INFO DAGScheduler: Submitting ResultStage 6 (itemOutBlocks MapPartitionsRDD[25] at mapValues at ALS.scala:1636), which has no missing parents 20/08/08 15:33:52.934 dag-scheduler-event-loop DEBUG DAGScheduler: submitMissingTasks(ResultStage 6) 20/08/08 15:33:52.940 Thread-4 DEBUG CountVectorizer: Input schema: {"type":"struct","fields":[{"name":"id","type":"long","nullable":true,"metadata":{}},{"name":"words","type":{"type":"array","elementType":"string","containsNull":true},"nullable":true,"metadata":{}},{"name":"expected","type":{"type":"udt","class":"org.apache.spark.ml.linalg.VectorUDT","pyClass":"pyspark.ml.linalg.VectorUDT","sqlType":{"type":"struct","fields":[{"name":"type","type":"byte","nullable":false,"metadata":{}},{"name":"size","type":"integer","nullable":true,"metadata":{}},{"name":"indices","type":{"type":"array","elementType":"integer","containsNull":false},"nullable":true,"metadata":{}},{"name":"values","type":{"type":"array","elementType":"double","containsNull":false},"nullable":true,"metadata":{}}]}},"nullable":true,"metadata":{}}]} 20/08/08 15:33:52.940 Thread-4 DEBUG CountVectorizer: Expected output schema: {"type":"struct","fields":[{"name":"id","type":"long","nullable":true,"metadata":{}},{"name":"words","type":{"type":"array","elementType":"string","containsNull":true},"nullable":true,"metadata":{}},{"name":"expected","type":{"type":"udt","class":"org.apache.spark.ml.linalg.VectorUDT","pyClass":"pyspark.ml.linalg.VectorUDT","sqlType":{"type":"struct","fields":[{"name":"type","type":"byte","nullable":false,"metadata":{}},{"name":"size","type":"integer","nullable":true,"metadata":{}},{"name":"indices","type":{"type":"array","elementType":"integer","containsNull":false},"nullable":true,"metadata":{}},{"name":"values","type":{"type":"array","elementType":"double","containsNull":false},"nullable":true,"metadata":{}}]}},"nullable":true,"metadata":{}},{"name":"features","type":{"type":"udt","class":"org.apache.spark.ml.linalg.VectorUDT","pyClass":"pyspark.ml.linalg.VectorUDT","sqlType":{"type":"struct","fields":[{"name":"type","type":"byte","nullable":false,"metadata":{}},{"name":"size","type":"integer","nullable":true,"metadata":{}},{"name":"indices","type":{"type":"array","elementType":"integer","containsNull":false},"nullable":true,"metadata":{}},{"name":"values","type":{"type":"array","elementType":"double","containsNull":false},"nullable":true,"metadata":{}}]}},"nullable":false,"metadata":{}}]} 20/08/08 15:33:52.943 Thread-4 DEBUG Analyzer$ResolveReferences: Resolving 'words to words#38 20/08/08 15:33:52.945 Executor task launch worker for task 4 INFO CodeGenerator: Code generated in 98.539814 ms 20/08/08 15:33:52.951 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_5 stored as values in memory (estimated size 33.8 KiB, free 366.1 MiB) 20/08/08 15:33:52.952 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_5 locally took 9 ms 20/08/08 15:33:52.952 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_5 without replication took 9 ms 20/08/08 15:33:52.953 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_5_piece0 stored as bytes in memory (estimated size 15.8 KiB, free 366.1 MiB) 20/08/08 15:33:52.953 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master broadcast_5_piece0 for BlockManagerId(driver, 192.168.1.225, 38159, None) 20/08/08 15:33:52.954 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added broadcast_5_piece0 in memory on 192.168.1.225:38159 (size: 15.8 KiB, free: 366.2 MiB) 20/08/08 15:33:52.955 Executor task launch worker for task 1 INFO PythonRunner: Times: total = 185, boot = 100, init = 84, finish = 1 20/08/08 15:33:52.956 dag-scheduler-event-loop DEBUG BlockManagerMaster: Updated info of block broadcast_5_piece0 20/08/08 15:33:52.956 dag-scheduler-event-loop DEBUG BlockManager: Told master about block broadcast_5_piece0 20/08/08 15:33:52.956 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_5_piece0 locally took 3 ms 20/08/08 15:33:52.956 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_5_piece0 without replication took 3 ms 20/08/08 15:33:52.956 dag-scheduler-event-loop INFO SparkContext: Created broadcast 5 from broadcast at DAGScheduler.scala:1348 20/08/08 15:33:52.957 dag-scheduler-event-loop INFO DAGScheduler: Submitting 10 missing tasks from ResultStage 6 (itemOutBlocks MapPartitionsRDD[25] at mapValues at ALS.scala:1636) (first 15 tasks are for partitions Vector(0, 1, 2, 3, 4, 5, 6, 7, 8, 9)) 20/08/08 15:33:52.957 dag-scheduler-event-loop INFO TaskSchedulerImpl: Adding task set 6.0 with 10 tasks resource profile 0 20/08/08 15:33:52.958 dag-scheduler-event-loop DEBUG TaskSetManager: Epoch for TaskSet 6.0: 3 20/08/08 15:33:52.958 dag-scheduler-event-loop DEBUG TaskSetManager: Adding pending tasks took 0 ms 20/08/08 15:33:52.957 Executor task launch worker for task 3 INFO PythonRunner: Times: total = 123, boot = 20, init = 103, finish = 0 20/08/08 15:33:52.958 dag-scheduler-event-loop DEBUG TaskSetManager: Valid locality levels for TaskSet 6.0: NODE_LOCAL, NO_PREF, ANY 20/08/08 15:33:52.958 Executor task launch worker for task 2 INFO PythonRunner: Times: total = 123, boot = 5, init = 117, finish = 1 20/08/08 15:33:52.960 dispatcher-event-loop-2 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_6.0, runningTasks: 0 20/08/08 15:33:52.960 dispatcher-event-loop-2 INFO TaskSetManager: Starting task 0.0 in stage 6.0 (TID 23, 192.168.1.225, executor driver, partition 0, NODE_LOCAL, 7143 bytes) taskResourceAssignments Map() 20/08/08 15:33:52.961 dispatcher-event-loop-2 INFO TaskSetManager: Starting task 1.0 in stage 6.0 (TID 24, 192.168.1.225, executor driver, partition 1, NODE_LOCAL, 7143 bytes) taskResourceAssignments Map() 20/08/08 15:33:52.961 dispatcher-event-loop-2 INFO TaskSetManager: Starting task 2.0 in stage 6.0 (TID 25, 192.168.1.225, executor driver, partition 2, NODE_LOCAL, 7143 bytes) taskResourceAssignments Map() 20/08/08 15:33:52.961 dispatcher-event-loop-2 DEBUG TaskSetManager: No tasks for locality level NODE_LOCAL, so moving to locality level NO_PREF 20/08/08 15:33:52.961 dispatcher-event-loop-2 DEBUG TaskSetManager: Moving to ANY after waiting for 0ms 20/08/08 15:33:52.962 dispatcher-event-loop-2 INFO TaskSetManager: Starting task 3.0 in stage 6.0 (TID 26, 192.168.1.225, executor driver, partition 3, PROCESS_LOCAL, 7143 bytes) taskResourceAssignments Map() 20/08/08 15:33:52.962 Executor task launch worker for task 23 INFO Executor: Running task 0.0 in stage 6.0 (TID 23) 20/08/08 15:33:52.962 Executor task launch worker for task 26 INFO Executor: Running task 3.0 in stage 6.0 (TID 26) 20/08/08 15:33:52.962 Executor task launch worker for task 25 INFO Executor: Running task 2.0 in stage 6.0 (TID 25) 20/08/08 15:33:52.962 Executor task launch worker for task 24 INFO Executor: Running task 1.0 in stage 6.0 (TID 24) 20/08/08 15:33:52.963 Executor task launch worker for task 4 INFO PythonRunner: Times: total = 172, boot = 77, init = 95, finish = 0 20/08/08 15:33:52.965 Executor task launch worker for task 24 DEBUG ExecutorMetricsPoller: stageTCMP: (6, 0) -> 1 20/08/08 15:33:52.965 Executor task launch worker for task 25 DEBUG ExecutorMetricsPoller: stageTCMP: (6, 0) -> 2 20/08/08 15:33:52.965 Executor task launch worker for task 26 DEBUG ExecutorMetricsPoller: stageTCMP: (6, 0) -> 3 20/08/08 15:33:52.965 Executor task launch worker for task 23 DEBUG ExecutorMetricsPoller: stageTCMP: (6, 0) -> 4 20/08/08 15:33:52.966 Executor task launch worker for task 24 DEBUG BlockManager: Getting local block broadcast_5 20/08/08 15:33:52.966 Executor task launch worker for task 24 DEBUG BlockManager: Level for block broadcast_5 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:52.974 Executor task launch worker for task 24 DEBUG BlockManager: Getting local block rdd_25_1 20/08/08 15:33:52.974 Executor task launch worker for task 24 DEBUG BlockManager: Block rdd_25_1 was not found 20/08/08 15:33:52.974 Executor task launch worker for task 24 DEBUG BlockManager: Getting remote block rdd_25_1 20/08/08 15:33:52.974 Executor task launch worker for task 23 DEBUG BlockManager: Getting local block rdd_25_0 20/08/08 15:33:52.974 Executor task launch worker for task 23 DEBUG BlockManager: Block rdd_25_0 was not found 20/08/08 15:33:52.974 Executor task launch worker for task 23 DEBUG BlockManager: Getting remote block rdd_25_0 20/08/08 15:33:52.974 Executor task launch worker for task 23 DEBUG BlockManager: Block rdd_25_0 is unknown by block manager master 20/08/08 15:33:52.974 Executor task launch worker for task 23 DEBUG BlockManager: Getting local block rdd_24_0 20/08/08 15:33:52.974 Executor task launch worker for task 23 DEBUG BlockManager: Block rdd_24_0 was not found 20/08/08 15:33:52.975 Executor task launch worker for task 23 DEBUG BlockManager: Getting remote block rdd_24_0 20/08/08 15:33:52.975 Executor task launch worker for task 23 DEBUG BlockManager: Block rdd_24_0 is unknown by block manager master 20/08/08 15:33:52.975 Executor task launch worker for task 23 DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 2 20/08/08 15:33:52.975 Executor task launch worker for task 23 DEBUG MapOutputTrackerMaster: Convert map statuses for shuffle 2, mappers 0-4, partitions 0-1 20/08/08 15:33:52.975 Executor task launch worker for task 23 DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 20/08/08 15:33:52.976 Executor task launch worker for task 23 INFO ShuffleBlockFetcherIterator: Getting 1 (251.0 B) non-empty blocks including 1 (251.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 20/08/08 15:33:52.976 Executor task launch worker for task 23 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 20/08/08 15:33:52.976 Executor task launch worker for task 23 DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_2_20_0,1) 20/08/08 15:33:52.976 Executor task launch worker for task 23 INFO BlockManager: Getting local shuffle block shuffle_2_20_0 20/08/08 15:33:52.976 Executor task launch worker for task 23 DEBUG ShuffleBlockFetcherIterator: Got local blocks in 1 ms 20/08/08 15:33:52.978 Executor task launch worker for task 23 DEBUG ALS: Start sorting an uncompressed in-block of size 1. (sortId = 428592722) 20/08/08 15:33:52.978 Executor task launch worker for task 23 DEBUG ALS: Sorting took 1.203E-5 seconds. (sortId = 428592722) 20/08/08 15:33:52.978 Executor task launch worker for task 23 DEBUG TaskMemoryManager: Task 23 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@5b3a8ac9 20/08/08 15:33:52.978 Executor task launch worker for task 23 INFO MemoryStore: Block rdd_24_0 stored as values in memory (estimated size 192.0 B, free 366.1 MiB) 20/08/08 15:33:52.974 Executor task launch worker for task 25 DEBUG BlockManager: Getting local block rdd_25_2 20/08/08 15:33:52.982 Executor task launch worker for task 25 DEBUG BlockManager: Block rdd_25_2 was not found 20/08/08 15:33:52.982 Executor task launch worker for task 25 DEBUG BlockManager: Getting remote block rdd_25_2 20/08/08 15:33:52.976 Executor task launch worker for task 24 DEBUG BlockManager: Block rdd_25_1 is unknown by block manager master 20/08/08 15:33:52.982 Executor task launch worker for task 24 DEBUG BlockManager: Getting local block rdd_24_1 20/08/08 15:33:52.982 Executor task launch worker for task 24 DEBUG BlockManager: Block rdd_24_1 was not found 20/08/08 15:33:52.982 Executor task launch worker for task 24 DEBUG BlockManager: Getting remote block rdd_24_1 20/08/08 15:33:52.975 Executor task launch worker for task 26 DEBUG BlockManager: Getting local block rdd_25_3 20/08/08 15:33:52.982 Executor task launch worker for task 26 DEBUG BlockManager: Block rdd_25_3 was not found 20/08/08 15:33:52.982 Executor task launch worker for task 26 DEBUG BlockManager: Getting remote block rdd_25_3 20/08/08 15:33:52.983 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_24_0 for BlockManagerId(driver, 192.168.1.225, 38159, None) 20/08/08 15:33:52.983 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_24_0 in memory on 192.168.1.225:38159 (size: 192.0 B, free: 366.2 MiB) 20/08/08 15:33:52.983 Executor task launch worker for task 25 DEBUG BlockManager: Block rdd_25_2 is unknown by block manager master 20/08/08 15:33:52.984 Executor task launch worker for task 24 DEBUG BlockManager: Block rdd_24_1 is unknown by block manager master 20/08/08 15:33:52.984 Executor task launch worker for task 25 DEBUG BlockManager: Getting local block rdd_24_2 20/08/08 15:33:52.984 Executor task launch worker for task 24 DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 2 20/08/08 15:33:52.984 Executor task launch worker for task 24 DEBUG MapOutputTrackerMaster: Convert map statuses for shuffle 2, mappers 0-4, partitions 1-2 20/08/08 15:33:52.984 Executor task launch worker for task 24 DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 20/08/08 15:33:52.985 Executor task launch worker for task 24 INFO ShuffleBlockFetcherIterator: Getting 3 (753.0 B) non-empty blocks including 3 (753.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 20/08/08 15:33:52.985 Executor task launch worker for task 24 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 20/08/08 15:33:52.985 Executor task launch worker for task 24 DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_2_19_1,0), (shuffle_2_21_1,2), (shuffle_2_22_1,3) 20/08/08 15:33:52.985 Executor task launch worker for task 24 INFO BlockManager: Getting local shuffle block shuffle_2_19_1 20/08/08 15:33:52.985 Executor task launch worker for task 24 INFO BlockManager: Getting local shuffle block shuffle_2_21_1 20/08/08 15:33:52.985 Executor task launch worker for task 24 INFO BlockManager: Getting local shuffle block shuffle_2_22_1 20/08/08 15:33:52.986 Executor task launch worker for task 24 DEBUG ShuffleBlockFetcherIterator: Got local blocks in 1 ms 20/08/08 15:33:52.983 Executor task launch worker for task 26 DEBUG BlockManager: Block rdd_25_3 is unknown by block manager master 20/08/08 15:33:52.987 Executor task launch worker for task 26 DEBUG BlockManager: Getting local block rdd_24_3 20/08/08 15:33:52.987 Executor task launch worker for task 26 DEBUG BlockManager: Block rdd_24_3 was not found 20/08/08 15:33:52.987 Executor task launch worker for task 26 DEBUG BlockManager: Getting remote block rdd_24_3 20/08/08 15:33:52.984 Executor task launch worker for task 25 DEBUG BlockManager: Block rdd_24_2 was not found 20/08/08 15:33:52.987 Executor task launch worker for task 25 DEBUG BlockManager: Getting remote block rdd_24_2 20/08/08 15:33:52.988 Executor task launch worker for task 24 DEBUG ALS: Start sorting an uncompressed in-block of size 3. (sortId = -1875253765) 20/08/08 15:33:52.988 Executor task launch worker for task 24 DEBUG ALS: Sorting took 3.842E-5 seconds. (sortId = -1875253765) 20/08/08 15:33:52.988 Executor task launch worker for task 24 DEBUG TaskMemoryManager: Task 24 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@13519ae 20/08/08 15:33:52.989 Executor task launch worker for task 24 INFO MemoryStore: Block rdd_24_1 stored as values in memory (estimated size 208.0 B, free 366.1 MiB) 20/08/08 15:33:52.990 Executor task launch worker for task 23 DEBUG BlockManagerMaster: Updated info of block rdd_24_0 20/08/08 15:33:52.990 Executor task launch worker for task 23 DEBUG BlockManager: Told master about block rdd_24_0 20/08/08 15:33:52.990 Executor task launch worker for task 23 DEBUG BlockManager: Put block rdd_24_0 locally took 14 ms 20/08/08 15:33:52.990 Executor task launch worker for task 23 DEBUG BlockManager: Putting block rdd_24_0 without replication took 15 ms 20/08/08 15:33:52.990 Executor task launch worker for task 23 DEBUG BlockManager: Getting local block rdd_24_0 20/08/08 15:33:52.990 Executor task launch worker for task 23 DEBUG BlockManager: Level for block rdd_24_0 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:52.991 Executor task launch worker for task 23 INFO MemoryStore: Block rdd_25_0 stored as values in memory (estimated size 288.0 B, free 366.1 MiB) 20/08/08 15:33:52.996 Executor task launch worker for task 1 DEBUG TaskMemoryManager: Task 1 release 0.0 B from org.apache.spark.util.collection.ExternalSorter@d4b4c09 20/08/08 15:33:52.996 Executor task launch worker for task 3 DEBUG TaskMemoryManager: Task 3 release 0.0 B from org.apache.spark.util.collection.ExternalSorter@6e14e56b 20/08/08 15:33:52.997 Executor task launch worker for task 4 DEBUG TaskMemoryManager: Task 4 release 0.0 B from org.apache.spark.util.collection.ExternalSorter@1ce59664 20/08/08 15:33:53.000 Executor task launch worker for task 1 INFO Executor: Finished task 0.0 in stage 1.0 (TID 1). 2116 bytes result sent to driver 20/08/08 15:33:53.001 Executor task launch worker for task 2 DEBUG TaskMemoryManager: Task 2 release 0.0 B from org.apache.spark.util.collection.ExternalSorter@408687fc 20/08/08 15:33:53.001 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_24_1 for BlockManagerId(driver, 192.168.1.225, 38159, None) 20/08/08 15:33:53.002 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_24_1 in memory on 192.168.1.225:38159 (size: 208.0 B, free: 366.2 MiB) 20/08/08 15:33:53.002 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_25_0 for BlockManagerId(driver, 192.168.1.225, 38159, None) 20/08/08 15:33:53.002 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_25_0 in memory on 192.168.1.225:38159 (size: 288.0 B, free: 366.2 MiB) 20/08/08 15:33:53.002 Executor task launch worker for task 26 DEBUG BlockManager: Block rdd_24_3 is unknown by block manager master 20/08/08 15:33:53.002 Executor task launch worker for task 4 INFO Executor: Finished task 3.0 in stage 1.0 (TID 4). 2073 bytes result sent to driver 20/08/08 15:33:53.002 Executor task launch worker for task 26 DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 2 20/08/08 15:33:53.002 Executor task launch worker for task 26 DEBUG MapOutputTrackerMaster: Convert map statuses for shuffle 2, mappers 0-4, partitions 3-4 20/08/08 15:33:53.002 Executor task launch worker for task 3 INFO Executor: Finished task 2.0 in stage 1.0 (TID 3). 2073 bytes result sent to driver 20/08/08 15:33:53.002 Executor task launch worker for task 2 INFO Executor: Finished task 1.0 in stage 1.0 (TID 2). 2073 bytes result sent to driver 20/08/08 15:33:53.003 Executor task launch worker for task 26 DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 20/08/08 15:33:53.003 Executor task launch worker for task 23 DEBUG BlockManagerMaster: Updated info of block rdd_25_0 20/08/08 15:33:53.003 Executor task launch worker for task 1 DEBUG ExecutorMetricsPoller: stageTCMP: (1, 0) -> 3 20/08/08 15:33:53.003 Executor task launch worker for task 23 DEBUG BlockManager: Told master about block rdd_25_0 20/08/08 15:33:53.003 Executor task launch worker for task 23 DEBUG BlockManager: Put block rdd_25_0 locally took 28 ms 20/08/08 15:33:53.003 Executor task launch worker for task 2 DEBUG ExecutorMetricsPoller: stageTCMP: (1, 0) -> 2 20/08/08 15:33:53.003 Executor task launch worker for task 3 DEBUG ExecutorMetricsPoller: stageTCMP: (1, 0) -> 1 20/08/08 15:33:53.003 Executor task launch worker for task 23 DEBUG BlockManager: Putting block rdd_25_0 without replication took 29 ms 20/08/08 15:33:53.003 Executor task launch worker for task 23 DEBUG BlockManager: Getting local block rdd_25_0 20/08/08 15:33:53.004 Executor task launch worker for task 4 DEBUG ExecutorMetricsPoller: removing (1, 0) from stageTCMP 20/08/08 15:33:53.004 Executor task launch worker for task 23 DEBUG BlockManager: Level for block rdd_25_0 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:53.005 Executor task launch worker for task 24 DEBUG BlockManagerMaster: Updated info of block rdd_24_1 20/08/08 15:33:53.005 Executor task launch worker for task 24 DEBUG BlockManager: Told master about block rdd_24_1 20/08/08 15:33:53.005 Executor task launch worker for task 24 DEBUG BlockManager: Put block rdd_24_1 locally took 21 ms 20/08/08 15:33:53.005 Executor task launch worker for task 23 INFO Executor: Finished task 0.0 in stage 6.0 (TID 23). 2170 bytes result sent to driver 20/08/08 15:33:53.001 Executor task launch worker for task 25 DEBUG BlockManager: Block rdd_24_2 is unknown by block manager master 20/08/08 15:33:53.005 Executor task launch worker for task 23 DEBUG ExecutorMetricsPoller: stageTCMP: (6, 0) -> 3 20/08/08 15:33:53.005 Executor task launch worker for task 25 DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 2 20/08/08 15:33:53.005 Executor task launch worker for task 25 DEBUG MapOutputTrackerMaster: Convert map statuses for shuffle 2, mappers 0-4, partitions 2-3 20/08/08 15:33:53.006 dispatcher-event-loop-0 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_6.0, runningTasks: 3 20/08/08 15:33:53.006 dispatcher-event-loop-0 DEBUG TaskSetManager: No tasks for locality level NODE_LOCAL, so moving to locality level NO_PREF 20/08/08 15:33:53.006 dispatcher-event-loop-0 DEBUG TaskSetManager: Moving to ANY after waiting for 0ms 20/08/08 15:33:53.006 dispatcher-event-loop-0 INFO TaskSetManager: Starting task 4.0 in stage 6.0 (TID 27, 192.168.1.225, executor driver, partition 4, PROCESS_LOCAL, 7143 bytes) taskResourceAssignments Map() 20/08/08 15:33:53.007 Executor task launch worker for task 27 INFO Executor: Running task 4.0 in stage 6.0 (TID 27) 20/08/08 15:33:53.007 task-result-getter-1 INFO TaskSetManager: Finished task 0.0 in stage 6.0 (TID 23) in 47 ms on 192.168.1.225 (executor driver) (1/10) 20/08/08 15:33:53.008 Executor task launch worker for task 27 DEBUG ExecutorMetricsPoller: stageTCMP: (6, 0) -> 4 20/08/08 15:33:53.005 Executor task launch worker for task 24 DEBUG BlockManager: Putting block rdd_24_1 without replication took 21 ms 20/08/08 15:33:53.003 Executor task launch worker for task 26 INFO ShuffleBlockFetcherIterator: Getting 0 (0.0 B) non-empty blocks including 0 (0.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 20/08/08 15:33:53.006 Executor task launch worker for task 25 DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 20/08/08 15:33:53.009 Executor task launch worker for task 24 DEBUG BlockManager: Getting local block rdd_24_1 20/08/08 15:33:53.009 Executor task launch worker for task 26 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 6 ms 20/08/08 15:33:53.009 Executor task launch worker for task 26 DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: 20/08/08 15:33:53.009 Executor task launch worker for task 26 DEBUG ShuffleBlockFetcherIterator: Got local blocks in 6 ms 20/08/08 15:33:53.010 Executor task launch worker for task 25 INFO ShuffleBlockFetcherIterator: Getting 2 (527.0 B) non-empty blocks including 2 (527.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 20/08/08 15:33:53.010 Executor task launch worker for task 25 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 3 ms 20/08/08 15:33:53.010 Executor task launch worker for task 25 DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_2_21_2,2), (shuffle_2_22_2,3) 20/08/08 15:33:53.010 Executor task launch worker for task 25 INFO BlockManager: Getting local shuffle block shuffle_2_21_2 20/08/08 15:33:53.010 Executor task launch worker for task 26 DEBUG TaskMemoryManager: Task 26 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@6847761e 20/08/08 15:33:53.010 Executor task launch worker for task 25 INFO BlockManager: Getting local shuffle block shuffle_2_22_2 20/08/08 15:33:53.010 Executor task launch worker for task 25 DEBUG ShuffleBlockFetcherIterator: Got local blocks in 4 ms 20/08/08 15:33:53.010 Executor task launch worker for task 26 INFO MemoryStore: Block rdd_24_3 stored as values in memory (estimated size 16.0 B, free 366.1 MiB) 20/08/08 15:33:53.011 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_24_3 for BlockManagerId(driver, 192.168.1.225, 38159, None) 20/08/08 15:33:53.011 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_24_3 in memory on 192.168.1.225:38159 (size: 16.0 B, free: 366.2 MiB) 20/08/08 15:33:53.012 Executor task launch worker for task 26 DEBUG BlockManagerMaster: Updated info of block rdd_24_3 20/08/08 15:33:53.012 Executor task launch worker for task 26 DEBUG BlockManager: Told master about block rdd_24_3 20/08/08 15:33:53.012 Executor task launch worker for task 25 DEBUG ALS: Start sorting an uncompressed in-block of size 2. (sortId = -1149407694) 20/08/08 15:33:53.012 Executor task launch worker for task 24 DEBUG BlockManager: Level for block rdd_24_1 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:53.013 Executor task launch worker for task 24 INFO MemoryStore: Block rdd_25_1 stored as values in memory (estimated size 304.0 B, free 366.1 MiB) 20/08/08 15:33:53.012 Executor task launch worker for task 26 DEBUG BlockManager: Put block rdd_24_3 locally took 9 ms 20/08/08 15:33:53.012 Executor task launch worker for task 25 DEBUG ALS: Sorting took 3.362E-5 seconds. (sortId = -1149407694) 20/08/08 15:33:53.013 Executor task launch worker for task 26 DEBUG BlockManager: Putting block rdd_24_3 without replication took 10 ms 20/08/08 15:33:53.013 Executor task launch worker for task 26 DEBUG BlockManager: Getting local block rdd_24_3 20/08/08 15:33:53.013 Executor task launch worker for task 26 DEBUG BlockManager: Level for block rdd_24_3 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:53.013 Executor task launch worker for task 25 DEBUG TaskMemoryManager: Task 25 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@6ab2ce99 20/08/08 15:33:53.014 Executor task launch worker for task 26 INFO MemoryStore: Block rdd_25_3 stored as values in memory (estimated size 16.0 B, free 366.1 MiB) 20/08/08 15:33:53.014 Executor task launch worker for task 25 INFO MemoryStore: Block rdd_24_2 stored as values in memory (estimated size 192.0 B, free 366.1 MiB) 20/08/08 15:33:53.014 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_25_1 for BlockManagerId(driver, 192.168.1.225, 38159, None) 20/08/08 15:33:53.015 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_25_1 in memory on 192.168.1.225:38159 (size: 304.0 B, free: 366.2 MiB) 20/08/08 15:33:53.015 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_24_2 for BlockManagerId(driver, 192.168.1.225, 38159, None) 20/08/08 15:33:53.016 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_24_2 in memory on 192.168.1.225:38159 (size: 192.0 B, free: 366.2 MiB) 20/08/08 15:33:53.016 dispatcher-event-loop-3 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_1.0, runningTasks: 3 20/08/08 15:33:53.016 Executor task launch worker for task 25 DEBUG BlockManagerMaster: Updated info of block rdd_24_2 20/08/08 15:33:53.016 Executor task launch worker for task 25 DEBUG BlockManager: Told master about block rdd_24_2 20/08/08 15:33:53.016 dispatcher-event-loop-3 DEBUG TaskSetManager: No tasks for locality level NO_PREF, so moving to locality level ANY 20/08/08 15:33:53.016 Executor task launch worker for task 25 DEBUG BlockManager: Put block rdd_24_2 locally took 10 ms 20/08/08 15:33:53.016 Executor task launch worker for task 25 DEBUG BlockManager: Putting block rdd_24_2 without replication took 10 ms 20/08/08 15:33:53.016 Executor task launch worker for task 25 DEBUG BlockManager: Getting local block rdd_24_2 20/08/08 15:33:53.016 Executor task launch worker for task 25 DEBUG BlockManager: Level for block rdd_24_2 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:53.017 Executor task launch worker for task 25 INFO MemoryStore: Block rdd_25_2 stored as values in memory (estimated size 296.0 B, free 366.1 MiB) 20/08/08 15:33:53.017 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_25_2 for BlockManagerId(driver, 192.168.1.225, 38159, None) 20/08/08 15:33:53.018 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_25_2 in memory on 192.168.1.225:38159 (size: 296.0 B, free: 366.2 MiB) 20/08/08 15:33:53.018 Executor task launch worker for task 25 DEBUG BlockManagerMaster: Updated info of block rdd_25_2 20/08/08 15:33:53.018 Executor task launch worker for task 25 DEBUG BlockManager: Told master about block rdd_25_2 20/08/08 15:33:53.018 Executor task launch worker for task 25 DEBUG BlockManager: Put block rdd_25_2 locally took 34 ms 20/08/08 15:33:53.019 Executor task launch worker for task 25 DEBUG BlockManager: Putting block rdd_25_2 without replication took 34 ms 20/08/08 15:33:53.019 Executor task launch worker for task 25 DEBUG BlockManager: Getting local block rdd_25_2 20/08/08 15:33:53.019 Executor task launch worker for task 25 DEBUG BlockManager: Level for block rdd_25_2 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:53.019 Executor task launch worker for task 27 DEBUG BlockManager: Getting local block rdd_25_4 20/08/08 15:33:53.019 Executor task launch worker for task 27 DEBUG BlockManager: Block rdd_25_4 was not found 20/08/08 15:33:53.019 Executor task launch worker for task 27 DEBUG BlockManager: Getting remote block rdd_25_4 20/08/08 15:33:53.020 Executor task launch worker for task 25 INFO Executor: Finished task 2.0 in stage 6.0 (TID 25). 2170 bytes result sent to driver 20/08/08 15:33:53.020 Executor task launch worker for task 25 DEBUG ExecutorMetricsPoller: stageTCMP: (6, 0) -> 3 20/08/08 15:33:53.020 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_25_3 for BlockManagerId(driver, 192.168.1.225, 38159, None) 20/08/08 15:33:53.020 Executor task launch worker for task 24 DEBUG BlockManagerMaster: Updated info of block rdd_25_1 20/08/08 15:33:53.020 Executor task launch worker for task 24 DEBUG BlockManager: Told master about block rdd_25_1 20/08/08 15:33:53.020 Executor task launch worker for task 24 DEBUG BlockManager: Put block rdd_25_1 locally took 38 ms 20/08/08 15:33:53.020 dispatcher-event-loop-3 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_6.0, runningTasks: 3 20/08/08 15:33:53.020 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_25_3 in memory on 192.168.1.225:38159 (size: 16.0 B, free: 366.2 MiB) 20/08/08 15:33:53.021 dispatcher-event-loop-3 DEBUG TaskSetManager: No tasks for locality level NODE_LOCAL, so moving to locality level NO_PREF 20/08/08 15:33:53.021 dispatcher-event-loop-3 DEBUG TaskSetManager: Moving to ANY after waiting for 0ms 20/08/08 15:33:53.021 Executor task launch worker for task 26 DEBUG BlockManagerMaster: Updated info of block rdd_25_3 20/08/08 15:33:53.021 Executor task launch worker for task 26 DEBUG BlockManager: Told master about block rdd_25_3 20/08/08 15:33:53.021 Executor task launch worker for task 26 DEBUG BlockManager: Put block rdd_25_3 locally took 34 ms 20/08/08 15:33:53.021 Executor task launch worker for task 26 DEBUG BlockManager: Putting block rdd_25_3 without replication took 34 ms 20/08/08 15:33:53.021 Executor task launch worker for task 26 DEBUG BlockManager: Getting local block rdd_25_3 20/08/08 15:33:53.021 Executor task launch worker for task 26 DEBUG BlockManager: Level for block rdd_25_3 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:53.022 dispatcher-event-loop-3 INFO TaskSetManager: Starting task 5.0 in stage 6.0 (TID 28, 192.168.1.225, executor driver, partition 5, PROCESS_LOCAL, 7143 bytes) taskResourceAssignments Map() 20/08/08 15:33:53.022 Executor task launch worker for task 26 INFO Executor: Finished task 3.0 in stage 6.0 (TID 26). 2170 bytes result sent to driver 20/08/08 15:33:53.022 Executor task launch worker for task 26 DEBUG ExecutorMetricsPoller: stageTCMP: (6, 0) -> 2 20/08/08 15:33:53.022 task-result-getter-3 INFO TaskSetManager: Finished task 2.0 in stage 6.0 (TID 25) in 61 ms on 192.168.1.225 (executor driver) (2/10) 20/08/08 15:33:53.023 dispatcher-event-loop-2 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_6.0, runningTasks: 3 20/08/08 15:33:53.023 Executor task launch worker for task 28 INFO Executor: Running task 5.0 in stage 6.0 (TID 28) 20/08/08 15:33:53.023 dispatcher-event-loop-2 DEBUG TaskSetManager: No tasks for locality level NODE_LOCAL, so moving to locality level NO_PREF 20/08/08 15:33:53.023 dispatcher-event-loop-2 DEBUG TaskSetManager: Moving to ANY after waiting for 0ms 20/08/08 15:33:53.023 dispatcher-event-loop-2 INFO TaskSetManager: Starting task 6.0 in stage 6.0 (TID 29, 192.168.1.225, executor driver, partition 6, PROCESS_LOCAL, 7143 bytes) taskResourceAssignments Map() 20/08/08 15:33:53.020 Executor task launch worker for task 24 DEBUG BlockManager: Putting block rdd_25_1 without replication took 38 ms 20/08/08 15:33:53.024 Executor task launch worker for task 24 DEBUG BlockManager: Getting local block rdd_25_1 20/08/08 15:33:53.024 Executor task launch worker for task 24 DEBUG BlockManager: Level for block rdd_25_1 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:53.024 task-result-getter-2 INFO TaskSetManager: Finished task 3.0 in stage 6.0 (TID 26) in 63 ms on 192.168.1.225 (executor driver) (3/10) 20/08/08 15:33:53.025 Executor task launch worker for task 29 INFO Executor: Running task 6.0 in stage 6.0 (TID 29) 20/08/08 15:33:53.025 dispatcher-event-loop-3 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_1.0, runningTasks: 2 20/08/08 15:33:53.026 Executor task launch worker for task 29 DEBUG ExecutorMetricsPoller: stageTCMP: (6, 0) -> 4 20/08/08 15:33:53.023 Executor task launch worker for task 28 DEBUG ExecutorMetricsPoller: stageTCMP: (6, 0) -> 3 20/08/08 15:33:53.026 task-result-getter-1 INFO TaskSetManager: Finished task 0.0 in stage 1.0 (TID 1) in 962 ms on 192.168.1.225 (executor driver) (1/4) 20/08/08 15:33:53.027 Executor task launch worker for task 24 INFO Executor: Finished task 1.0 in stage 6.0 (TID 24). 2170 bytes result sent to driver 20/08/08 15:33:53.027 Executor task launch worker for task 24 DEBUG ExecutorMetricsPoller: stageTCMP: (6, 0) -> 3 20/08/08 15:33:53.027 dispatcher-event-loop-3 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_6.0, runningTasks: 3 20/08/08 15:33:53.026 Executor task launch worker for task 27 DEBUG BlockManager: Block rdd_25_4 is unknown by block manager master 20/08/08 15:33:53.027 dispatcher-event-loop-3 DEBUG TaskSetManager: No tasks for locality level NODE_LOCAL, so moving to locality level NO_PREF 20/08/08 15:33:53.028 dispatcher-event-loop-3 DEBUG TaskSetManager: Moving to ANY after waiting for 0ms 20/08/08 15:33:53.028 Executor task launch worker for task 27 DEBUG BlockManager: Getting local block rdd_24_4 20/08/08 15:33:53.028 Executor task launch worker for task 27 DEBUG BlockManager: Block rdd_24_4 was not found 20/08/08 15:33:53.028 Executor task launch worker for task 27 DEBUG BlockManager: Getting remote block rdd_24_4 20/08/08 15:33:53.028 dag-scheduler-event-loop DEBUG DAGScheduler: ShuffleMapTask finished on driver 20/08/08 15:33:53.028 dispatcher-event-loop-3 INFO TaskSetManager: Starting task 7.0 in stage 6.0 (TID 30, 192.168.1.225, executor driver, partition 7, PROCESS_LOCAL, 7143 bytes) taskResourceAssignments Map() 20/08/08 15:33:53.029 task-result-getter-0 INFO TaskSetManager: Finished task 1.0 in stage 6.0 (TID 24) in 68 ms on 192.168.1.225 (executor driver) (4/10) 20/08/08 15:33:53.034 Executor task launch worker for task 29 DEBUG BlockManager: Getting local block rdd_25_6 20/08/08 15:33:53.034 Executor task launch worker for task 29 DEBUG BlockManager: Block rdd_25_6 was not found 20/08/08 15:33:53.034 Executor task launch worker for task 29 DEBUG BlockManager: Getting remote block rdd_25_6 20/08/08 15:33:53.034 Executor task launch worker for task 27 DEBUG BlockManager: Block rdd_24_4 is unknown by block manager master 20/08/08 15:33:53.035 Executor task launch worker for task 27 DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 2 20/08/08 15:33:53.035 Executor task launch worker for task 27 DEBUG MapOutputTrackerMaster: Convert map statuses for shuffle 2, mappers 0-4, partitions 4-5 20/08/08 15:33:53.035 Executor task launch worker for task 27 DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 20/08/08 15:33:53.035 Executor task launch worker for task 27 INFO ShuffleBlockFetcherIterator: Getting 0 (0.0 B) non-empty blocks including 0 (0.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 20/08/08 15:33:53.035 Executor task launch worker for task 27 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 20/08/08 15:33:53.036 Executor task launch worker for task 27 DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: 20/08/08 15:33:53.036 Executor task launch worker for task 27 DEBUG ShuffleBlockFetcherIterator: Got local blocks in 0 ms 20/08/08 15:33:53.036 Executor task launch worker for task 28 DEBUG BlockManager: Getting local block rdd_25_5 20/08/08 15:33:53.036 Executor task launch worker for task 28 DEBUG BlockManager: Block rdd_25_5 was not found 20/08/08 15:33:53.036 Executor task launch worker for task 27 DEBUG TaskMemoryManager: Task 27 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@576facf2 20/08/08 15:33:53.037 Executor task launch worker for task 27 INFO MemoryStore: Block rdd_24_4 stored as values in memory (estimated size 16.0 B, free 366.1 MiB) 20/08/08 15:33:53.037 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_24_4 for BlockManagerId(driver, 192.168.1.225, 38159, None) 20/08/08 15:33:53.037 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_24_4 in memory on 192.168.1.225:38159 (size: 16.0 B, free: 366.2 MiB) 20/08/08 15:33:53.038 Executor task launch worker for task 27 DEBUG BlockManagerMaster: Updated info of block rdd_24_4 20/08/08 15:33:53.038 Executor task launch worker for task 27 DEBUG BlockManager: Told master about block rdd_24_4 20/08/08 15:33:53.038 Executor task launch worker for task 27 DEBUG BlockManager: Put block rdd_24_4 locally took 3 ms 20/08/08 15:33:53.038 Executor task launch worker for task 27 DEBUG BlockManager: Putting block rdd_24_4 without replication took 3 ms 20/08/08 15:33:53.038 Executor task launch worker for task 27 DEBUG BlockManager: Getting local block rdd_24_4 20/08/08 15:33:53.038 Executor task launch worker for task 27 DEBUG BlockManager: Level for block rdd_24_4 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:53.036 Executor task launch worker for task 28 DEBUG BlockManager: Getting remote block rdd_25_5 20/08/08 15:33:53.038 Executor task launch worker for task 27 INFO MemoryStore: Block rdd_25_4 stored as values in memory (estimated size 16.0 B, free 366.1 MiB) 20/08/08 15:33:53.039 Executor task launch worker for task 28 DEBUG BlockManager: Block rdd_25_5 is unknown by block manager master 20/08/08 15:33:53.036 Executor task launch worker for task 30 INFO Executor: Running task 7.0 in stage 6.0 (TID 30) 20/08/08 15:33:53.039 Executor task launch worker for task 28 DEBUG BlockManager: Getting local block rdd_24_5 20/08/08 15:33:53.039 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_25_4 for BlockManagerId(driver, 192.168.1.225, 38159, None) 20/08/08 15:33:53.039 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_25_4 in memory on 192.168.1.225:38159 (size: 16.0 B, free: 366.2 MiB) 20/08/08 15:33:53.039 Executor task launch worker for task 30 DEBUG ExecutorMetricsPoller: stageTCMP: (6, 0) -> 4 20/08/08 15:33:53.039 Executor task launch worker for task 28 DEBUG BlockManager: Block rdd_24_5 was not found 20/08/08 15:33:53.040 Executor task launch worker for task 28 DEBUG BlockManager: Getting remote block rdd_24_5 20/08/08 15:33:53.044 Executor task launch worker for task 28 DEBUG BlockManager: Block rdd_24_5 is unknown by block manager master 20/08/08 15:33:53.044 Executor task launch worker for task 28 DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 2 20/08/08 15:33:53.044 Executor task launch worker for task 28 DEBUG MapOutputTrackerMaster: Convert map statuses for shuffle 2, mappers 0-4, partitions 5-6 20/08/08 15:33:53.045 Executor task launch worker for task 28 DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 20/08/08 15:33:53.045 Executor task launch worker for task 28 INFO ShuffleBlockFetcherIterator: Getting 0 (0.0 B) non-empty blocks including 0 (0.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 20/08/08 15:33:53.045 Executor task launch worker for task 28 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 20/08/08 15:33:53.045 Executor task launch worker for task 28 DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: 20/08/08 15:33:53.045 Executor task launch worker for task 28 DEBUG ShuffleBlockFetcherIterator: Got local blocks in 0 ms 20/08/08 15:33:53.046 Executor task launch worker for task 30 DEBUG BlockManager: Getting local block rdd_25_7 20/08/08 15:33:53.046 Executor task launch worker for task 30 DEBUG BlockManager: Block rdd_25_7 was not found 20/08/08 15:33:53.046 Executor task launch worker for task 30 DEBUG BlockManager: Getting remote block rdd_25_7 20/08/08 15:33:53.046 Executor task launch worker for task 27 DEBUG BlockManagerMaster: Updated info of block rdd_25_4 20/08/08 15:33:53.046 Executor task launch worker for task 30 DEBUG BlockManager: Block rdd_25_7 is unknown by block manager master 20/08/08 15:33:53.046 Executor task launch worker for task 29 DEBUG BlockManager: Block rdd_25_6 is unknown by block manager master 20/08/08 15:33:53.046 Executor task launch worker for task 28 DEBUG TaskMemoryManager: Task 28 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@675097f8 20/08/08 15:33:53.046 Executor task launch worker for task 27 DEBUG BlockManager: Told master about block rdd_25_4 20/08/08 15:33:53.046 Executor task launch worker for task 27 DEBUG BlockManager: Put block rdd_25_4 locally took 18 ms 20/08/08 15:33:53.046 Executor task launch worker for task 27 DEBUG BlockManager: Putting block rdd_25_4 without replication took 18 ms 20/08/08 15:33:53.046 Executor task launch worker for task 27 DEBUG BlockManager: Getting local block rdd_25_4 20/08/08 15:33:53.046 Executor task launch worker for task 27 DEBUG BlockManager: Level for block rdd_25_4 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:53.046 Executor task launch worker for task 28 INFO MemoryStore: Block rdd_24_5 stored as values in memory (estimated size 16.0 B, free 366.1 MiB) 20/08/08 15:33:53.046 Executor task launch worker for task 29 DEBUG BlockManager: Getting local block rdd_24_6 20/08/08 15:33:53.047 Executor task launch worker for task 29 DEBUG BlockManager: Block rdd_24_6 was not found 20/08/08 15:33:53.047 Executor task launch worker for task 29 DEBUG BlockManager: Getting remote block rdd_24_6 20/08/08 15:33:53.048 dispatcher-event-loop-3 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_1.0, runningTasks: 1 20/08/08 15:33:53.046 Executor task launch worker for task 30 DEBUG BlockManager: Getting local block rdd_24_7 20/08/08 15:33:53.049 Executor task launch worker for task 30 DEBUG BlockManager: Block rdd_24_7 was not found 20/08/08 15:33:53.049 Executor task launch worker for task 30 DEBUG BlockManager: Getting remote block rdd_24_7 20/08/08 15:33:53.049 dispatcher-event-loop-3 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_1.0, runningTasks: 0 20/08/08 15:33:53.049 Executor task launch worker for task 27 INFO Executor: Finished task 4.0 in stage 6.0 (TID 27). 2170 bytes result sent to driver 20/08/08 15:33:53.050 task-result-getter-0 INFO TaskSetManager: Finished task 3.0 in stage 1.0 (TID 4) in 983 ms on 192.168.1.225 (executor driver) (2/4) 20/08/08 15:33:53.050 Executor task launch worker for task 27 DEBUG ExecutorMetricsPoller: stageTCMP: (6, 0) -> 3 20/08/08 15:33:53.050 task-result-getter-2 INFO TaskSetManager: Finished task 1.0 in stage 1.0 (TID 2) in 985 ms on 192.168.1.225 (executor driver) (3/4) 20/08/08 15:33:53.050 dispatcher-event-loop-2 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_6.0, runningTasks: 3 20/08/08 15:33:53.050 dispatcher-event-loop-2 DEBUG TaskSetManager: No tasks for locality level NODE_LOCAL, so moving to locality level NO_PREF 20/08/08 15:33:53.050 dispatcher-event-loop-2 DEBUG TaskSetManager: Moving to ANY after waiting for 0ms 20/08/08 15:33:53.051 dispatcher-event-loop-2 INFO TaskSetManager: Starting task 8.0 in stage 6.0 (TID 31, 192.168.1.225, executor driver, partition 8, PROCESS_LOCAL, 7143 bytes) taskResourceAssignments Map() 20/08/08 15:33:53.051 Executor task launch worker for task 31 INFO Executor: Running task 8.0 in stage 6.0 (TID 31) 20/08/08 15:33:53.051 task-result-getter-1 INFO TaskSetManager: Finished task 4.0 in stage 6.0 (TID 27) in 45 ms on 192.168.1.225 (executor driver) (5/10) 20/08/08 15:33:53.052 Executor task launch worker for task 31 DEBUG ExecutorMetricsPoller: stageTCMP: (6, 0) -> 4 20/08/08 15:33:53.052 Executor task launch worker for task 29 DEBUG BlockManager: Block rdd_24_6 is unknown by block manager master 20/08/08 15:33:53.052 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_24_5 for BlockManagerId(driver, 192.168.1.225, 38159, None) 20/08/08 15:33:53.053 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_24_5 in memory on 192.168.1.225:38159 (size: 16.0 B, free: 366.2 MiB) 20/08/08 15:33:53.053 Executor task launch worker for task 28 DEBUG BlockManagerMaster: Updated info of block rdd_24_5 20/08/08 15:33:53.053 Executor task launch worker for task 28 DEBUG BlockManager: Told master about block rdd_24_5 20/08/08 15:33:53.053 Executor task launch worker for task 28 DEBUG BlockManager: Put block rdd_24_5 locally took 9 ms 20/08/08 15:33:53.053 Executor task launch worker for task 29 DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 2 20/08/08 15:33:53.053 Executor task launch worker for task 29 DEBUG MapOutputTrackerMaster: Convert map statuses for shuffle 2, mappers 0-4, partitions 6-7 20/08/08 15:33:53.053 Executor task launch worker for task 28 DEBUG BlockManager: Putting block rdd_24_5 without replication took 9 ms 20/08/08 15:33:53.054 Executor task launch worker for task 28 DEBUG BlockManager: Getting local block rdd_24_5 20/08/08 15:33:53.054 Executor task launch worker for task 28 DEBUG BlockManager: Level for block rdd_24_5 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:53.054 Executor task launch worker for task 29 DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 20/08/08 15:33:53.054 Executor task launch worker for task 29 INFO ShuffleBlockFetcherIterator: Getting 0 (0.0 B) non-empty blocks including 0 (0.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 20/08/08 15:33:53.054 Executor task launch worker for task 29 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 20/08/08 15:33:53.054 Executor task launch worker for task 28 INFO MemoryStore: Block rdd_25_5 stored as values in memory (estimated size 16.0 B, free 366.1 MiB) 20/08/08 15:33:53.054 Executor task launch worker for task 29 DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: 20/08/08 15:33:53.055 Executor task launch worker for task 29 DEBUG ShuffleBlockFetcherIterator: Got local blocks in 0 ms 20/08/08 15:33:53.055 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_25_5 for BlockManagerId(driver, 192.168.1.225, 38159, None) 20/08/08 15:33:53.055 Executor task launch worker for task 29 DEBUG TaskMemoryManager: Task 29 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@544a3392 20/08/08 15:33:53.055 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_25_5 in memory on 192.168.1.225:38159 (size: 16.0 B, free: 366.2 MiB) 20/08/08 15:33:53.055 Executor task launch worker for task 28 DEBUG BlockManagerMaster: Updated info of block rdd_25_5 20/08/08 15:33:53.055 Executor task launch worker for task 28 DEBUG BlockManager: Told master about block rdd_25_5 20/08/08 15:33:53.056 Executor task launch worker for task 28 DEBUG BlockManager: Put block rdd_25_5 locally took 16 ms 20/08/08 15:33:53.056 Executor task launch worker for task 28 DEBUG BlockManager: Putting block rdd_25_5 without replication took 16 ms 20/08/08 15:33:53.056 Executor task launch worker for task 28 DEBUG BlockManager: Getting local block rdd_25_5 20/08/08 15:33:53.056 Executor task launch worker for task 28 DEBUG BlockManager: Level for block rdd_25_5 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:53.056 Executor task launch worker for task 29 INFO MemoryStore: Block rdd_24_6 stored as values in memory (estimated size 16.0 B, free 366.1 MiB) 20/08/08 15:33:53.056 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_24_6 for BlockManagerId(driver, 192.168.1.225, 38159, None) 20/08/08 15:33:53.057 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_24_6 in memory on 192.168.1.225:38159 (size: 16.0 B, free: 366.2 MiB) 20/08/08 15:33:53.057 Executor task launch worker for task 28 INFO Executor: Finished task 5.0 in stage 6.0 (TID 28). 2170 bytes result sent to driver 20/08/08 15:33:53.057 Executor task launch worker for task 30 DEBUG BlockManager: Block rdd_24_7 is unknown by block manager master 20/08/08 15:33:53.057 Executor task launch worker for task 28 DEBUG ExecutorMetricsPoller: stageTCMP: (6, 0) -> 3 20/08/08 15:33:53.057 Executor task launch worker for task 29 DEBUG BlockManagerMaster: Updated info of block rdd_24_6 20/08/08 15:33:53.057 Executor task launch worker for task 29 DEBUG BlockManager: Told master about block rdd_24_6 20/08/08 15:33:53.057 Executor task launch worker for task 29 DEBUG BlockManager: Put block rdd_24_6 locally took 3 ms 20/08/08 15:33:53.057 Executor task launch worker for task 29 DEBUG BlockManager: Putting block rdd_24_6 without replication took 3 ms 20/08/08 15:33:53.057 Executor task launch worker for task 29 DEBUG BlockManager: Getting local block rdd_24_6 20/08/08 15:33:53.057 dispatcher-event-loop-3 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_6.0, runningTasks: 3 20/08/08 15:33:53.052 task-result-getter-3 INFO TaskSetManager: Finished task 2.0 in stage 1.0 (TID 3) in 986 ms on 192.168.1.225 (executor driver) (4/4) 20/08/08 15:33:53.058 Executor task launch worker for task 31 DEBUG BlockManager: Getting local block rdd_25_8 20/08/08 15:33:53.058 Executor task launch worker for task 31 DEBUG BlockManager: Block rdd_25_8 was not found 20/08/08 15:33:53.058 Executor task launch worker for task 31 DEBUG BlockManager: Getting remote block rdd_25_8 20/08/08 15:33:53.053 dag-scheduler-event-loop DEBUG DAGScheduler: ShuffleMapTask finished on driver 20/08/08 15:33:53.058 dispatcher-event-loop-3 DEBUG TaskSetManager: No tasks for locality level NODE_LOCAL, so moving to locality level NO_PREF 20/08/08 15:33:53.057 Executor task launch worker for task 30 DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 2 20/08/08 15:33:53.058 dispatcher-event-loop-3 DEBUG TaskSetManager: Moving to ANY after waiting for 0ms 20/08/08 15:33:53.058 dispatcher-event-loop-3 INFO TaskSetManager: Starting task 9.0 in stage 6.0 (TID 32, 192.168.1.225, executor driver, partition 9, PROCESS_LOCAL, 7143 bytes) taskResourceAssignments Map() 20/08/08 15:33:53.058 Executor task launch worker for task 31 DEBUG BlockManager: Block rdd_25_8 is unknown by block manager master 20/08/08 15:33:53.059 Executor task launch worker for task 32 INFO Executor: Running task 9.0 in stage 6.0 (TID 32) 20/08/08 15:33:53.057 Executor task launch worker for task 29 DEBUG BlockManager: Level for block rdd_24_6 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:53.059 task-result-getter-3 INFO TaskSetManager: Finished task 5.0 in stage 6.0 (TID 28) in 37 ms on 192.168.1.225 (executor driver) (6/10) 20/08/08 15:33:53.059 Executor task launch worker for task 32 DEBUG ExecutorMetricsPoller: stageTCMP: (6, 0) -> 4 20/08/08 15:33:53.059 dag-scheduler-event-loop DEBUG DAGScheduler: ShuffleMapTask finished on driver 20/08/08 15:33:53.059 task-result-getter-3 INFO TaskSchedulerImpl: Removed TaskSet 1.0, whose tasks have all completed, from pool 20/08/08 15:33:53.058 Executor task launch worker for task 30 DEBUG MapOutputTrackerMaster: Convert map statuses for shuffle 2, mappers 0-4, partitions 7-8 20/08/08 15:33:53.061 Executor task launch worker for task 30 DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 20/08/08 15:33:53.061 dag-scheduler-event-loop DEBUG DAGScheduler: ShuffleMapTask finished on driver 20/08/08 15:33:53.061 dag-scheduler-event-loop INFO DAGScheduler: ShuffleMapStage 1 (map at ClusteringMetrics.scala:500) finished in 1.102 s 20/08/08 15:33:53.061 Executor task launch worker for task 30 INFO ShuffleBlockFetcherIterator: Getting 0 (0.0 B) non-empty blocks including 0 (0.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 20/08/08 15:33:53.061 Executor task launch worker for task 30 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 20/08/08 15:33:53.061 Executor task launch worker for task 30 DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: 20/08/08 15:33:53.061 Executor task launch worker for task 30 DEBUG ShuffleBlockFetcherIterator: Got local blocks in 0 ms 20/08/08 15:33:53.062 dag-scheduler-event-loop INFO DAGScheduler: looking for newly runnable stages 20/08/08 15:33:53.062 Executor task launch worker for task 30 DEBUG TaskMemoryManager: Task 30 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@99e794b 20/08/08 15:33:53.062 Thread-4 DEBUG WholeStageCodegenExec: /* 001 */ public Object generate(Object[] references) { /* 002 */ return new GeneratedIteratorForCodegenStage1(references); /* 003 */ } /* 004 */ /* 005 */ // codegenStageId=1 /* 006 */ final class GeneratedIteratorForCodegenStage1 extends org.apache.spark.sql.execution.BufferedRowIterator { /* 007 */ private Object[] references; /* 008 */ private scala.collection.Iterator[] inputs; /* 009 */ private scala.collection.Iterator rdd_input_0; /* 010 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[] rdd_mutableStateArray_0 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[3]; /* 011 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[] rdd_mutableStateArray_1 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[4]; /* 012 */ /* 013 */ public GeneratedIteratorForCodegenStage1(Object[] references) { /* 014 */ this.references = references; /* 015 */ } /* 016 */ /* 017 */ public void init(int index, scala.collection.Iterator[] inputs) { /* 018 */ partitionIndex = index; /* 019 */ this.inputs = inputs; /* 020 */ rdd_input_0 = inputs[0]; /* 021 */ rdd_mutableStateArray_0[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(3, 64); /* 022 */ rdd_mutableStateArray_1[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[0], 8); /* 023 */ rdd_mutableStateArray_0[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(rdd_mutableStateArray_0[0], 4); /* 024 */ rdd_mutableStateArray_1[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[1], 4); /* 025 */ rdd_mutableStateArray_1[2] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[1], 8); /* 026 */ rdd_mutableStateArray_0[2] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(1, 32); /* 027 */ rdd_mutableStateArray_1[3] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[2], 8); /* 028 */ /* 029 */ } /* 030 */ /* 031 */ protected void processNext() throws java.io.IOException { /* 032 */ while ( rdd_input_0.hasNext()) { /* 033 */ InternalRow rdd_row_0 = (InternalRow) rdd_input_0.next(); /* 034 */ ((org.apache.spark.sql.execution.metric.SQLMetric) references[0] /* numOutputRows */).add(1); /* 035 */ boolean rdd_isNull_1 = rdd_row_0.isNullAt(1); /* 036 */ ArrayData rdd_value_1 = rdd_isNull_1 ? /* 037 */ null : (rdd_row_0.getArray(1)); /* 038 */ rdd_mutableStateArray_0[2].reset(); /* 039 */ /* 040 */ rdd_mutableStateArray_0[2].zeroOutNullBytes(); /* 041 */ /* 042 */ if (rdd_isNull_1) { /* 043 */ rdd_mutableStateArray_0[2].setNullAt(0); /* 044 */ } else { /* 045 */ // Remember the current cursor so that we can calculate how many bytes are /* 046 */ // written later. /* 047 */ final int project_previousCursor_0 = rdd_mutableStateArray_0[2].cursor(); /* 048 */ /* 049 */ final ArrayData project_tmpInput_0 = rdd_value_1; /* 050 */ if (project_tmpInput_0 instanceof UnsafeArrayData) { /* 051 */ rdd_mutableStateArray_0[2].write((UnsafeArrayData) project_tmpInput_0); /* 052 */ } else { /* 053 */ final int project_numElements_0 = project_tmpInput_0.numElements(); /* 054 */ rdd_mutableStateArray_1[3].initialize(project_numElements_0); /* 055 */ /* 056 */ for (int project_index_0 = 0; project_index_0 < project_numElements_0; project_index_0++) { /* 057 */ if (project_tmpInput_0.isNullAt(project_index_0)) { /* 058 */ rdd_mutableStateArray_1[3].setNull8Bytes(project_index_0); /* 059 */ } else { /* 060 */ rdd_mutableStateArray_1[3].write(project_index_0, project_tmpInput_0.getUTF8String(project_index_0)); /* 061 */ } /* 062 */ /* 063 */ } /* 064 */ } /* 065 */ /* 066 */ rdd_mutableStateArray_0[2].setOffsetAndSizeFromPreviousCursor(0, project_previousCursor_0); /* 067 */ } /* 068 */ append((rdd_mutableStateArray_0[2].getRow())); /* 069 */ if (shouldStop()) return; /* 070 */ } /* 071 */ } /* 072 */ /* 073 */ } 20/08/08 15:33:53.062 dag-scheduler-event-loop INFO DAGScheduler: running: Set() 20/08/08 15:33:53.062 Executor task launch worker for task 30 INFO MemoryStore: Block rdd_24_7 stored as values in memory (estimated size 16.0 B, free 366.1 MiB) 20/08/08 15:33:53.062 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_24_7 for BlockManagerId(driver, 192.168.1.225, 38159, None) 20/08/08 15:33:53.062 Executor task launch worker for task 31 DEBUG BlockManager: Getting local block rdd_24_8 20/08/08 15:33:53.062 Executor task launch worker for task 31 DEBUG BlockManager: Block rdd_24_8 was not found 20/08/08 15:33:53.062 Executor task launch worker for task 31 DEBUG BlockManager: Getting remote block rdd_24_8 20/08/08 15:33:53.063 dag-scheduler-event-loop INFO DAGScheduler: waiting: Set(ResultStage 2) 20/08/08 15:33:53.063 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_24_7 in memory on 192.168.1.225:38159 (size: 16.0 B, free: 366.2 MiB) 20/08/08 15:33:53.063 Executor task launch worker for task 30 DEBUG BlockManagerMaster: Updated info of block rdd_24_7 20/08/08 15:33:53.063 Executor task launch worker for task 30 DEBUG BlockManager: Told master about block rdd_24_7 20/08/08 15:33:53.063 Executor task launch worker for task 30 DEBUG BlockManager: Put block rdd_24_7 locally took 5 ms 20/08/08 15:33:53.063 dag-scheduler-event-loop INFO DAGScheduler: failed: Set() 20/08/08 15:33:53.063 Executor task launch worker for task 30 DEBUG BlockManager: Putting block rdd_24_7 without replication took 6 ms 20/08/08 15:33:53.063 Executor task launch worker for task 30 DEBUG BlockManager: Getting local block rdd_24_7 20/08/08 15:33:53.063 Executor task launch worker for task 30 DEBUG BlockManager: Level for block rdd_24_7 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:53.063 dag-scheduler-event-loop DEBUG MapOutputTrackerMaster: Increasing epoch to 1 20/08/08 15:33:53.064 Executor task launch worker for task 30 INFO MemoryStore: Block rdd_25_7 stored as values in memory (estimated size 16.0 B, free 366.1 MiB) 20/08/08 15:33:53.064 Executor task launch worker for task 29 INFO MemoryStore: Block rdd_25_6 stored as values in memory (estimated size 16.0 B, free 366.1 MiB) 20/08/08 15:33:53.064 Executor task launch worker for task 31 DEBUG BlockManager: Block rdd_24_8 is unknown by block manager master 20/08/08 15:33:53.064 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_25_7 for BlockManagerId(driver, 192.168.1.225, 38159, None) 20/08/08 15:33:53.064 Executor task launch worker for task 31 DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 2 20/08/08 15:33:53.064 Executor task launch worker for task 31 DEBUG MapOutputTrackerMaster: Convert map statuses for shuffle 2, mappers 0-4, partitions 8-9 20/08/08 15:33:53.064 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_25_7 in memory on 192.168.1.225:38159 (size: 16.0 B, free: 366.2 MiB) 20/08/08 15:33:53.064 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_25_6 for BlockManagerId(driver, 192.168.1.225, 38159, None) 20/08/08 15:33:53.064 Executor task launch worker for task 31 DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 20/08/08 15:33:53.065 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_25_6 in memory on 192.168.1.225:38159 (size: 16.0 B, free: 366.2 MiB) 20/08/08 15:33:53.065 Executor task launch worker for task 31 INFO ShuffleBlockFetcherIterator: Getting 0 (0.0 B) non-empty blocks including 0 (0.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 20/08/08 15:33:53.065 Executor task launch worker for task 31 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 20/08/08 15:33:53.065 Executor task launch worker for task 31 DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: 20/08/08 15:33:53.065 Executor task launch worker for task 29 DEBUG BlockManagerMaster: Updated info of block rdd_25_6 20/08/08 15:33:53.065 Executor task launch worker for task 29 DEBUG BlockManager: Told master about block rdd_25_6 20/08/08 15:33:53.065 Executor task launch worker for task 29 DEBUG BlockManager: Put block rdd_25_6 locally took 18 ms 20/08/08 15:33:53.065 Executor task launch worker for task 30 DEBUG BlockManagerMaster: Updated info of block rdd_25_7 20/08/08 15:33:53.065 Executor task launch worker for task 30 DEBUG BlockManager: Told master about block rdd_25_7 20/08/08 15:33:53.065 Executor task launch worker for task 30 DEBUG BlockManager: Put block rdd_25_7 locally took 19 ms 20/08/08 15:33:53.065 Executor task launch worker for task 30 DEBUG BlockManager: Putting block rdd_25_7 without replication took 19 ms 20/08/08 15:33:53.065 Executor task launch worker for task 30 DEBUG BlockManager: Getting local block rdd_25_7 20/08/08 15:33:53.065 Executor task launch worker for task 30 DEBUG BlockManager: Level for block rdd_25_7 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:53.066 dag-scheduler-event-loop DEBUG DAGScheduler: submitStage(ResultStage 2 (name=collectAsMap at ClusteringMetrics.scala:515;jobs=1)) 20/08/08 15:33:53.066 Executor task launch worker for task 30 INFO Executor: Finished task 7.0 in stage 6.0 (TID 30). 2170 bytes result sent to driver 20/08/08 15:33:53.066 Executor task launch worker for task 30 DEBUG ExecutorMetricsPoller: stageTCMP: (6, 0) -> 3 20/08/08 15:33:53.065 Executor task launch worker for task 31 DEBUG ShuffleBlockFetcherIterator: Got local blocks in 0 ms 20/08/08 15:33:53.066 dag-scheduler-event-loop DEBUG DAGScheduler: missing: List() 20/08/08 15:33:53.067 dag-scheduler-event-loop INFO DAGScheduler: Submitting ResultStage 2 (ShuffledRDD[12] at aggregateByKey at ClusteringMetrics.scala:501), which has no missing parents 20/08/08 15:33:53.067 dag-scheduler-event-loop DEBUG DAGScheduler: submitMissingTasks(ResultStage 2) 20/08/08 15:33:53.067 dispatcher-event-loop-2 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_6.0, runningTasks: 3 20/08/08 15:33:53.067 dispatcher-event-loop-2 DEBUG TaskSetManager: No tasks for locality level NODE_LOCAL, so moving to locality level NO_PREF 20/08/08 15:33:53.067 dispatcher-event-loop-2 DEBUG TaskSetManager: No tasks for locality level NO_PREF, so moving to locality level ANY 20/08/08 15:33:53.067 Executor task launch worker for task 31 DEBUG TaskMemoryManager: Task 31 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@6b376f8 20/08/08 15:33:53.067 Executor task launch worker for task 31 INFO MemoryStore: Block rdd_24_8 stored as values in memory (estimated size 16.0 B, free 366.1 MiB) 20/08/08 15:33:53.068 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_24_8 for BlockManagerId(driver, 192.168.1.225, 38159, None) 20/08/08 15:33:53.068 task-result-getter-2 INFO TaskSetManager: Finished task 7.0 in stage 6.0 (TID 30) in 40 ms on 192.168.1.225 (executor driver) (7/10) 20/08/08 15:33:53.065 Executor task launch worker for task 29 DEBUG BlockManager: Putting block rdd_25_6 without replication took 19 ms 20/08/08 15:33:53.068 Executor task launch worker for task 29 DEBUG BlockManager: Getting local block rdd_25_6 20/08/08 15:33:53.068 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_24_8 in memory on 192.168.1.225:38159 (size: 16.0 B, free: 366.2 MiB) 20/08/08 15:33:53.068 Executor task launch worker for task 31 DEBUG BlockManagerMaster: Updated info of block rdd_24_8 20/08/08 15:33:53.068 Executor task launch worker for task 31 DEBUG BlockManager: Told master about block rdd_24_8 20/08/08 15:33:53.068 Executor task launch worker for task 31 DEBUG BlockManager: Put block rdd_24_8 locally took 4 ms 20/08/08 15:33:53.069 Executor task launch worker for task 29 DEBUG BlockManager: Level for block rdd_25_6 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:53.069 Executor task launch worker for task 31 DEBUG BlockManager: Putting block rdd_24_8 without replication took 4 ms 20/08/08 15:33:53.069 Executor task launch worker for task 31 DEBUG BlockManager: Getting local block rdd_24_8 20/08/08 15:33:53.069 Executor task launch worker for task 32 DEBUG BlockManager: Getting local block rdd_25_9 20/08/08 15:33:53.069 Executor task launch worker for task 31 DEBUG BlockManager: Level for block rdd_24_8 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:53.069 Executor task launch worker for task 32 DEBUG BlockManager: Block rdd_25_9 was not found 20/08/08 15:33:53.069 Executor task launch worker for task 32 DEBUG BlockManager: Getting remote block rdd_25_9 20/08/08 15:33:53.069 Executor task launch worker for task 32 DEBUG BlockManager: Block rdd_25_9 is unknown by block manager master 20/08/08 15:33:53.071 Executor task launch worker for task 29 INFO Executor: Finished task 6.0 in stage 6.0 (TID 29). 2170 bytes result sent to driver 20/08/08 15:33:53.073 Executor task launch worker for task 32 DEBUG BlockManager: Getting local block rdd_24_9 20/08/08 15:33:53.073 Executor task launch worker for task 32 DEBUG BlockManager: Block rdd_24_9 was not found 20/08/08 15:33:53.073 Executor task launch worker for task 32 DEBUG BlockManager: Getting remote block rdd_24_9 20/08/08 15:33:53.073 Executor task launch worker for task 31 INFO MemoryStore: Block rdd_25_8 stored as values in memory (estimated size 16.0 B, free 366.1 MiB) 20/08/08 15:33:53.075 Thread-4 DEBUG CodeGenerator: /* 001 */ public Object generate(Object[] references) { /* 002 */ return new GeneratedIteratorForCodegenStage1(references); /* 003 */ } /* 004 */ /* 005 */ // codegenStageId=1 /* 006 */ final class GeneratedIteratorForCodegenStage1 extends org.apache.spark.sql.execution.BufferedRowIterator { /* 007 */ private Object[] references; /* 008 */ private scala.collection.Iterator[] inputs; /* 009 */ private scala.collection.Iterator rdd_input_0; /* 010 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[] rdd_mutableStateArray_0 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[3]; /* 011 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[] rdd_mutableStateArray_1 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[4]; /* 012 */ /* 013 */ public GeneratedIteratorForCodegenStage1(Object[] references) { /* 014 */ this.references = references; /* 015 */ } /* 016 */ /* 017 */ public void init(int index, scala.collection.Iterator[] inputs) { /* 018 */ partitionIndex = index; /* 019 */ this.inputs = inputs; /* 020 */ rdd_input_0 = inputs[0]; /* 021 */ rdd_mutableStateArray_0[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(3, 64); /* 022 */ rdd_mutableStateArray_1[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[0], 8); /* 023 */ rdd_mutableStateArray_0[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(rdd_mutableStateArray_0[0], 4); /* 024 */ rdd_mutableStateArray_1[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[1], 4); /* 025 */ rdd_mutableStateArray_1[2] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[1], 8); /* 026 */ rdd_mutableStateArray_0[2] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(1, 32); /* 027 */ rdd_mutableStateArray_1[3] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[2], 8); /* 028 */ /* 029 */ } /* 030 */ /* 031 */ protected void processNext() throws java.io.IOException { /* 032 */ while ( rdd_input_0.hasNext()) { /* 033 */ InternalRow rdd_row_0 = (InternalRow) rdd_input_0.next(); /* 034 */ ((org.apache.spark.sql.execution.metric.SQLMetric) references[0] /* numOutputRows */).add(1); /* 035 */ boolean rdd_isNull_1 = rdd_row_0.isNullAt(1); /* 036 */ ArrayData rdd_value_1 = rdd_isNull_1 ? /* 037 */ null : (rdd_row_0.getArray(1)); /* 038 */ rdd_mutableStateArray_0[2].reset(); /* 039 */ /* 040 */ rdd_mutableStateArray_0[2].zeroOutNullBytes(); /* 041 */ /* 042 */ if (rdd_isNull_1) { /* 043 */ rdd_mutableStateArray_0[2].setNullAt(0); /* 044 */ } else { /* 045 */ // Remember the current cursor so that we can calculate how many bytes are /* 046 */ // written later. /* 047 */ final int project_previousCursor_0 = rdd_mutableStateArray_0[2].cursor(); /* 048 */ /* 049 */ final ArrayData project_tmpInput_0 = rdd_value_1; /* 050 */ if (project_tmpInput_0 instanceof UnsafeArrayData) { /* 051 */ rdd_mutableStateArray_0[2].write((UnsafeArrayData) project_tmpInput_0); /* 052 */ } else { /* 053 */ final int project_numElements_0 = project_tmpInput_0.numElements(); /* 054 */ rdd_mutableStateArray_1[3].initialize(project_numElements_0); /* 055 */ /* 056 */ for (int project_index_0 = 0; project_index_0 < project_numElements_0; project_index_0++) { /* 057 */ if (project_tmpInput_0.isNullAt(project_index_0)) { /* 058 */ rdd_mutableStateArray_1[3].setNull8Bytes(project_index_0); /* 059 */ } else { /* 060 */ rdd_mutableStateArray_1[3].write(project_index_0, project_tmpInput_0.getUTF8String(project_index_0)); /* 061 */ } /* 062 */ /* 063 */ } /* 064 */ } /* 065 */ /* 066 */ rdd_mutableStateArray_0[2].setOffsetAndSizeFromPreviousCursor(0, project_previousCursor_0); /* 067 */ } /* 068 */ append((rdd_mutableStateArray_0[2].getRow())); /* 069 */ if (shouldStop()) return; /* 070 */ } /* 071 */ } /* 072 */ /* 073 */ } 20/08/08 15:33:53.078 Executor task launch worker for task 29 DEBUG ExecutorMetricsPoller: stageTCMP: (6, 0) -> 2 20/08/08 15:33:53.078 dispatcher-event-loop-1 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_6.0, runningTasks: 2 20/08/08 15:33:53.079 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_25_8 for BlockManagerId(driver, 192.168.1.225, 38159, None) 20/08/08 15:33:53.079 task-result-getter-0 INFO TaskSetManager: Finished task 6.0 in stage 6.0 (TID 29) in 56 ms on 192.168.1.225 (executor driver) (8/10) 20/08/08 15:33:53.079 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_25_8 in memory on 192.168.1.225:38159 (size: 16.0 B, free: 366.2 MiB) 20/08/08 15:33:53.079 Executor task launch worker for task 31 DEBUG BlockManagerMaster: Updated info of block rdd_25_8 20/08/08 15:33:53.079 Executor task launch worker for task 31 DEBUG BlockManager: Told master about block rdd_25_8 20/08/08 15:33:53.080 Executor task launch worker for task 31 DEBUG BlockManager: Put block rdd_25_8 locally took 17 ms 20/08/08 15:33:53.080 Executor task launch worker for task 31 DEBUG BlockManager: Putting block rdd_25_8 without replication took 17 ms 20/08/08 15:33:53.080 Executor task launch worker for task 31 DEBUG BlockManager: Getting local block rdd_25_8 20/08/08 15:33:53.080 Executor task launch worker for task 31 DEBUG BlockManager: Level for block rdd_25_8 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:53.080 Executor task launch worker for task 32 DEBUG BlockManager: Block rdd_24_9 is unknown by block manager master 20/08/08 15:33:53.080 Executor task launch worker for task 32 DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 2 20/08/08 15:33:53.080 Executor task launch worker for task 32 DEBUG MapOutputTrackerMaster: Convert map statuses for shuffle 2, mappers 0-4, partitions 9-10 20/08/08 15:33:53.081 Executor task launch worker for task 32 DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 20/08/08 15:33:53.081 Executor task launch worker for task 31 INFO Executor: Finished task 8.0 in stage 6.0 (TID 31). 2170 bytes result sent to driver 20/08/08 15:33:53.081 Executor task launch worker for task 31 DEBUG ExecutorMetricsPoller: stageTCMP: (6, 0) -> 1 20/08/08 15:33:53.081 Executor task launch worker for task 32 INFO ShuffleBlockFetcherIterator: Getting 0 (0.0 B) non-empty blocks including 0 (0.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 20/08/08 15:33:53.081 Executor task launch worker for task 32 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 20/08/08 15:33:53.081 Executor task launch worker for task 32 DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: 20/08/08 15:33:53.081 Executor task launch worker for task 32 DEBUG ShuffleBlockFetcherIterator: Got local blocks in 0 ms 20/08/08 15:33:53.081 dispatcher-event-loop-3 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_6.0, runningTasks: 1 20/08/08 15:33:53.082 Executor task launch worker for task 32 DEBUG TaskMemoryManager: Task 32 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@709697ef 20/08/08 15:33:53.082 task-result-getter-1 INFO TaskSetManager: Finished task 8.0 in stage 6.0 (TID 31) in 31 ms on 192.168.1.225 (executor driver) (9/10) 20/08/08 15:33:53.083 Executor task launch worker for task 32 INFO MemoryStore: Block rdd_24_9 stored as values in memory (estimated size 16.0 B, free 366.1 MiB) 20/08/08 15:33:53.083 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_24_9 for BlockManagerId(driver, 192.168.1.225, 38159, None) 20/08/08 15:33:53.084 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_24_9 in memory on 192.168.1.225:38159 (size: 16.0 B, free: 366.2 MiB) 20/08/08 15:33:53.084 Executor task launch worker for task 32 DEBUG BlockManagerMaster: Updated info of block rdd_24_9 20/08/08 15:33:53.084 Executor task launch worker for task 32 DEBUG BlockManager: Told master about block rdd_24_9 20/08/08 15:33:53.084 Executor task launch worker for task 32 DEBUG BlockManager: Put block rdd_24_9 locally took 4 ms 20/08/08 15:33:53.084 Executor task launch worker for task 32 DEBUG BlockManager: Putting block rdd_24_9 without replication took 4 ms 20/08/08 15:33:53.084 Executor task launch worker for task 32 DEBUG BlockManager: Getting local block rdd_24_9 20/08/08 15:33:53.084 Executor task launch worker for task 32 DEBUG BlockManager: Level for block rdd_24_9 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:53.085 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_2 stored as values in memory (estimated size 4.5 KiB, free 366.3 MiB) 20/08/08 15:33:53.085 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_2 locally took 1 ms 20/08/08 15:33:53.085 Executor task launch worker for task 32 INFO MemoryStore: Block rdd_25_9 stored as values in memory (estimated size 16.0 B, free 366.1 MiB) 20/08/08 15:33:53.085 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_2 without replication took 1 ms 20/08/08 15:33:53.085 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_25_9 for BlockManagerId(driver, 192.168.1.225, 38159, None) 20/08/08 15:33:53.086 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_25_9 in memory on 192.168.1.225:38159 (size: 16.0 B, free: 366.2 MiB) 20/08/08 15:33:53.086 Executor task launch worker for task 32 DEBUG BlockManagerMaster: Updated info of block rdd_25_9 20/08/08 15:33:53.086 Executor task launch worker for task 32 DEBUG BlockManager: Told master about block rdd_25_9 20/08/08 15:33:53.086 Executor task launch worker for task 32 DEBUG BlockManager: Put block rdd_25_9 locally took 13 ms 20/08/08 15:33:53.086 Executor task launch worker for task 32 DEBUG BlockManager: Putting block rdd_25_9 without replication took 13 ms 20/08/08 15:33:53.086 Executor task launch worker for task 32 DEBUG BlockManager: Getting local block rdd_25_9 20/08/08 15:33:53.086 Executor task launch worker for task 32 DEBUG BlockManager: Level for block rdd_25_9 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:53.086 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_2_piece0 stored as bytes in memory (estimated size 2.5 KiB, free 366.3 MiB) 20/08/08 15:33:53.087 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master broadcast_2_piece0 for BlockManagerId(driver, 192.168.1.225, 36391, None) 20/08/08 15:33:53.087 Executor task launch worker for task 32 INFO Executor: Finished task 9.0 in stage 6.0 (TID 32). 2170 bytes result sent to driver 20/08/08 15:33:53.087 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added broadcast_2_piece0 in memory on 192.168.1.225:36391 (size: 2.5 KiB, free: 366.3 MiB) 20/08/08 15:33:53.088 dispatcher-event-loop-0 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_6.0, runningTasks: 0 20/08/08 15:33:53.088 dag-scheduler-event-loop DEBUG BlockManagerMaster: Updated info of block broadcast_2_piece0 20/08/08 15:33:53.088 Executor task launch worker for task 32 DEBUG ExecutorMetricsPoller: removing (6, 0) from stageTCMP 20/08/08 15:33:53.088 dag-scheduler-event-loop DEBUG BlockManager: Told master about block broadcast_2_piece0 20/08/08 15:33:53.088 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_2_piece0 locally took 2 ms 20/08/08 15:33:53.088 task-result-getter-3 INFO TaskSetManager: Finished task 9.0 in stage 6.0 (TID 32) in 30 ms on 192.168.1.225 (executor driver) (10/10) 20/08/08 15:33:53.088 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_2_piece0 without replication took 2 ms 20/08/08 15:33:53.088 task-result-getter-3 INFO TaskSchedulerImpl: Removed TaskSet 6.0, whose tasks have all completed, from pool 20/08/08 15:33:53.089 dag-scheduler-event-loop INFO SparkContext: Created broadcast 2 from broadcast at DAGScheduler.scala:1348 20/08/08 15:33:53.089 dag-scheduler-event-loop INFO DAGScheduler: ResultStage 6 (count at ALS.scala:967) finished in 0.153 s 20/08/08 15:33:53.089 dag-scheduler-event-loop INFO DAGScheduler: Submitting 4 missing tasks from ResultStage 2 (ShuffledRDD[12] at aggregateByKey at ClusteringMetrics.scala:501) (first 15 tasks are for partitions Vector(0, 1, 2, 3)) 20/08/08 15:33:53.089 dag-scheduler-event-loop DEBUG DAGScheduler: After removal of stage 5, remaining stages = 2 20/08/08 15:33:53.089 dag-scheduler-event-loop INFO TaskSchedulerImpl: Adding task set 2.0 with 4 tasks resource profile 0 20/08/08 15:33:53.090 dag-scheduler-event-loop DEBUG DAGScheduler: After removal of stage 4, remaining stages = 1 20/08/08 15:33:53.090 dag-scheduler-event-loop DEBUG DAGScheduler: After removal of stage 6, remaining stages = 0 20/08/08 15:33:53.090 dag-scheduler-event-loop INFO DAGScheduler: Job 2 is finished. Cancelling potential speculative or zombie tasks for this job 20/08/08 15:33:53.090 dag-scheduler-event-loop INFO TaskSchedulerImpl: Killing all running tasks in stage 6: Stage finished 20/08/08 15:33:53.090 Thread-4 INFO DAGScheduler: Job 2 finished: count at ALS.scala:967, took 0.241077 s 20/08/08 15:33:53.092 Thread-4 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$initialize$1 20/08/08 15:33:53.094 Thread-4 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$initialize$1) is now cleaned +++ 20/08/08 15:33:53.096 Thread-4 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$initialize$1 20/08/08 15:33:53.097 Thread-4 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$initialize$1) is now cleaned +++ 20/08/08 15:33:53.103 dag-scheduler-event-loop DEBUG TaskSetManager: Epoch for TaskSet 2.0: 1 20/08/08 15:33:53.107 Thread-4 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$cogroup$4 20/08/08 15:33:53.109 dag-scheduler-event-loop DEBUG TaskSetManager: Adding pending tasks took 6 ms 20/08/08 15:33:53.110 dag-scheduler-event-loop DEBUG TaskSetManager: Valid locality levels for TaskSet 2.0: NODE_LOCAL, NO_PREF, ANY 20/08/08 15:33:53.111 dispatcher-event-loop-1 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_2.0, runningTasks: 0 20/08/08 15:33:53.111 Thread-4 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$cogroup$4) is now cleaned +++ 20/08/08 15:33:53.113 Thread-4 INFO CodeGenerator: Code generated in 50.620736 ms 20/08/08 15:33:53.113 dispatcher-event-loop-1 INFO TaskSetManager: Starting task 0.0 in stage 2.0 (TID 5, 192.168.1.225, executor driver, partition 0, NODE_LOCAL, 7143 bytes) taskResourceAssignments Map() 20/08/08 15:33:53.113 dispatcher-event-loop-1 DEBUG TaskSetManager: No tasks for locality level NODE_LOCAL, so moving to locality level NO_PREF 20/08/08 15:33:53.114 Thread-4 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$doExecute$4$adapted 20/08/08 15:33:53.114 dispatcher-event-loop-1 DEBUG TaskSetManager: Moving to ANY after waiting for 0ms 20/08/08 15:33:53.114 dispatcher-event-loop-1 INFO TaskSetManager: Starting task 1.0 in stage 2.0 (TID 6, 192.168.1.225, executor driver, partition 1, PROCESS_LOCAL, 7143 bytes) taskResourceAssignments Map() 20/08/08 15:33:53.115 dispatcher-event-loop-1 INFO TaskSetManager: Starting task 2.0 in stage 2.0 (TID 7, 192.168.1.225, executor driver, partition 2, PROCESS_LOCAL, 7143 bytes) taskResourceAssignments Map() 20/08/08 15:33:53.115 Thread-4 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$join$2 20/08/08 15:33:53.115 dispatcher-event-loop-1 INFO TaskSetManager: Starting task 3.0 in stage 2.0 (TID 8, 192.168.1.225, executor driver, partition 3, PROCESS_LOCAL, 7143 bytes) taskResourceAssignments Map() 20/08/08 15:33:53.115 Thread-4 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$doExecute$4$adapted) is now cleaned +++ 20/08/08 15:33:53.116 Executor task launch worker for task 5 INFO Executor: Running task 0.0 in stage 2.0 (TID 5) 20/08/08 15:33:53.116 Executor task launch worker for task 7 INFO Executor: Running task 2.0 in stage 2.0 (TID 7) 20/08/08 15:33:53.116 Executor task launch worker for task 6 INFO Executor: Running task 1.0 in stage 2.0 (TID 6) 20/08/08 15:33:53.116 Executor task launch worker for task 8 INFO Executor: Running task 3.0 in stage 2.0 (TID 8) 20/08/08 15:33:53.121 Executor task launch worker for task 8 DEBUG ExecutorMetricsPoller: stageTCMP: (2, 0) -> 1 20/08/08 15:33:53.121 Thread-4 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$rdd$1 20/08/08 15:33:53.122 Executor task launch worker for task 8 DEBUG BlockManager: Getting local block broadcast_2 20/08/08 15:33:53.122 Thread-4 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$join$2) is now cleaned +++ 20/08/08 15:33:53.122 Executor task launch worker for task 8 DEBUG BlockManager: Level for block broadcast_2 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:53.125 Thread-4 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$rdd$1) is now cleaned +++ 20/08/08 15:33:53.125 Executor task launch worker for task 5 DEBUG ExecutorMetricsPoller: stageTCMP: (2, 0) -> 2 20/08/08 15:33:53.130 Executor task launch worker for task 6 DEBUG ExecutorMetricsPoller: stageTCMP: (2, 0) -> 3 20/08/08 15:33:53.130 Executor task launch worker for task 7 DEBUG ExecutorMetricsPoller: stageTCMP: (2, 0) -> 4 20/08/08 15:33:53.133 Thread-4 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$fit$2 20/08/08 15:33:53.134 Thread-4 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$fit$2) is now cleaned +++ 20/08/08 15:33:53.137 Executor task launch worker for task 6 DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 0 20/08/08 15:33:53.140 Executor task launch worker for task 6 DEBUG MapOutputTrackerMaster: Convert map statuses for shuffle 0, mappers 0-4, partitions 1-2 20/08/08 15:33:53.140 Executor task launch worker for task 5 DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 0 20/08/08 15:33:53.140 Executor task launch worker for task 5 DEBUG MapOutputTrackerMaster: Convert map statuses for shuffle 0, mappers 0-4, partitions 0-1 20/08/08 15:33:53.133 Executor task launch worker for task 8 DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 0 20/08/08 15:33:53.141 Executor task launch worker for task 8 DEBUG MapOutputTrackerMaster: Convert map statuses for shuffle 0, mappers 0-4, partitions 3-4 20/08/08 15:33:53.141 Executor task launch worker for task 7 DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 0 20/08/08 15:33:53.141 Executor task launch worker for task 7 DEBUG MapOutputTrackerMaster: Convert map statuses for shuffle 0, mappers 0-4, partitions 2-3 20/08/08 15:33:53.143 Thread-4 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$fit$4 20/08/08 15:33:53.144 Thread-4 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$fit$4) is now cleaned +++ 20/08/08 15:33:53.149 Thread-4 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$computeFactors$1 20/08/08 15:33:53.151 Thread-4 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$computeFactors$1) is now cleaned +++ 20/08/08 15:33:53.154 Thread-4 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$groupByKey$2 20/08/08 15:33:53.158 Thread-4 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$groupByKey$2) is now cleaned +++ 20/08/08 15:33:53.160 Thread-4 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$reduceByKey$2 20/08/08 15:33:53.159 Executor task launch worker for task 6 DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 20/08/08 15:33:53.159 Executor task launch worker for task 7 DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 20/08/08 15:33:53.160 Executor task launch worker for task 8 DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 20/08/08 15:33:53.161 Thread-4 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$reduceByKey$2) is now cleaned +++ 20/08/08 15:33:53.162 Thread-4 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$groupByKey$3 20/08/08 15:33:53.165 Executor task launch worker for task 8 INFO ShuffleBlockFetcherIterator: Getting 0 (0.0 B) non-empty blocks including 0 (0.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 20/08/08 15:33:53.166 Thread-4 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$groupByKey$3) is now cleaned +++ 20/08/08 15:33:53.166 Thread-4 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$groupByKey$4 20/08/08 15:33:53.167 Executor task launch worker for task 8 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 9 ms 20/08/08 15:33:53.167 Executor task launch worker for task 8 DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: 20/08/08 15:33:53.168 Executor task launch worker for task 5 DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 20/08/08 15:33:53.168 Thread-4 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$fit$9 20/08/08 15:33:53.169 Executor task launch worker for task 8 DEBUG ShuffleBlockFetcherIterator: Got local blocks in 10 ms 20/08/08 15:33:53.169 Executor task launch worker for task 6 INFO ShuffleBlockFetcherIterator: Getting 0 (0.0 B) non-empty blocks including 0 (0.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 20/08/08 15:33:53.169 Executor task launch worker for task 6 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 10 ms 20/08/08 15:33:53.169 Thread-4 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$groupByKey$4) is now cleaned +++ 20/08/08 15:33:53.169 Executor task launch worker for task 6 DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: 20/08/08 15:33:53.169 Executor task launch worker for task 6 DEBUG ShuffleBlockFetcherIterator: Got local blocks in 11 ms 20/08/08 15:33:53.169 Thread-4 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$fit$9) is now cleaned +++ 20/08/08 15:33:53.170 Thread-4 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$fit$9 20/08/08 15:33:53.171 Thread-4 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$cogroup$4 20/08/08 15:33:53.171 Thread-4 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$fit$9) is now cleaned +++ 20/08/08 15:33:53.173 Executor task launch worker for task 7 INFO ShuffleBlockFetcherIterator: Getting 0 (0.0 B) non-empty blocks including 0 (0.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 20/08/08 15:33:53.173 Thread-4 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$fit$11 20/08/08 15:33:53.173 Executor task launch worker for task 7 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 14 ms 20/08/08 15:33:53.173 Executor task launch worker for task 7 DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: 20/08/08 15:33:53.173 Executor task launch worker for task 7 DEBUG ShuffleBlockFetcherIterator: Got local blocks in 15 ms 20/08/08 15:33:53.178 Executor task launch worker for task 5 INFO ShuffleBlockFetcherIterator: Getting 4 (1246.0 B) non-empty blocks including 4 (1246.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 20/08/08 15:33:53.178 Executor task launch worker for task 5 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 19 ms 20/08/08 15:33:53.178 Executor task launch worker for task 5 DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_0_1_0,0), (shuffle_0_2_0,1), (shuffle_0_3_0,2), (shuffle_0_4_0,3) 20/08/08 15:33:53.179 Thread-4 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$cogroup$4) is now cleaned +++ 20/08/08 15:33:53.179 Executor task launch worker for task 5 INFO BlockManager: Getting local shuffle block shuffle_0_1_0 20/08/08 15:33:53.179 Thread-4 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$join$2 20/08/08 15:33:53.182 Executor task launch worker for task 6 DEBUG TaskMemoryManager: Task 6 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@291caa46 20/08/08 15:33:53.183 Thread-4 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$join$2) is now cleaned +++ 20/08/08 15:33:53.183 Thread-4 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$fit$11) is now cleaned +++ 20/08/08 15:33:53.183 Executor task launch worker for task 6 INFO Executor: Finished task 1.0 in stage 2.0 (TID 6). 1263 bytes result sent to driver 20/08/08 15:33:53.184 Executor task launch worker for task 7 DEBUG TaskMemoryManager: Task 7 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@4bfffecb 20/08/08 15:33:53.184 Executor task launch worker for task 8 DEBUG TaskMemoryManager: Task 8 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@62fc2a7b 20/08/08 15:33:53.185 Executor task launch worker for task 7 INFO Executor: Finished task 2.0 in stage 2.0 (TID 7). 1263 bytes result sent to driver 20/08/08 15:33:53.185 Executor task launch worker for task 8 INFO Executor: Finished task 3.0 in stage 2.0 (TID 8). 1263 bytes result sent to driver 20/08/08 15:33:53.186 dispatcher-event-loop-0 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_2.0, runningTasks: 3 20/08/08 15:33:53.186 dispatcher-event-loop-0 DEBUG TaskSetManager: No tasks for locality level NODE_LOCAL, so moving to locality level NO_PREF 20/08/08 15:33:53.186 dispatcher-event-loop-0 DEBUG TaskSetManager: No tasks for locality level NO_PREF, so moving to locality level ANY 20/08/08 15:33:53.187 Executor task launch worker for task 7 DEBUG ExecutorMetricsPoller: stageTCMP: (2, 0) -> 3 20/08/08 15:33:53.187 dispatcher-event-loop-2 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_2.0, runningTasks: 2 20/08/08 15:33:53.187 task-result-getter-1 INFO TaskSetManager: Finished task 1.0 in stage 2.0 (TID 6) in 73 ms on 192.168.1.225 (executor driver) (1/4) 20/08/08 15:33:53.188 Executor task launch worker for task 8 DEBUG ExecutorMetricsPoller: stageTCMP: (2, 0) -> 2 20/08/08 15:33:53.188 Executor task launch worker for task 6 DEBUG ExecutorMetricsPoller: stageTCMP: (2, 0) -> 1 20/08/08 15:33:53.188 dispatcher-event-loop-2 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_2.0, runningTasks: 1 20/08/08 15:33:53.189 task-result-getter-0 INFO TaskSetManager: Finished task 3.0 in stage 2.0 (TID 8) in 74 ms on 192.168.1.225 (executor driver) (2/4) 20/08/08 15:33:53.189 task-result-getter-2 INFO TaskSetManager: Finished task 2.0 in stage 2.0 (TID 7) in 74 ms on 192.168.1.225 (executor driver) (3/4) 20/08/08 15:33:53.190 Thread-4 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$computeFactors$4 20/08/08 15:33:53.191 Executor task launch worker for task 5 INFO BlockManager: Getting local shuffle block shuffle_0_2_0 20/08/08 15:33:53.191 Thread-4 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$count$1$adapted 20/08/08 15:33:53.192 Thread-4 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$computeFactors$4) is now cleaned +++ 20/08/08 15:33:53.192 Executor task launch worker for task 5 INFO BlockManager: Getting local shuffle block shuffle_0_3_0 20/08/08 15:33:53.192 Executor task launch worker for task 5 INFO BlockManager: Getting local shuffle block shuffle_0_4_0 20/08/08 15:33:53.192 Executor task launch worker for task 5 DEBUG ShuffleBlockFetcherIterator: Got local blocks in 34 ms 20/08/08 15:33:53.199 Thread-4 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$count$1$adapted) is now cleaned +++ 20/08/08 15:33:53.200 Thread-4 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$runJob$5 20/08/08 15:33:53.200 Thread-4 DEBUG CoGroupedRDD: Adding one-to-one dependency with itemInBlocks MapPartitionsRDD[24] at mapValues at ALS.scala:1627 20/08/08 15:33:53.200 Thread-4 DEBUG CoGroupedRDD: Adding one-to-one dependency with ShuffledRDD[32] at groupByKey at ALS.scala:1694 20/08/08 15:33:53.203 Thread-4 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$runJob$5) is now cleaned +++ 20/08/08 15:33:53.203 Thread-4 INFO SparkContext: Starting job: count at CountVectorizer.scala:233 20/08/08 15:33:53.208 dag-scheduler-event-loop DEBUG SortShuffleManager: Can't use serialized shuffle for shuffle 0 because the serializer, org.apache.spark.serializer.JavaSerializer, does not support object relocation 20/08/08 15:33:53.210 dag-scheduler-event-loop DEBUG DAGScheduler: Merging stage rdd profiles: Set() 20/08/08 15:33:53.212 dag-scheduler-event-loop DEBUG DAGScheduler: Merging stage rdd profiles: Set() 20/08/08 15:33:53.214 dag-scheduler-event-loop INFO DAGScheduler: Registering RDD 19 (flatMap at CountVectorizer.scala:212) as input to shuffle 0 20/08/08 15:33:53.214 Thread-4 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$cogroup$4 20/08/08 15:33:53.217 dag-scheduler-event-loop INFO DAGScheduler: Got job 2 (count at CountVectorizer.scala:233) with 4 output partitions 20/08/08 15:33:53.217 dag-scheduler-event-loop INFO DAGScheduler: Final stage: ResultStage 3 (count at CountVectorizer.scala:233) 20/08/08 15:33:53.217 dag-scheduler-event-loop INFO DAGScheduler: Parents of final stage: List(ShuffleMapStage 2) 20/08/08 15:33:53.217 Thread-4 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$cogroup$4) is now cleaned +++ 20/08/08 15:33:53.229 Thread-4 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$join$2 20/08/08 15:33:53.230 Executor task launch worker for task 5 DEBUG TaskMemoryManager: Task 5 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@22fd67df 20/08/08 15:33:53.231 Executor task launch worker for task 5 INFO Executor: Finished task 0.0 in stage 2.0 (TID 5). 1683 bytes result sent to driver 20/08/08 15:33:53.231 Executor task launch worker for task 5 DEBUG ExecutorMetricsPoller: removing (2, 0) from stageTCMP 20/08/08 15:33:53.232 dispatcher-event-loop-1 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_2.0, runningTasks: 0 20/08/08 15:33:53.232 task-result-getter-3 INFO TaskSetManager: Finished task 0.0 in stage 2.0 (TID 5) in 121 ms on 192.168.1.225 (executor driver) (4/4) 20/08/08 15:33:53.233 task-result-getter-3 INFO TaskSchedulerImpl: Removed TaskSet 2.0, whose tasks have all completed, from pool 20/08/08 15:33:53.233 dag-scheduler-event-loop INFO DAGScheduler: ResultStage 2 (collectAsMap at ClusteringMetrics.scala:515) finished in 0.150 s 20/08/08 15:33:53.234 dag-scheduler-event-loop DEBUG DAGScheduler: After removal of stage 2, remaining stages = 1 20/08/08 15:33:53.234 dag-scheduler-event-loop DEBUG DAGScheduler: After removal of stage 1, remaining stages = 0 20/08/08 15:33:53.234 dag-scheduler-event-loop INFO DAGScheduler: Job 1 is finished. Cancelling potential speculative or zombie tasks for this job 20/08/08 15:33:53.234 Thread-4 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$join$2) is now cleaned +++ 20/08/08 15:33:53.234 dag-scheduler-event-loop INFO TaskSchedulerImpl: Killing all running tasks in stage 2: Stage finished 20/08/08 15:33:53.234 Thread-4 INFO DAGScheduler: Job 1 finished: collectAsMap at ClusteringMetrics.scala:515, took 1.317073 s 20/08/08 15:33:53.237 dag-scheduler-event-loop INFO DAGScheduler: Missing parents: List(ShuffleMapStage 2) 20/08/08 15:33:53.237 dag-scheduler-event-loop DEBUG DAGScheduler: submitStage(ResultStage 3 (name=count at CountVectorizer.scala:233;jobs=2)) 20/08/08 15:33:53.238 dag-scheduler-event-loop DEBUG DAGScheduler: missing: List(ShuffleMapStage 2) 20/08/08 15:33:53.238 dag-scheduler-event-loop DEBUG DAGScheduler: submitStage(ShuffleMapStage 2 (name=flatMap at CountVectorizer.scala:212;jobs=2)) 20/08/08 15:33:53.239 Thread-4 INFO MemoryStore: Block broadcast_3 stored as values in memory (estimated size 424.0 B, free 366.3 MiB) 20/08/08 15:33:53.239 dag-scheduler-event-loop DEBUG DAGScheduler: missing: List() 20/08/08 15:33:53.239 dag-scheduler-event-loop INFO DAGScheduler: Submitting ShuffleMapStage 2 (MapPartitionsRDD[19] at flatMap at CountVectorizer.scala:212), which has no missing parents 20/08/08 15:33:53.239 Thread-4 DEBUG BlockManager: Put block broadcast_3 locally took 1 ms 20/08/08 15:33:53.239 dag-scheduler-event-loop DEBUG DAGScheduler: submitMissingTasks(ShuffleMapStage 2) 20/08/08 15:33:53.239 Thread-4 DEBUG BlockManager: Putting block broadcast_3 without replication took 1 ms 20/08/08 15:33:53.242 Thread-4 INFO MemoryStore: Block broadcast_3_piece0 stored as bytes in memory (estimated size 443.0 B, free 366.3 MiB) 20/08/08 15:33:53.242 Thread-4 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$computeFactors$1 20/08/08 15:33:53.242 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master broadcast_3_piece0 for BlockManagerId(driver, 192.168.1.225, 36391, None) 20/08/08 15:33:53.242 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added broadcast_3_piece0 in memory on 192.168.1.225:36391 (size: 443.0 B, free: 366.3 MiB) 20/08/08 15:33:53.243 Thread-4 DEBUG BlockManagerMaster: Updated info of block broadcast_3_piece0 20/08/08 15:33:53.243 Thread-4 DEBUG BlockManager: Told master about block broadcast_3_piece0 20/08/08 15:33:53.243 Thread-4 DEBUG BlockManager: Put block broadcast_3_piece0 locally took 1 ms 20/08/08 15:33:53.243 Thread-4 DEBUG BlockManager: Putting block broadcast_3_piece0 without replication took 1 ms 20/08/08 15:33:53.243 Thread-4 INFO SparkContext: Created broadcast 3 from broadcast at ClusteringMetrics.scala:579 20/08/08 15:33:53.244 Thread-4 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$computeFactors$1) is now cleaned +++ 20/08/08 15:33:53.246 Thread-4 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$groupByKey$2 20/08/08 15:33:53.251 Thread-4 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$groupByKey$2) is now cleaned +++ 20/08/08 15:33:53.251 Thread-4 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$groupByKey$3 20/08/08 15:33:53.255 Thread-4 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$groupByKey$3) is now cleaned +++ 20/08/08 15:33:53.256 Thread-4 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$groupByKey$4 20/08/08 15:33:53.259 Thread-4 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$groupByKey$4) is now cleaned +++ 20/08/08 15:33:53.261 Thread-4 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$cogroup$4 20/08/08 15:33:53.269 Thread-4 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$cogroup$4) is now cleaned +++ 20/08/08 15:33:53.270 Thread-4 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$join$2 20/08/08 15:33:53.274 Thread-4 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$join$2) is now cleaned +++ 20/08/08 15:33:53.274 Thread-4 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$computeFactors$4 20/08/08 15:33:53.276 Thread-4 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$computeFactors$4) is now cleaned +++ 20/08/08 15:33:53.277 Thread-4 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$train$11 20/08/08 15:33:53.279 Thread-4 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$train$11) is now cleaned +++ 20/08/08 15:33:53.281 Thread-4 DEBUG CoGroupedRDD: Adding one-to-one dependency with userInBlocks MapPartitionsRDD[19] at mapValues at ALS.scala:1627 20/08/08 15:33:53.281 Thread-4 DEBUG CoGroupedRDD: Adding one-to-one dependency with ShuffledRDD[41] at groupByKey at ALS.scala:1694 20/08/08 15:33:53.282 Thread-4 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$cogroup$4 20/08/08 15:33:53.285 Thread-4 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$cogroup$4) is now cleaned +++ 20/08/08 15:33:53.286 Thread-4 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$join$2 20/08/08 15:33:53.289 Thread-4 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$join$2) is now cleaned +++ 20/08/08 15:33:53.290 Thread-4 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$train$12 20/08/08 15:33:53.292 Thread-4 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$train$12) is now cleaned +++ 20/08/08 15:33:53.294 Thread-4 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$train$14 20/08/08 15:33:53.296 Thread-4 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$train$14) is now cleaned +++ 20/08/08 15:33:53.298 Thread-4 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$cogroup$4 20/08/08 15:33:53.301 Thread-4 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$cogroup$4) is now cleaned +++ 20/08/08 15:33:53.302 Thread-4 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$join$2 20/08/08 15:33:53.304 Thread-4 DEBUG Analyzer$ResolveReferences: Resolving 'normalizedFeatures to normalizedFeatures#11 20/08/08 15:33:53.304 Thread-4 DEBUG Analyzer$ResolveReferences: Resolving 'prediction to prediction#1 20/08/08 15:33:53.305 Thread-4 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$join$2) is now cleaned +++ 20/08/08 15:33:53.305 Thread-4 DEBUG Analyzer$ResolveReferences: Resolving 'weightCol to weightCol#5 20/08/08 15:33:53.305 Thread-4 DEBUG Analyzer$ResolveReferences: Resolving 'weightCol to weightCol#5 20/08/08 15:33:53.305 Thread-4 DEBUG Analyzer$ResolveReferences: Resolving 'weightCol to weightCol#5 20/08/08 15:33:53.306 Thread-4 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$train$15 20/08/08 15:33:53.308 Thread-4 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$train$15) is now cleaned +++ 20/08/08 15:33:53.309 Thread-4 DEBUG CoGroupedRDD: Adding one-to-one dependency with MapPartitionsRDD[46] at mapValues at ALS.scala:1040 20/08/08 15:33:53.309 Thread-4 DEBUG CoGroupedRDD: Adding one-to-one dependency with MapPartitionsRDD[45] at mapValues at ALS.scala:1702 20/08/08 15:33:53.310 Thread-4 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$count$1$adapted 20/08/08 15:33:53.315 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_4 stored as values in memory (estimated size 21.6 KiB, free 366.2 MiB) 20/08/08 15:33:53.315 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_4 locally took 1 ms 20/08/08 15:33:53.315 Thread-4 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$count$1$adapted) is now cleaned +++ 20/08/08 15:33:53.315 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_4 without replication took 1 ms 20/08/08 15:33:53.316 Thread-4 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$runJob$5 20/08/08 15:33:53.318 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_4_piece0 stored as bytes in memory (estimated size 10.6 KiB, free 366.2 MiB) 20/08/08 15:33:53.318 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master broadcast_4_piece0 for BlockManagerId(driver, 192.168.1.225, 36433, None) 20/08/08 15:33:53.319 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added broadcast_4_piece0 in memory on 192.168.1.225:36433 (size: 10.6 KiB, free: 366.3 MiB) 20/08/08 15:33:53.319 dag-scheduler-event-loop DEBUG BlockManagerMaster: Updated info of block broadcast_4_piece0 20/08/08 15:33:53.319 dag-scheduler-event-loop DEBUG BlockManager: Told master about block broadcast_4_piece0 20/08/08 15:33:53.320 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_4_piece0 locally took 3 ms 20/08/08 15:33:53.320 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_4_piece0 without replication took 3 ms 20/08/08 15:33:53.320 dag-scheduler-event-loop INFO SparkContext: Created broadcast 4 from broadcast at DAGScheduler.scala:1348 20/08/08 15:33:53.320 Thread-4 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$runJob$5) is now cleaned +++ 20/08/08 15:33:53.321 Thread-4 INFO SparkContext: Starting job: count at ALS.scala:1062 20/08/08 15:33:53.322 dag-scheduler-event-loop DEBUG DAGScheduler: Merging stage rdd profiles: Set() 20/08/08 15:33:53.322 dag-scheduler-event-loop DEBUG DAGScheduler: Merging stage rdd profiles: Set() 20/08/08 15:33:53.323 dag-scheduler-event-loop INFO DAGScheduler: Submitting 4 missing tasks from ShuffleMapStage 2 (MapPartitionsRDD[19] at flatMap at CountVectorizer.scala:212) (first 15 tasks are for partitions Vector(0, 1, 2, 3)) 20/08/08 15:33:53.323 dag-scheduler-event-loop INFO TaskSchedulerImpl: Adding task set 2.0 with 4 tasks resource profile 0 20/08/08 15:33:53.323 dag-scheduler-event-loop DEBUG DAGScheduler: Merging stage rdd profiles: Set() 20/08/08 15:33:53.323 dag-scheduler-event-loop DEBUG TaskSetManager: Epoch for TaskSet 2.0: 0 20/08/08 15:33:53.324 dag-scheduler-event-loop DEBUG TaskSetManager: Adding pending tasks took 0 ms 20/08/08 15:33:53.324 dag-scheduler-event-loop DEBUG TaskSetManager: Valid locality levels for TaskSet 2.0: NO_PREF, ANY 20/08/08 15:33:53.324 dag-scheduler-event-loop DEBUG CoGroupedRDD: Adding one-to-one dependency with itemOutBlocks MapPartitionsRDD[25] at mapValues at ALS.scala:1636 20/08/08 15:33:53.324 dag-scheduler-event-loop DEBUG CoGroupedRDD: Adding one-to-one dependency with MapPartitionsRDD[36] at mapValues at ALS.scala:1702 20/08/08 15:33:53.324 dispatcher-event-loop-2 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_2.0, runningTasks: 0 20/08/08 15:33:53.326 dispatcher-event-loop-2 INFO TaskSetManager: Starting task 0.0 in stage 2.0 (TID 8, 192.168.1.225, executor driver, partition 0, PROCESS_LOCAL, 7421 bytes) taskResourceAssignments Map() 20/08/08 15:33:53.326 dispatcher-event-loop-2 INFO TaskSetManager: Starting task 1.0 in stage 2.0 (TID 9, 192.168.1.225, executor driver, partition 1, PROCESS_LOCAL, 7385 bytes) taskResourceAssignments Map() 20/08/08 15:33:53.326 dispatcher-event-loop-2 INFO TaskSetManager: Starting task 2.0 in stage 2.0 (TID 10, 192.168.1.225, executor driver, partition 2, PROCESS_LOCAL, 7400 bytes) taskResourceAssignments Map() 20/08/08 15:33:53.327 dispatcher-event-loop-2 INFO TaskSetManager: Starting task 3.0 in stage 2.0 (TID 11, 192.168.1.225, executor driver, partition 3, PROCESS_LOCAL, 7382 bytes) taskResourceAssignments Map() 20/08/08 15:33:53.327 Executor task launch worker for task 8 INFO Executor: Running task 0.0 in stage 2.0 (TID 8) 20/08/08 15:33:53.327 Executor task launch worker for task 10 INFO Executor: Running task 2.0 in stage 2.0 (TID 10) 20/08/08 15:33:53.327 Executor task launch worker for task 9 INFO Executor: Running task 1.0 in stage 2.0 (TID 9) 20/08/08 15:33:53.328 dag-scheduler-event-loop DEBUG CoGroupedRDD: Adding one-to-one dependency with userOutBlocks MapPartitionsRDD[20] at mapValues at ALS.scala:1636 20/08/08 15:33:53.328 dag-scheduler-event-loop DEBUG CoGroupedRDD: Adding one-to-one dependency with MapPartitionsRDD[26] at mapPartitions at ALS.scala:1263 20/08/08 15:33:53.328 Executor task launch worker for task 9 DEBUG ExecutorMetricsPoller: stageTCMP: (2, 0) -> 1 20/08/08 15:33:53.327 Executor task launch worker for task 11 INFO Executor: Running task 3.0 in stage 2.0 (TID 11) 20/08/08 15:33:53.328 Executor task launch worker for task 10 DEBUG ExecutorMetricsPoller: stageTCMP: (2, 0) -> 2 20/08/08 15:33:53.330 Executor task launch worker for task 8 DEBUG ExecutorMetricsPoller: stageTCMP: (2, 0) -> 3 20/08/08 15:33:53.330 Executor task launch worker for task 11 DEBUG ExecutorMetricsPoller: stageTCMP: (2, 0) -> 4 20/08/08 15:33:53.331 Executor task launch worker for task 9 DEBUG BlockManager: Getting local block broadcast_4 20/08/08 15:33:53.331 Executor task launch worker for task 9 DEBUG BlockManager: Level for block broadcast_4 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:53.332 dag-scheduler-event-loop DEBUG DAGScheduler: Merging stage rdd profiles: Set() 20/08/08 15:33:53.333 dag-scheduler-event-loop INFO DAGScheduler: Registering RDD 31 (flatMap at ALS.scala:1688) as input to shuffle 4 20/08/08 15:33:53.333 dag-scheduler-event-loop DEBUG DAGScheduler: Merging stage rdd profiles: Set() 20/08/08 15:33:53.334 dag-scheduler-event-loop DEBUG DAGScheduler: Merging stage rdd profiles: Set() 20/08/08 15:33:53.335 dag-scheduler-event-loop INFO DAGScheduler: Registering RDD 40 (flatMap at ALS.scala:1688) as input to shuffle 3 20/08/08 15:33:53.335 dag-scheduler-event-loop INFO DAGScheduler: Got job 3 (count at ALS.scala:1062) with 10 output partitions 20/08/08 15:33:53.335 dag-scheduler-event-loop INFO DAGScheduler: Final stage: ResultStage 12 (count at ALS.scala:1062) 20/08/08 15:33:53.335 dag-scheduler-event-loop INFO DAGScheduler: Parents of final stage: List(ShuffleMapStage 11, ShuffleMapStage 8) 20/08/08 15:33:53.340 dag-scheduler-event-loop INFO DAGScheduler: Missing parents: List(ShuffleMapStage 11) 20/08/08 15:33:53.340 dag-scheduler-event-loop DEBUG DAGScheduler: submitStage(ResultStage 12 (name=count at ALS.scala:1062;jobs=3)) 20/08/08 15:33:53.341 dag-scheduler-event-loop DEBUG DAGScheduler: missing: List(ShuffleMapStage 11) 20/08/08 15:33:53.341 dag-scheduler-event-loop DEBUG DAGScheduler: submitStage(ShuffleMapStage 11 (name=flatMap at ALS.scala:1688;jobs=3)) 20/08/08 15:33:53.373 dag-scheduler-event-loop DEBUG DAGScheduler: missing: List(ShuffleMapStage 9) 20/08/08 15:33:53.373 dag-scheduler-event-loop DEBUG DAGScheduler: submitStage(ShuffleMapStage 9 (name=flatMap at ALS.scala:1688;jobs=3)) 20/08/08 15:33:53.375 dag-scheduler-event-loop DEBUG DAGScheduler: missing: List() 20/08/08 15:33:53.376 dag-scheduler-event-loop INFO DAGScheduler: Submitting ShuffleMapStage 9 (MapPartitionsRDD[31] at flatMap at ALS.scala:1688), which has no missing parents 20/08/08 15:33:53.376 dag-scheduler-event-loop DEBUG DAGScheduler: submitMissingTasks(ShuffleMapStage 9) 20/08/08 15:33:53.382 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_6 stored as values in memory (estimated size 34.6 KiB, free 366.0 MiB) 20/08/08 15:33:53.382 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_6 locally took 0 ms 20/08/08 15:33:53.382 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_6 without replication took 1 ms 20/08/08 15:33:53.383 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_6_piece0 stored as bytes in memory (estimated size 16.1 KiB, free 366.0 MiB) 20/08/08 15:33:53.383 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master broadcast_6_piece0 for BlockManagerId(driver, 192.168.1.225, 38159, None) 20/08/08 15:33:53.384 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added broadcast_6_piece0 in memory on 192.168.1.225:38159 (size: 16.1 KiB, free: 366.2 MiB) 20/08/08 15:33:53.384 dag-scheduler-event-loop DEBUG BlockManagerMaster: Updated info of block broadcast_6_piece0 20/08/08 15:33:53.384 dag-scheduler-event-loop DEBUG BlockManager: Told master about block broadcast_6_piece0 20/08/08 15:33:53.384 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_6_piece0 locally took 1 ms 20/08/08 15:33:53.384 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_6_piece0 without replication took 1 ms 20/08/08 15:33:53.385 dag-scheduler-event-loop INFO SparkContext: Created broadcast 6 from broadcast at DAGScheduler.scala:1348 20/08/08 15:33:53.385 dag-scheduler-event-loop INFO DAGScheduler: Submitting 10 missing tasks from ShuffleMapStage 9 (MapPartitionsRDD[31] at flatMap at ALS.scala:1688) (first 15 tasks are for partitions Vector(0, 1, 2, 3, 4, 5, 6, 7, 8, 9)) 20/08/08 15:33:53.385 dag-scheduler-event-loop INFO TaskSchedulerImpl: Adding task set 9.0 with 10 tasks resource profile 0 20/08/08 15:33:53.386 dag-scheduler-event-loop DEBUG TaskSetManager: Epoch for TaskSet 9.0: 3 20/08/08 15:33:53.386 dag-scheduler-event-loop DEBUG TaskSetManager: Adding pending tasks took 0 ms 20/08/08 15:33:53.386 dag-scheduler-event-loop DEBUG TaskSetManager: Valid locality levels for TaskSet 9.0: PROCESS_LOCAL, NODE_LOCAL, ANY 20/08/08 15:33:53.387 dispatcher-event-loop-3 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_9.0, runningTasks: 0 20/08/08 15:33:53.390 dispatcher-event-loop-3 INFO TaskSetManager: Starting task 0.0 in stage 9.0 (TID 33, 192.168.1.225, executor driver, partition 0, PROCESS_LOCAL, 7362 bytes) taskResourceAssignments Map() 20/08/08 15:33:53.390 dispatcher-event-loop-3 INFO TaskSetManager: Starting task 1.0 in stage 9.0 (TID 34, 192.168.1.225, executor driver, partition 1, PROCESS_LOCAL, 7362 bytes) taskResourceAssignments Map() 20/08/08 15:33:53.391 dispatcher-event-loop-3 INFO TaskSetManager: Starting task 2.0 in stage 9.0 (TID 35, 192.168.1.225, executor driver, partition 2, PROCESS_LOCAL, 7362 bytes) taskResourceAssignments Map() 20/08/08 15:33:53.391 dispatcher-event-loop-3 INFO TaskSetManager: Starting task 3.0 in stage 9.0 (TID 36, 192.168.1.225, executor driver, partition 3, PROCESS_LOCAL, 7362 bytes) taskResourceAssignments Map() 20/08/08 15:33:53.392 Executor task launch worker for task 33 INFO Executor: Running task 0.0 in stage 9.0 (TID 33) 20/08/08 15:33:53.392 Executor task launch worker for task 36 INFO Executor: Running task 3.0 in stage 9.0 (TID 36) 20/08/08 15:33:53.392 Executor task launch worker for task 35 INFO Executor: Running task 2.0 in stage 9.0 (TID 35) 20/08/08 15:33:53.392 Executor task launch worker for task 34 INFO Executor: Running task 1.0 in stage 9.0 (TID 34) 20/08/08 15:33:53.392 Executor task launch worker for task 34 DEBUG ExecutorMetricsPoller: stageTCMP: (9, 0) -> 1 20/08/08 15:33:53.393 Executor task launch worker for task 35 DEBUG ExecutorMetricsPoller: stageTCMP: (9, 0) -> 4 20/08/08 15:33:53.393 Executor task launch worker for task 33 DEBUG ExecutorMetricsPoller: stageTCMP: (9, 0) -> 2 20/08/08 15:33:53.393 Executor task launch worker for task 36 DEBUG ExecutorMetricsPoller: stageTCMP: (9, 0) -> 3 20/08/08 15:33:53.411 Executor task launch worker for task 34 DEBUG BlockManager: Getting local block broadcast_6 20/08/08 15:33:53.411 Executor task launch worker for task 34 DEBUG BlockManager: Level for block broadcast_6 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:53.442 Executor task launch worker for task 34 DEBUG BlockManager: Getting local block rdd_20_1 20/08/08 15:33:53.442 Executor task launch worker for task 34 DEBUG BlockManager: Level for block rdd_20_1 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:53.442 Executor task launch worker for task 34 INFO BlockManager: Found block rdd_20_1 locally 20/08/08 15:33:53.443 Executor task launch worker for task 34 DEBUG BlockManager: Getting local block rdd_19_1 20/08/08 15:33:53.443 Executor task launch worker for task 34 DEBUG BlockManager: Level for block rdd_19_1 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:53.443 Executor task launch worker for task 34 INFO BlockManager: Found block rdd_19_1 locally 20/08/08 15:33:53.444 Executor task launch worker for task 35 DEBUG BlockManager: Getting local block rdd_20_2 20/08/08 15:33:53.444 Executor task launch worker for task 35 DEBUG BlockManager: Level for block rdd_20_2 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:53.445 Executor task launch worker for task 35 INFO BlockManager: Found block rdd_20_2 locally 20/08/08 15:33:53.445 Executor task launch worker for task 35 DEBUG BlockManager: Getting local block rdd_19_2 20/08/08 15:33:53.445 Executor task launch worker for task 35 DEBUG BlockManager: Level for block rdd_19_2 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:53.445 Executor task launch worker for task 35 INFO BlockManager: Found block rdd_19_2 locally 20/08/08 15:33:53.450 Executor task launch worker for task 36 DEBUG BlockManager: Getting local block rdd_20_3 20/08/08 15:33:53.450 Executor task launch worker for task 36 DEBUG BlockManager: Level for block rdd_20_3 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:53.450 Executor task launch worker for task 36 INFO BlockManager: Found block rdd_20_3 locally 20/08/08 15:33:53.450 Executor task launch worker for task 36 DEBUG BlockManager: Getting local block rdd_19_3 20/08/08 15:33:53.450 Executor task launch worker for task 36 DEBUG BlockManager: Level for block rdd_19_3 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:53.450 Executor task launch worker for task 36 INFO BlockManager: Found block rdd_19_3 locally 20/08/08 15:33:53.451 Executor task launch worker for task 33 DEBUG BlockManager: Getting local block rdd_20_0 20/08/08 15:33:53.451 Executor task launch worker for task 33 DEBUG BlockManager: Level for block rdd_20_0 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:53.451 Executor task launch worker for task 33 INFO BlockManager: Found block rdd_20_0 locally 20/08/08 15:33:53.451 Executor task launch worker for task 33 DEBUG BlockManager: Getting local block rdd_19_0 20/08/08 15:33:53.452 Executor task launch worker for task 36 DEBUG TaskMemoryManager: Task 36 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@2456ee75 20/08/08 15:33:53.469 Executor task launch worker for task 33 DEBUG BlockManager: Level for block rdd_19_0 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:53.469 Executor task launch worker for task 33 INFO BlockManager: Found block rdd_19_0 locally 20/08/08 15:33:53.470 Executor task launch worker for task 36 INFO Executor: Finished task 3.0 in stage 9.0 (TID 36). 1950 bytes result sent to driver 20/08/08 15:33:53.470 Executor task launch worker for task 36 DEBUG ExecutorMetricsPoller: stageTCMP: (9, 0) -> 3 20/08/08 15:33:53.470 dispatcher-event-loop-0 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_9.0, runningTasks: 3 20/08/08 15:33:53.471 dispatcher-event-loop-0 INFO TaskSetManager: Starting task 4.0 in stage 9.0 (TID 37, 192.168.1.225, executor driver, partition 4, PROCESS_LOCAL, 7362 bytes) taskResourceAssignments Map() 20/08/08 15:33:53.471 task-result-getter-2 INFO TaskSetManager: Finished task 3.0 in stage 9.0 (TID 36) in 80 ms on 192.168.1.225 (executor driver) (1/10) 20/08/08 15:33:53.472 Executor task launch worker for task 37 INFO Executor: Running task 4.0 in stage 9.0 (TID 37) 20/08/08 15:33:53.472 Executor task launch worker for task 37 DEBUG ExecutorMetricsPoller: stageTCMP: (9, 0) -> 4 20/08/08 15:33:53.475 Executor task launch worker for task 35 WARN BLAS: Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS 20/08/08 15:33:53.476 Executor task launch worker for task 35 WARN BLAS: Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS 20/08/08 15:33:53.485 Executor task launch worker for task 37 DEBUG BlockManager: Getting local block rdd_20_4 20/08/08 15:33:53.486 Executor task launch worker for task 37 DEBUG BlockManager: Level for block rdd_20_4 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:53.486 Executor task launch worker for task 37 INFO BlockManager: Found block rdd_20_4 locally 20/08/08 15:33:53.486 Executor task launch worker for task 37 DEBUG BlockManager: Getting local block rdd_19_4 20/08/08 15:33:53.486 Executor task launch worker for task 37 DEBUG BlockManager: Level for block rdd_19_4 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:53.486 Executor task launch worker for task 37 INFO BlockManager: Found block rdd_19_4 locally 20/08/08 15:33:53.486 Executor task launch worker for task 37 DEBUG TaskMemoryManager: Task 37 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@16a023d0 20/08/08 15:33:53.488 Executor task launch worker for task 33 DEBUG TaskMemoryManager: Task 33 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@7f763680 20/08/08 15:33:53.488 Executor task launch worker for task 37 INFO Executor: Finished task 4.0 in stage 9.0 (TID 37). 1950 bytes result sent to driver 20/08/08 15:33:53.488 Executor task launch worker for task 37 DEBUG ExecutorMetricsPoller: stageTCMP: (9, 0) -> 3 20/08/08 15:33:53.489 dispatcher-event-loop-1 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_9.0, runningTasks: 3 20/08/08 15:33:53.489 dispatcher-event-loop-1 INFO TaskSetManager: Starting task 5.0 in stage 9.0 (TID 38, 192.168.1.225, executor driver, partition 5, PROCESS_LOCAL, 7362 bytes) taskResourceAssignments Map() 20/08/08 15:33:53.490 Executor task launch worker for task 38 INFO Executor: Running task 5.0 in stage 9.0 (TID 38) 20/08/08 15:33:53.490 task-result-getter-0 INFO TaskSetManager: Finished task 4.0 in stage 9.0 (TID 37) in 19 ms on 192.168.1.225 (executor driver) (2/10) 20/08/08 15:33:53.490 Executor task launch worker for task 38 DEBUG ExecutorMetricsPoller: stageTCMP: (9, 0) -> 4 20/08/08 15:33:53.491 dag-scheduler-event-loop DEBUG DAGScheduler: ShuffleMapTask finished on driver 20/08/08 15:33:53.492 dag-scheduler-event-loop DEBUG DAGScheduler: ShuffleMapTask finished on driver 20/08/08 15:33:53.497 Executor task launch worker for task 34 DEBUG TaskMemoryManager: Task 34 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@3682424d 20/08/08 15:33:53.500 Executor task launch worker for task 34 INFO Executor: Finished task 1.0 in stage 9.0 (TID 34). 2122 bytes result sent to driver 20/08/08 15:33:53.500 Executor task launch worker for task 34 DEBUG ExecutorMetricsPoller: stageTCMP: (9, 0) -> 3 20/08/08 15:33:53.500 dispatcher-event-loop-0 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_9.0, runningTasks: 3 20/08/08 15:33:53.501 dispatcher-event-loop-0 INFO TaskSetManager: Starting task 6.0 in stage 9.0 (TID 39, 192.168.1.225, executor driver, partition 6, PROCESS_LOCAL, 7362 bytes) taskResourceAssignments Map() 20/08/08 15:33:53.501 Executor task launch worker for task 38 DEBUG BlockManager: Getting local block rdd_20_5 20/08/08 15:33:53.501 Executor task launch worker for task 38 DEBUG BlockManager: Level for block rdd_20_5 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:53.501 Executor task launch worker for task 38 INFO BlockManager: Found block rdd_20_5 locally 20/08/08 15:33:53.501 Executor task launch worker for task 38 DEBUG BlockManager: Getting local block rdd_19_5 20/08/08 15:33:53.501 Executor task launch worker for task 38 DEBUG BlockManager: Level for block rdd_19_5 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:53.501 Executor task launch worker for task 39 INFO Executor: Running task 6.0 in stage 9.0 (TID 39) 20/08/08 15:33:53.501 task-result-getter-1 INFO TaskSetManager: Finished task 1.0 in stage 9.0 (TID 34) in 111 ms on 192.168.1.225 (executor driver) (3/10) 20/08/08 15:33:53.502 Executor task launch worker for task 39 DEBUG ExecutorMetricsPoller: stageTCMP: (9, 0) -> 4 20/08/08 15:33:53.501 Executor task launch worker for task 38 INFO BlockManager: Found block rdd_19_5 locally 20/08/08 15:33:53.506 Executor task launch worker for task 38 DEBUG TaskMemoryManager: Task 38 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@68ab5b78 20/08/08 15:33:53.517 Executor task launch worker for task 33 INFO Executor: Finished task 0.0 in stage 9.0 (TID 33). 2122 bytes result sent to driver 20/08/08 15:33:53.517 Executor task launch worker for task 33 DEBUG ExecutorMetricsPoller: stageTCMP: (9, 0) -> 3 20/08/08 15:33:53.518 Executor task launch worker for task 35 DEBUG TaskMemoryManager: Task 35 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@6dfbf378 20/08/08 15:33:53.519 Executor task launch worker for task 39 DEBUG BlockManager: Getting local block rdd_20_6 20/08/08 15:33:53.519 Executor task launch worker for task 39 DEBUG BlockManager: Level for block rdd_20_6 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:53.519 Executor task launch worker for task 39 INFO BlockManager: Found block rdd_20_6 locally 20/08/08 15:33:53.519 Executor task launch worker for task 39 DEBUG BlockManager: Getting local block rdd_19_6 20/08/08 15:33:53.519 Executor task launch worker for task 39 DEBUG BlockManager: Level for block rdd_19_6 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:53.519 Executor task launch worker for task 39 INFO BlockManager: Found block rdd_19_6 locally 20/08/08 15:33:53.520 Executor task launch worker for task 39 DEBUG TaskMemoryManager: Task 39 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@2eaaddbe 20/08/08 15:33:53.520 dispatcher-event-loop-1 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_9.0, runningTasks: 3 20/08/08 15:33:53.521 dag-scheduler-event-loop DEBUG DAGScheduler: ShuffleMapTask finished on driver 20/08/08 15:33:53.522 dispatcher-event-loop-1 INFO TaskSetManager: Starting task 7.0 in stage 9.0 (TID 40, 192.168.1.225, executor driver, partition 7, PROCESS_LOCAL, 7362 bytes) taskResourceAssignments Map() 20/08/08 15:33:53.523 task-result-getter-3 INFO TaskSetManager: Finished task 0.0 in stage 9.0 (TID 33) in 136 ms on 192.168.1.225 (executor driver) (4/10) 20/08/08 15:33:53.523 Executor task launch worker for task 40 INFO Executor: Running task 7.0 in stage 9.0 (TID 40) 20/08/08 15:33:53.524 Executor task launch worker for task 40 DEBUG ExecutorMetricsPoller: stageTCMP: (9, 0) -> 4 20/08/08 15:33:53.526 dag-scheduler-event-loop DEBUG DAGScheduler: ShuffleMapTask finished on driver 20/08/08 15:33:53.529 Executor task launch worker for task 40 DEBUG BlockManager: Getting local block rdd_20_7 20/08/08 15:33:53.529 Executor task launch worker for task 40 DEBUG BlockManager: Level for block rdd_20_7 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:53.529 Executor task launch worker for task 40 INFO BlockManager: Found block rdd_20_7 locally 20/08/08 15:33:53.529 Executor task launch worker for task 40 DEBUG BlockManager: Getting local block rdd_19_7 20/08/08 15:33:53.530 Executor task launch worker for task 40 DEBUG BlockManager: Level for block rdd_19_7 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:53.530 Executor task launch worker for task 40 INFO BlockManager: Found block rdd_19_7 locally 20/08/08 15:33:53.530 Executor task launch worker for task 40 DEBUG TaskMemoryManager: Task 40 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@5025ca9e 20/08/08 15:33:53.542 Executor task launch worker for task 40 INFO Executor: Finished task 7.0 in stage 9.0 (TID 40). 1950 bytes result sent to driver 20/08/08 15:33:53.542 Executor task launch worker for task 38 INFO Executor: Finished task 5.0 in stage 9.0 (TID 38). 1950 bytes result sent to driver 20/08/08 15:33:53.542 dispatcher-event-loop-0 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_9.0, runningTasks: 3 20/08/08 15:33:53.543 Executor task launch worker for task 39 INFO Executor: Finished task 6.0 in stage 9.0 (TID 39). 1950 bytes result sent to driver 20/08/08 15:33:53.543 dispatcher-event-loop-0 INFO TaskSetManager: Starting task 8.0 in stage 9.0 (TID 41, 192.168.1.225, executor driver, partition 8, PROCESS_LOCAL, 7362 bytes) taskResourceAssignments Map() 20/08/08 15:33:53.543 Executor task launch worker for task 35 INFO Executor: Finished task 2.0 in stage 9.0 (TID 35). 2122 bytes result sent to driver 20/08/08 15:33:53.542 Executor task launch worker for task 40 DEBUG ExecutorMetricsPoller: stageTCMP: (9, 0) -> 3 20/08/08 15:33:53.544 Executor task launch worker for task 35 DEBUG ExecutorMetricsPoller: stageTCMP: (9, 0) -> 2 20/08/08 15:33:53.544 Executor task launch worker for task 38 DEBUG ExecutorMetricsPoller: stageTCMP: (9, 0) -> 1 20/08/08 15:33:53.544 dispatcher-event-loop-0 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_9.0, runningTasks: 3 20/08/08 15:33:53.545 dispatcher-event-loop-0 INFO TaskSetManager: Starting task 9.0 in stage 9.0 (TID 42, 192.168.1.225, executor driver, partition 9, PROCESS_LOCAL, 7362 bytes) taskResourceAssignments Map() 20/08/08 15:33:53.545 Executor task launch worker for task 42 INFO Executor: Running task 9.0 in stage 9.0 (TID 42) 20/08/08 15:33:53.545 dispatcher-event-loop-0 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_9.0, runningTasks: 3 20/08/08 15:33:53.545 dispatcher-event-loop-0 DEBUG TaskSetManager: No tasks for locality level PROCESS_LOCAL, so moving to locality level NODE_LOCAL 20/08/08 15:33:53.545 dispatcher-event-loop-0 DEBUG TaskSetManager: No tasks for locality level NODE_LOCAL, so moving to locality level ANY 20/08/08 15:33:53.546 Executor task launch worker for task 42 DEBUG ExecutorMetricsPoller: stageTCMP: (9, 0) -> 2 20/08/08 15:33:53.546 task-result-getter-2 INFO TaskSetManager: Finished task 7.0 in stage 9.0 (TID 40) in 26 ms on 192.168.1.225 (executor driver) (5/10) 20/08/08 15:33:53.547 task-result-getter-2 INFO TaskSetManager: Finished task 5.0 in stage 9.0 (TID 38) in 58 ms on 192.168.1.225 (executor driver) (6/10) 20/08/08 15:33:53.547 task-result-getter-2 INFO TaskSetManager: Finished task 2.0 in stage 9.0 (TID 35) in 156 ms on 192.168.1.225 (executor driver) (7/10) 20/08/08 15:33:53.547 Executor task launch worker for task 41 INFO Executor: Running task 8.0 in stage 9.0 (TID 41) 20/08/08 15:33:53.547 dag-scheduler-event-loop DEBUG DAGScheduler: ShuffleMapTask finished on driver 20/08/08 15:33:53.547 Executor task launch worker for task 41 DEBUG ExecutorMetricsPoller: stageTCMP: (9, 0) -> 3 20/08/08 15:33:53.548 dag-scheduler-event-loop DEBUG DAGScheduler: ShuffleMapTask finished on driver 20/08/08 15:33:53.549 dag-scheduler-event-loop DEBUG DAGScheduler: ShuffleMapTask finished on driver 20/08/08 15:33:53.551 Executor task launch worker for task 42 DEBUG BlockManager: Getting local block rdd_20_9 20/08/08 15:33:53.551 Executor task launch worker for task 42 DEBUG BlockManager: Level for block rdd_20_9 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:53.551 Executor task launch worker for task 42 INFO BlockManager: Found block rdd_20_9 locally 20/08/08 15:33:53.551 Executor task launch worker for task 42 DEBUG BlockManager: Getting local block rdd_19_9 20/08/08 15:33:53.551 Executor task launch worker for task 42 DEBUG BlockManager: Level for block rdd_19_9 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:53.551 Executor task launch worker for task 42 INFO BlockManager: Found block rdd_19_9 locally 20/08/08 15:33:53.552 Executor task launch worker for task 42 DEBUG TaskMemoryManager: Task 42 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@760faa84 20/08/08 15:33:53.552 Executor task launch worker for task 41 DEBUG BlockManager: Getting local block rdd_20_8 20/08/08 15:33:53.552 Executor task launch worker for task 41 DEBUG BlockManager: Level for block rdd_20_8 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:53.552 Executor task launch worker for task 41 INFO BlockManager: Found block rdd_20_8 locally 20/08/08 15:33:53.552 Executor task launch worker for task 41 DEBUG BlockManager: Getting local block rdd_19_8 20/08/08 15:33:53.553 Executor task launch worker for task 41 DEBUG BlockManager: Level for block rdd_19_8 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:53.553 Executor task launch worker for task 41 INFO BlockManager: Found block rdd_19_8 locally 20/08/08 15:33:53.553 Executor task launch worker for task 42 INFO Executor: Finished task 9.0 in stage 9.0 (TID 42). 1950 bytes result sent to driver 20/08/08 15:33:53.553 Executor task launch worker for task 42 DEBUG ExecutorMetricsPoller: stageTCMP: (9, 0) -> 2 20/08/08 15:33:53.553 Executor task launch worker for task 39 DEBUG ExecutorMetricsPoller: stageTCMP: (9, 0) -> 1 20/08/08 15:33:53.553 dispatcher-event-loop-0 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_9.0, runningTasks: 2 20/08/08 15:33:53.554 dispatcher-event-loop-0 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_9.0, runningTasks: 1 20/08/08 15:33:53.554 task-result-getter-3 INFO TaskSetManager: Finished task 9.0 in stage 9.0 (TID 42) in 10 ms on 192.168.1.225 (executor driver) (8/10) 20/08/08 15:33:53.555 task-result-getter-2 INFO TaskSetManager: Finished task 6.0 in stage 9.0 (TID 39) in 55 ms on 192.168.1.225 (executor driver) (9/10) 20/08/08 15:33:53.553 Executor task launch worker for task 41 DEBUG TaskMemoryManager: Task 41 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@605eba2b 20/08/08 15:33:53.555 dag-scheduler-event-loop DEBUG DAGScheduler: ShuffleMapTask finished on driver 20/08/08 15:33:53.556 Executor task launch worker for task 41 INFO Executor: Finished task 8.0 in stage 9.0 (TID 41). 1950 bytes result sent to driver 20/08/08 15:33:53.556 dispatcher-event-loop-3 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_9.0, runningTasks: 0 20/08/08 15:33:53.556 dag-scheduler-event-loop DEBUG DAGScheduler: ShuffleMapTask finished on driver 20/08/08 15:33:53.557 task-result-getter-0 INFO TaskSetManager: Finished task 8.0 in stage 9.0 (TID 41) in 14 ms on 192.168.1.225 (executor driver) (10/10) 20/08/08 15:33:53.557 task-result-getter-0 INFO TaskSchedulerImpl: Removed TaskSet 9.0, whose tasks have all completed, from pool 20/08/08 15:33:53.557 Executor task launch worker for task 41 DEBUG ExecutorMetricsPoller: removing (9, 0) from stageTCMP 20/08/08 15:33:53.558 dag-scheduler-event-loop DEBUG DAGScheduler: ShuffleMapTask finished on driver 20/08/08 15:33:53.558 dag-scheduler-event-loop INFO DAGScheduler: ShuffleMapStage 9 (flatMap at ALS.scala:1688) finished in 0.180 s 20/08/08 15:33:53.558 dag-scheduler-event-loop INFO DAGScheduler: looking for newly runnable stages 20/08/08 15:33:53.558 dag-scheduler-event-loop INFO DAGScheduler: running: Set() 20/08/08 15:33:53.558 dag-scheduler-event-loop INFO DAGScheduler: waiting: Set(ResultStage 12, ShuffleMapStage 11) 20/08/08 15:33:53.558 dag-scheduler-event-loop INFO DAGScheduler: failed: Set() 20/08/08 15:33:53.558 dag-scheduler-event-loop DEBUG MapOutputTrackerMaster: Increasing epoch to 4 20/08/08 15:33:53.559 dag-scheduler-event-loop DEBUG DAGScheduler: submitStage(ShuffleMapStage 11 (name=flatMap at ALS.scala:1688;jobs=3)) 20/08/08 15:33:53.564 dag-scheduler-event-loop DEBUG DAGScheduler: missing: List() 20/08/08 15:33:53.564 dag-scheduler-event-loop INFO DAGScheduler: Submitting ShuffleMapStage 11 (MapPartitionsRDD[40] at flatMap at ALS.scala:1688), which has no missing parents 20/08/08 15:33:53.564 dag-scheduler-event-loop DEBUG DAGScheduler: submitMissingTasks(ShuffleMapStage 11) 20/08/08 15:33:53.583 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_7 stored as values in memory (estimated size 37.6 KiB, free 366.0 MiB) 20/08/08 15:33:53.586 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_7 locally took 3 ms 20/08/08 15:33:53.586 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_7 without replication took 4 ms 20/08/08 15:33:53.588 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_7_piece0 stored as bytes in memory (estimated size 16.9 KiB, free 366.0 MiB) 20/08/08 15:33:53.588 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master broadcast_7_piece0 for BlockManagerId(driver, 192.168.1.225, 38159, None) 20/08/08 15:33:53.588 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added broadcast_7_piece0 in memory on 192.168.1.225:38159 (size: 16.9 KiB, free: 366.2 MiB) 20/08/08 15:33:53.588 dag-scheduler-event-loop DEBUG BlockManagerMaster: Updated info of block broadcast_7_piece0 20/08/08 15:33:53.589 dag-scheduler-event-loop DEBUG BlockManager: Told master about block broadcast_7_piece0 20/08/08 15:33:53.589 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_7_piece0 locally took 1 ms 20/08/08 15:33:53.589 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_7_piece0 without replication took 1 ms 20/08/08 15:33:53.589 dag-scheduler-event-loop INFO SparkContext: Created broadcast 7 from broadcast at DAGScheduler.scala:1348 20/08/08 15:33:53.590 dag-scheduler-event-loop INFO DAGScheduler: Submitting 10 missing tasks from ShuffleMapStage 11 (MapPartitionsRDD[40] at flatMap at ALS.scala:1688) (first 15 tasks are for partitions Vector(0, 1, 2, 3, 4, 5, 6, 7, 8, 9)) 20/08/08 15:33:53.590 dag-scheduler-event-loop INFO TaskSchedulerImpl: Adding task set 11.0 with 10 tasks resource profile 0 20/08/08 15:33:53.590 dag-scheduler-event-loop DEBUG TaskSetManager: Epoch for TaskSet 11.0: 4 20/08/08 15:33:53.590 dag-scheduler-event-loop DEBUG TaskSetManager: Adding pending tasks took 0 ms 20/08/08 15:33:53.590 dag-scheduler-event-loop DEBUG TaskSetManager: Valid locality levels for TaskSet 11.0: PROCESS_LOCAL, NODE_LOCAL, ANY 20/08/08 15:33:53.591 dispatcher-event-loop-0 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_11.0, runningTasks: 0 20/08/08 15:33:53.592 dispatcher-event-loop-0 INFO TaskSetManager: Starting task 0.0 in stage 11.0 (TID 43, 192.168.1.225, executor driver, partition 0, PROCESS_LOCAL, 7422 bytes) taskResourceAssignments Map() 20/08/08 15:33:53.592 dispatcher-event-loop-0 INFO TaskSetManager: Starting task 1.0 in stage 11.0 (TID 44, 192.168.1.225, executor driver, partition 1, PROCESS_LOCAL, 7422 bytes) taskResourceAssignments Map() 20/08/08 15:33:53.592 dispatcher-event-loop-0 INFO TaskSetManager: Starting task 2.0 in stage 11.0 (TID 45, 192.168.1.225, executor driver, partition 2, PROCESS_LOCAL, 7422 bytes) taskResourceAssignments Map() 20/08/08 15:33:53.593 dispatcher-event-loop-0 INFO TaskSetManager: Starting task 3.0 in stage 11.0 (TID 46, 192.168.1.225, executor driver, partition 3, PROCESS_LOCAL, 7422 bytes) taskResourceAssignments Map() 20/08/08 15:33:53.593 Executor task launch worker for task 43 INFO Executor: Running task 0.0 in stage 11.0 (TID 43) 20/08/08 15:33:53.593 Executor task launch worker for task 45 INFO Executor: Running task 2.0 in stage 11.0 (TID 45) 20/08/08 15:33:53.593 Executor task launch worker for task 46 INFO Executor: Running task 3.0 in stage 11.0 (TID 46) 20/08/08 15:33:53.593 Executor task launch worker for task 43 DEBUG ExecutorMetricsPoller: stageTCMP: (11, 0) -> 1 20/08/08 15:33:53.593 Executor task launch worker for task 46 DEBUG ExecutorMetricsPoller: stageTCMP: (11, 0) -> 3 20/08/08 15:33:53.594 Executor task launch worker for task 44 INFO Executor: Running task 1.0 in stage 11.0 (TID 44) 20/08/08 15:33:53.594 Executor task launch worker for task 44 DEBUG ExecutorMetricsPoller: stageTCMP: (11, 0) -> 4 20/08/08 15:33:53.593 Executor task launch worker for task 45 DEBUG ExecutorMetricsPoller: stageTCMP: (11, 0) -> 2 20/08/08 15:33:53.594 Executor task launch worker for task 43 DEBUG BlockManager: Getting local block broadcast_7 20/08/08 15:33:53.595 Executor task launch worker for task 43 DEBUG BlockManager: Level for block broadcast_7 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:53.607 Executor task launch worker for task 9 DEBUG GenerateSafeProjection: code for createexternalrow(staticinvoke(class scala.collection.mutable.WrappedArray$, ObjectType(interface scala.collection.Seq), make, mapobjects(lambdavariable(MapObject, StringType, true, -1), lambdavariable(MapObject, StringType, true, -1).toString, input[0, array, true], None).array, true, false), StructField(words,ArrayType(StringType,true),true)): /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificSafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificSafeProjection extends org.apache.spark.sql.catalyst.expressions.codegen.BaseProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private InternalRow mutableRow; /* 009 */ private boolean resultIsNull_0; /* 010 */ private UTF8String value_MapObject_lambda_variable_1; /* 011 */ private boolean isNull_MapObject_lambda_variable_1; /* 012 */ private boolean globalIsNull_0; /* 013 */ private java.lang.Object[] mutableStateArray_0 = new java.lang.Object[1]; /* 014 */ /* 015 */ public SpecificSafeProjection(Object[] references) { /* 016 */ this.references = references; /* 017 */ mutableRow = (InternalRow) references[references.length - 1]; /* 018 */ /* 019 */ /* 020 */ } /* 021 */ /* 022 */ public void initialize(int partitionIndex) { /* 023 */ /* 024 */ } /* 025 */ /* 026 */ public java.lang.Object apply(java.lang.Object _i) { /* 027 */ InternalRow i = (InternalRow) _i; /* 028 */ org.apache.spark.sql.Row value_7 = CreateExternalRow_0(i); /* 029 */ if (false) { /* 030 */ mutableRow.setNullAt(0); /* 031 */ } else { /* 032 */ /* 033 */ mutableRow.update(0, value_7); /* 034 */ } /* 035 */ /* 036 */ return mutableRow; /* 037 */ } /* 038 */ /* 039 */ /* 040 */ private org.apache.spark.sql.Row CreateExternalRow_0(InternalRow i) { /* 041 */ Object[] values_0 = new Object[1]; /* 042 */ /* 043 */ resultIsNull_0 = false; /* 044 */ if (!resultIsNull_0) { /* 045 */ ArrayData value_6 = MapObjects_0(i); /* 046 */ boolean isNull_2 = true; /* 047 */ java.lang.Object value_2 = null; /* 048 */ if (!globalIsNull_0) { /* 049 */ /* 050 */ isNull_2 = false; /* 051 */ if (!isNull_2) { /* 052 */ /* 053 */ Object funcResult_1 = null; /* 054 */ funcResult_1 = value_6.array(); /* 055 */ value_2 = (java.lang.Object) funcResult_1; /* 056 */ /* 057 */ } /* 058 */ } /* 059 */ resultIsNull_0 = isNull_2; /* 060 */ mutableStateArray_0[0] = value_2; /* 061 */ } /* 062 */ /* 063 */ boolean isNull_1 = resultIsNull_0; /* 064 */ scala.collection.Seq value_1 = null; /* 065 */ if (!resultIsNull_0) { /* 066 */ value_1 = scala.collection.mutable.WrappedArray.make(mutableStateArray_0[0]); /* 067 */ } /* 068 */ if (isNull_1) { /* 069 */ values_0[0] = null; /* 070 */ } else { /* 071 */ values_0[0] = value_1; /* 072 */ } /* 073 */ /* 074 */ final org.apache.spark.sql.Row value_0 = new org.apache.spark.sql.catalyst.expressions.GenericRowWithSchema(values_0, ((org.apache.spark.sql.types.StructType) references[0] /* schema */)); /* 075 */ /* 076 */ return value_0; /* 077 */ } /* 078 */ /* 079 */ /* 080 */ private ArrayData MapObjects_0(InternalRow i) { /* 081 */ boolean isNull_4 = i.isNullAt(0); /* 082 */ ArrayData value_4 = isNull_4 ? /* 083 */ null : (i.getArray(0)); /* 084 */ ArrayData value_3 = null; /* 085 */ /* 086 */ if (!isNull_4) { /* 087 */ /* 088 */ int dataLength_0 = value_4.numElements(); /* 089 */ /* 090 */ java.lang.String[] convertedArray_0 = null; /* 091 */ convertedArray_0 = new java.lang.String[dataLength_0]; /* 092 */ /* 093 */ /* 094 */ int loopIndex_0 = 0; /* 095 */ /* 096 */ while (loopIndex_0 < dataLength_0) { /* 097 */ value_MapObject_lambda_variable_1 = (UTF8String) (value_4.getUTF8String(loopIndex_0)); /* 098 */ isNull_MapObject_lambda_variable_1 = value_4.isNullAt(loopIndex_0); /* 099 */ /* 100 */ boolean isNull_5 = true; /* 101 */ java.lang.String value_5 = null; /* 102 */ if (!isNull_MapObject_lambda_variable_1) { /* 103 */ /* 104 */ isNull_5 = false; /* 105 */ if (!isNull_5) { /* 106 */ /* 107 */ Object funcResult_0 = null; /* 108 */ funcResult_0 = value_MapObject_lambda_variable_1.toString(); /* 109 */ value_5 = (java.lang.String) funcResult_0; /* 110 */ /* 111 */ } /* 112 */ } /* 113 */ if (isNull_5) { /* 114 */ convertedArray_0[loopIndex_0] = null; /* 115 */ } else { /* 116 */ convertedArray_0[loopIndex_0] = value_5; /* 117 */ } /* 118 */ /* 119 */ loopIndex_0 += 1; /* 120 */ } /* 121 */ /* 122 */ value_3 = new org.apache.spark.sql.catalyst.util.GenericArrayData(convertedArray_0); /* 123 */ } /* 124 */ globalIsNull_0 = isNull_4; /* 125 */ return value_3; /* 126 */ } /* 127 */ /* 128 */ } 20/08/08 15:33:53.614 Executor task launch worker for task 10 DEBUG GenerateSafeProjection: code for createexternalrow(staticinvoke(class scala.collection.mutable.WrappedArray$, ObjectType(interface scala.collection.Seq), make, mapobjects(lambdavariable(MapObject, StringType, true, -1), lambdavariable(MapObject, StringType, true, -1).toString, input[0, array, true], None).array, true, false), StructField(words,ArrayType(StringType,true),true)): /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificSafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificSafeProjection extends org.apache.spark.sql.catalyst.expressions.codegen.BaseProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private InternalRow mutableRow; /* 009 */ private boolean resultIsNull_0; /* 010 */ private UTF8String value_MapObject_lambda_variable_1; /* 011 */ private boolean isNull_MapObject_lambda_variable_1; /* 012 */ private boolean globalIsNull_0; /* 013 */ private java.lang.Object[] mutableStateArray_0 = new java.lang.Object[1]; /* 014 */ /* 015 */ public SpecificSafeProjection(Object[] references) { /* 016 */ this.references = references; /* 017 */ mutableRow = (InternalRow) references[references.length - 1]; /* 018 */ /* 019 */ /* 020 */ } /* 021 */ /* 022 */ public void initialize(int partitionIndex) { /* 023 */ /* 024 */ } /* 025 */ /* 026 */ public java.lang.Object apply(java.lang.Object _i) { /* 027 */ InternalRow i = (InternalRow) _i; /* 028 */ org.apache.spark.sql.Row value_7 = CreateExternalRow_0(i); /* 029 */ if (false) { /* 030 */ mutableRow.setNullAt(0); /* 031 */ } else { /* 032 */ /* 033 */ mutableRow.update(0, value_7); /* 034 */ } /* 035 */ /* 036 */ return mutableRow; /* 037 */ } /* 038 */ /* 039 */ /* 040 */ private org.apache.spark.sql.Row CreateExternalRow_0(InternalRow i) { /* 041 */ Object[] values_0 = new Object[1]; /* 042 */ /* 043 */ resultIsNull_0 = false; /* 044 */ if (!resultIsNull_0) { /* 045 */ ArrayData value_6 = MapObjects_0(i); /* 046 */ boolean isNull_2 = true; /* 047 */ java.lang.Object value_2 = null; /* 048 */ if (!globalIsNull_0) { /* 049 */ /* 050 */ isNull_2 = false; /* 051 */ if (!isNull_2) { /* 052 */ /* 053 */ Object funcResult_1 = null; /* 054 */ funcResult_1 = value_6.array(); /* 055 */ value_2 = (java.lang.Object) funcResult_1; /* 056 */ /* 057 */ } /* 058 */ } /* 059 */ resultIsNull_0 = isNull_2; /* 060 */ mutableStateArray_0[0] = value_2; /* 061 */ } /* 062 */ /* 063 */ boolean isNull_1 = resultIsNull_0; /* 064 */ scala.collection.Seq value_1 = null; /* 065 */ if (!resultIsNull_0) { /* 066 */ value_1 = scala.collection.mutable.WrappedArray.make(mutableStateArray_0[0]); /* 067 */ } /* 068 */ if (isNull_1) { /* 069 */ values_0[0] = null; /* 070 */ } else { /* 071 */ values_0[0] = value_1; /* 072 */ } /* 073 */ /* 074 */ final org.apache.spark.sql.Row value_0 = new org.apache.spark.sql.catalyst.expressions.GenericRowWithSchema(values_0, ((org.apache.spark.sql.types.StructType) references[0] /* schema */)); /* 075 */ /* 076 */ return value_0; /* 077 */ } /* 078 */ /* 079 */ /* 080 */ private ArrayData MapObjects_0(InternalRow i) { /* 081 */ boolean isNull_4 = i.isNullAt(0); /* 082 */ ArrayData value_4 = isNull_4 ? /* 083 */ null : (i.getArray(0)); /* 084 */ ArrayData value_3 = null; /* 085 */ /* 086 */ if (!isNull_4) { /* 087 */ /* 088 */ int dataLength_0 = value_4.numElements(); /* 089 */ /* 090 */ java.lang.String[] convertedArray_0 = null; /* 091 */ convertedArray_0 = new java.lang.String[dataLength_0]; /* 092 */ /* 093 */ /* 094 */ int loopIndex_0 = 0; /* 095 */ /* 096 */ while (loopIndex_0 < dataLength_0) { /* 097 */ value_MapObject_lambda_variable_1 = (UTF8String) (value_4.getUTF8String(loopIndex_0)); /* 098 */ isNull_MapObject_lambda_variable_1 = value_4.isNullAt(loopIndex_0); /* 099 */ /* 100 */ boolean isNull_5 = true; /* 101 */ java.lang.String value_5 = null; /* 102 */ if (!isNull_MapObject_lambda_variable_1) { /* 103 */ /* 104 */ isNull_5 = false; /* 105 */ if (!isNull_5) { /* 106 */ /* 107 */ Object funcResult_0 = null; /* 108 */ funcResult_0 = value_MapObject_lambda_variable_1.toString(); /* 109 */ value_5 = (java.lang.String) funcResult_0; /* 110 */ /* 111 */ } /* 112 */ } /* 113 */ if (isNull_5) { /* 114 */ convertedArray_0[loopIndex_0] = null; /* 115 */ } else { /* 116 */ convertedArray_0[loopIndex_0] = value_5; /* 117 */ } /* 118 */ /* 119 */ loopIndex_0 += 1; /* 120 */ } /* 121 */ /* 122 */ value_3 = new org.apache.spark.sql.catalyst.util.GenericArrayData(convertedArray_0); /* 123 */ } /* 124 */ globalIsNull_0 = isNull_4; /* 125 */ return value_3; /* 126 */ } /* 127 */ /* 128 */ } 20/08/08 15:33:53.616 Executor task launch worker for task 45 DEBUG BlockManager: Getting local block rdd_25_2 20/08/08 15:33:53.616 Executor task launch worker for task 45 DEBUG BlockManager: Level for block rdd_25_2 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:53.616 Executor task launch worker for task 45 INFO BlockManager: Found block rdd_25_2 locally 20/08/08 15:33:53.616 Executor task launch worker for task 45 DEBUG BlockManager: Getting local block rdd_24_2 20/08/08 15:33:53.616 Executor task launch worker for task 45 DEBUG BlockManager: Level for block rdd_24_2 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:53.616 Executor task launch worker for task 45 INFO BlockManager: Found block rdd_24_2 locally 20/08/08 15:33:53.617 Executor task launch worker for task 45 DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 4 20/08/08 15:33:53.614 Executor task launch worker for task 8 DEBUG GenerateSafeProjection: code for createexternalrow(staticinvoke(class scala.collection.mutable.WrappedArray$, ObjectType(interface scala.collection.Seq), make, mapobjects(lambdavariable(MapObject, StringType, true, -1), lambdavariable(MapObject, StringType, true, -1).toString, input[0, array, true], None).array, true, false), StructField(words,ArrayType(StringType,true),true)): /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificSafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificSafeProjection extends org.apache.spark.sql.catalyst.expressions.codegen.BaseProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private InternalRow mutableRow; /* 009 */ private boolean resultIsNull_0; /* 010 */ private UTF8String value_MapObject_lambda_variable_1; /* 011 */ private boolean isNull_MapObject_lambda_variable_1; /* 012 */ private boolean globalIsNull_0; /* 013 */ private java.lang.Object[] mutableStateArray_0 = new java.lang.Object[1]; /* 014 */ /* 015 */ public SpecificSafeProjection(Object[] references) { /* 016 */ this.references = references; /* 017 */ mutableRow = (InternalRow) references[references.length - 1]; /* 018 */ /* 019 */ /* 020 */ } /* 021 */ /* 022 */ public void initialize(int partitionIndex) { /* 023 */ /* 024 */ } /* 025 */ /* 026 */ public java.lang.Object apply(java.lang.Object _i) { /* 027 */ InternalRow i = (InternalRow) _i; /* 028 */ org.apache.spark.sql.Row value_7 = CreateExternalRow_0(i); /* 029 */ if (false) { /* 030 */ mutableRow.setNullAt(0); /* 031 */ } else { /* 032 */ /* 033 */ mutableRow.update(0, value_7); /* 034 */ } /* 035 */ /* 036 */ return mutableRow; /* 037 */ } /* 038 */ /* 039 */ /* 040 */ private org.apache.spark.sql.Row CreateExternalRow_0(InternalRow i) { /* 041 */ Object[] values_0 = new Object[1]; /* 042 */ /* 043 */ resultIsNull_0 = false; /* 044 */ if (!resultIsNull_0) { /* 045 */ ArrayData value_6 = MapObjects_0(i); /* 046 */ boolean isNull_2 = true; /* 047 */ java.lang.Object value_2 = null; /* 048 */ if (!globalIsNull_0) { /* 049 */ /* 050 */ isNull_2 = false; /* 051 */ if (!isNull_2) { /* 052 */ /* 053 */ Object funcResult_1 = null; /* 054 */ funcResult_1 = value_6.array(); /* 055 */ value_2 = (java.lang.Object) funcResult_1; /* 056 */ /* 057 */ } /* 058 */ } /* 059 */ resultIsNull_0 = isNull_2; /* 060 */ mutableStateArray_0[0] = value_2; /* 061 */ } /* 062 */ /* 063 */ boolean isNull_1 = resultIsNull_0; /* 064 */ scala.collection.Seq value_1 = null; /* 065 */ if (!resultIsNull_0) { /* 066 */ value_1 = scala.collection.mutable.WrappedArray.make(mutableStateArray_0[0]); /* 067 */ } /* 068 */ if (isNull_1) { /* 069 */ values_0[0] = null; /* 070 */ } else { /* 071 */ values_0[0] = value_1; /* 072 */ } /* 073 */ /* 074 */ final org.apache.spark.sql.Row value_0 = new org.apache.spark.sql.catalyst.expressions.GenericRowWithSchema(values_0, ((org.apache.spark.sql.types.StructType) references[0] /* schema */)); /* 075 */ /* 076 */ return value_0; /* 077 */ } /* 078 */ /* 079 */ /* 080 */ private ArrayData MapObjects_0(InternalRow i) { /* 081 */ boolean isNull_4 = i.isNullAt(0); /* 082 */ ArrayData value_4 = isNull_4 ? /* 083 */ null : (i.getArray(0)); /* 084 */ ArrayData value_3 = null; /* 085 */ /* 086 */ if (!isNull_4) { /* 087 */ /* 088 */ int dataLength_0 = value_4.numElements(); /* 089 */ /* 090 */ java.lang.String[] convertedArray_0 = null; /* 091 */ convertedArray_0 = new java.lang.String[dataLength_0]; /* 092 */ /* 093 */ /* 094 */ int loopIndex_0 = 0; /* 095 */ /* 096 */ while (loopIndex_0 < dataLength_0) { /* 097 */ value_MapObject_lambda_variable_1 = (UTF8String) (value_4.getUTF8String(loopIndex_0)); /* 098 */ isNull_MapObject_lambda_variable_1 = value_4.isNullAt(loopIndex_0); /* 099 */ /* 100 */ boolean isNull_5 = true; /* 101 */ java.lang.String value_5 = null; /* 102 */ if (!isNull_MapObject_lambda_variable_1) { /* 103 */ /* 104 */ isNull_5 = false; /* 105 */ if (!isNull_5) { /* 106 */ /* 107 */ Object funcResult_0 = null; /* 108 */ funcResult_0 = value_MapObject_lambda_variable_1.toString(); /* 109 */ value_5 = (java.lang.String) funcResult_0; /* 110 */ /* 111 */ } /* 112 */ } /* 113 */ if (isNull_5) { /* 114 */ convertedArray_0[loopIndex_0] = null; /* 115 */ } else { /* 116 */ convertedArray_0[loopIndex_0] = value_5; /* 117 */ } /* 118 */ /* 119 */ loopIndex_0 += 1; /* 120 */ } /* 121 */ /* 122 */ value_3 = new org.apache.spark.sql.catalyst.util.GenericArrayData(convertedArray_0); /* 123 */ } /* 124 */ globalIsNull_0 = isNull_4; /* 125 */ return value_3; /* 126 */ } /* 127 */ /* 128 */ } 20/08/08 15:33:53.617 Executor task launch worker for task 45 DEBUG MapOutputTrackerMaster: Convert map statuses for shuffle 4, mappers 0-10, partitions 2-3 20/08/08 15:33:53.617 Executor task launch worker for task 45 DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 20/08/08 15:33:53.618 Executor task launch worker for task 45 INFO ShuffleBlockFetcherIterator: Getting 3 (707.0 B) non-empty blocks including 3 (707.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 20/08/08 15:33:53.618 Executor task launch worker for task 45 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 1 ms 20/08/08 15:33:53.618 Executor task launch worker for task 45 DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_4_33_2,0), (shuffle_4_34_2,1), (shuffle_4_35_2,2) 20/08/08 15:33:53.618 Executor task launch worker for task 45 INFO BlockManager: Getting local shuffle block shuffle_4_33_2 20/08/08 15:33:53.618 Executor task launch worker for task 44 DEBUG BlockManager: Getting local block rdd_25_1 20/08/08 15:33:53.618 Executor task launch worker for task 44 DEBUG BlockManager: Level for block rdd_25_1 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:53.618 Executor task launch worker for task 45 INFO BlockManager: Getting local shuffle block shuffle_4_34_2 20/08/08 15:33:53.619 Executor task launch worker for task 44 INFO BlockManager: Found block rdd_25_1 locally 20/08/08 15:33:53.619 Executor task launch worker for task 44 DEBUG BlockManager: Getting local block rdd_24_1 20/08/08 15:33:53.619 Executor task launch worker for task 45 INFO BlockManager: Getting local shuffle block shuffle_4_35_2 20/08/08 15:33:53.619 Executor task launch worker for task 44 DEBUG BlockManager: Level for block rdd_24_1 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:53.619 Executor task launch worker for task 44 INFO BlockManager: Found block rdd_24_1 locally 20/08/08 15:33:53.619 Executor task launch worker for task 45 DEBUG ShuffleBlockFetcherIterator: Got local blocks in 1 ms 20/08/08 15:33:53.619 Executor task launch worker for task 44 DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 4 20/08/08 15:33:53.619 Executor task launch worker for task 44 DEBUG MapOutputTrackerMaster: Convert map statuses for shuffle 4, mappers 0-10, partitions 1-2 20/08/08 15:33:53.615 Executor task launch worker for task 11 DEBUG GenerateSafeProjection: code for createexternalrow(staticinvoke(class scala.collection.mutable.WrappedArray$, ObjectType(interface scala.collection.Seq), make, mapobjects(lambdavariable(MapObject, StringType, true, -1), lambdavariable(MapObject, StringType, true, -1).toString, input[0, array, true], None).array, true, false), StructField(words,ArrayType(StringType,true),true)): /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificSafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificSafeProjection extends org.apache.spark.sql.catalyst.expressions.codegen.BaseProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private InternalRow mutableRow; /* 009 */ private boolean resultIsNull_0; /* 010 */ private UTF8String value_MapObject_lambda_variable_1; /* 011 */ private boolean isNull_MapObject_lambda_variable_1; /* 012 */ private boolean globalIsNull_0; /* 013 */ private java.lang.Object[] mutableStateArray_0 = new java.lang.Object[1]; /* 014 */ /* 015 */ public SpecificSafeProjection(Object[] references) { /* 016 */ this.references = references; /* 017 */ mutableRow = (InternalRow) references[references.length - 1]; /* 018 */ /* 019 */ /* 020 */ } /* 021 */ /* 022 */ public void initialize(int partitionIndex) { /* 023 */ /* 024 */ } /* 025 */ /* 026 */ public java.lang.Object apply(java.lang.Object _i) { /* 027 */ InternalRow i = (InternalRow) _i; /* 028 */ org.apache.spark.sql.Row value_7 = CreateExternalRow_0(i); /* 029 */ if (false) { /* 030 */ mutableRow.setNullAt(0); /* 031 */ } else { /* 032 */ /* 033 */ mutableRow.update(0, value_7); /* 034 */ } /* 035 */ /* 036 */ return mutableRow; /* 037 */ } /* 038 */ /* 039 */ /* 040 */ private org.apache.spark.sql.Row CreateExternalRow_0(InternalRow i) { /* 041 */ Object[] values_0 = new Object[1]; /* 042 */ /* 043 */ resultIsNull_0 = false; /* 044 */ if (!resultIsNull_0) { /* 045 */ ArrayData value_6 = MapObjects_0(i); /* 046 */ boolean isNull_2 = true; /* 047 */ java.lang.Object value_2 = null; /* 048 */ if (!globalIsNull_0) { /* 049 */ /* 050 */ isNull_2 = false; /* 051 */ if (!isNull_2) { /* 052 */ /* 053 */ Object funcResult_1 = null; /* 054 */ funcResult_1 = value_6.array(); /* 055 */ value_2 = (java.lang.Object) funcResult_1; /* 056 */ /* 057 */ } /* 058 */ } /* 059 */ resultIsNull_0 = isNull_2; /* 060 */ mutableStateArray_0[0] = value_2; /* 061 */ } /* 062 */ /* 063 */ boolean isNull_1 = resultIsNull_0; /* 064 */ scala.collection.Seq value_1 = null; /* 065 */ if (!resultIsNull_0) { /* 066 */ value_1 = scala.collection.mutable.WrappedArray.make(mutableStateArray_0[0]); /* 067 */ } /* 068 */ if (isNull_1) { /* 069 */ values_0[0] = null; /* 070 */ } else { /* 071 */ values_0[0] = value_1; /* 072 */ } /* 073 */ /* 074 */ final org.apache.spark.sql.Row value_0 = new org.apache.spark.sql.catalyst.expressions.GenericRowWithSchema(values_0, ((org.apache.spark.sql.types.StructType) references[0] /* schema */)); /* 075 */ /* 076 */ return value_0; /* 077 */ } /* 078 */ /* 079 */ /* 080 */ private ArrayData MapObjects_0(InternalRow i) { /* 081 */ boolean isNull_4 = i.isNullAt(0); /* 082 */ ArrayData value_4 = isNull_4 ? /* 083 */ null : (i.getArray(0)); /* 084 */ ArrayData value_3 = null; /* 085 */ /* 086 */ if (!isNull_4) { /* 087 */ /* 088 */ int dataLength_0 = value_4.numElements(); /* 089 */ /* 090 */ java.lang.String[] convertedArray_0 = null; /* 091 */ convertedArray_0 = new java.lang.String[dataLength_0]; /* 092 */ /* 093 */ /* 094 */ int loopIndex_0 = 0; /* 095 */ /* 096 */ while (loopIndex_0 < dataLength_0) { /* 097 */ value_MapObject_lambda_variable_1 = (UTF8String) (value_4.getUTF8String(loopIndex_0)); /* 098 */ isNull_MapObject_lambda_variable_1 = value_4.isNullAt(loopIndex_0); /* 099 */ /* 100 */ boolean isNull_5 = true; /* 101 */ java.lang.String value_5 = null; /* 102 */ if (!isNull_MapObject_lambda_variable_1) { /* 103 */ /* 104 */ isNull_5 = false; /* 105 */ if (!isNull_5) { /* 106 */ /* 107 */ Object funcResult_0 = null; /* 108 */ funcResult_0 = value_MapObject_lambda_variable_1.toString(); /* 109 */ value_5 = (java.lang.String) funcResult_0; /* 110 */ /* 111 */ } /* 112 */ } /* 113 */ if (isNull_5) { /* 114 */ convertedArray_0[loopIndex_0] = null; /* 115 */ } else { /* 116 */ convertedArray_0[loopIndex_0] = value_5; /* 117 */ } /* 118 */ /* 119 */ loopIndex_0 += 1; /* 120 */ } /* 121 */ /* 122 */ value_3 = new org.apache.spark.sql.catalyst.util.GenericArrayData(convertedArray_0); /* 123 */ } /* 124 */ globalIsNull_0 = isNull_4; /* 125 */ return value_3; /* 126 */ } /* 127 */ /* 128 */ } 20/08/08 15:33:53.619 Executor task launch worker for task 44 DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 20/08/08 15:33:53.620 Executor task launch worker for task 44 INFO ShuffleBlockFetcherIterator: Getting 3 (730.0 B) non-empty blocks including 3 (730.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 20/08/08 15:33:53.620 Executor task launch worker for task 44 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 20/08/08 15:33:53.620 Executor task launch worker for task 44 DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_4_33_1,0), (shuffle_4_34_1,1), (shuffle_4_35_1,2) 20/08/08 15:33:53.620 Executor task launch worker for task 44 INFO BlockManager: Getting local shuffle block shuffle_4_33_1 20/08/08 15:33:53.620 Executor task launch worker for task 44 INFO BlockManager: Getting local shuffle block shuffle_4_34_1 20/08/08 15:33:53.620 Executor task launch worker for task 44 INFO BlockManager: Getting local shuffle block shuffle_4_35_1 20/08/08 15:33:53.620 Executor task launch worker for task 44 DEBUG ShuffleBlockFetcherIterator: Got local blocks in 1 ms 20/08/08 15:33:53.623 Executor task launch worker for task 44 DEBUG TaskMemoryManager: Task 44 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@7ce82c9e 20/08/08 15:33:53.624 Executor task launch worker for task 46 DEBUG BlockManager: Getting local block rdd_25_3 20/08/08 15:33:53.624 Executor task launch worker for task 46 DEBUG BlockManager: Level for block rdd_25_3 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:53.624 Executor task launch worker for task 46 INFO BlockManager: Found block rdd_25_3 locally 20/08/08 15:33:53.624 Executor task launch worker for task 46 DEBUG BlockManager: Getting local block rdd_24_3 20/08/08 15:33:53.624 Executor task launch worker for task 46 DEBUG BlockManager: Level for block rdd_24_3 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:53.624 Executor task launch worker for task 46 INFO BlockManager: Found block rdd_24_3 locally 20/08/08 15:33:53.624 Executor task launch worker for task 46 DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 4 20/08/08 15:33:53.624 Executor task launch worker for task 46 DEBUG MapOutputTrackerMaster: Convert map statuses for shuffle 4, mappers 0-10, partitions 3-4 20/08/08 15:33:53.625 Executor task launch worker for task 46 DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 20/08/08 15:33:53.625 Executor task launch worker for task 46 INFO ShuffleBlockFetcherIterator: Getting 3 (684.0 B) non-empty blocks including 3 (684.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 20/08/08 15:33:53.625 Executor task launch worker for task 46 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 20/08/08 15:33:53.626 Executor task launch worker for task 46 DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_4_33_3,0), (shuffle_4_34_3,1), (shuffle_4_35_3,2) 20/08/08 15:33:53.626 Executor task launch worker for task 46 INFO BlockManager: Getting local shuffle block shuffle_4_33_3 20/08/08 15:33:53.626 Executor task launch worker for task 46 INFO BlockManager: Getting local shuffle block shuffle_4_34_3 20/08/08 15:33:53.626 Executor task launch worker for task 46 INFO BlockManager: Getting local shuffle block shuffle_4_35_3 20/08/08 15:33:53.626 Executor task launch worker for task 46 DEBUG ShuffleBlockFetcherIterator: Got local blocks in 1 ms 20/08/08 15:33:53.626 Executor task launch worker for task 45 DEBUG TaskMemoryManager: Task 45 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@67089714 20/08/08 15:33:53.628 Executor task launch worker for task 46 DEBUG TaskMemoryManager: Task 46 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@47f80d30 20/08/08 15:33:53.628 Executor task launch worker for task 46 DEBUG TaskMemoryManager: Task 46 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@6ea8d0a4 20/08/08 15:33:53.629 Executor task launch worker for task 46 DEBUG TaskMemoryManager: Task 46 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@465d7f1c 20/08/08 15:33:53.630 Executor task launch worker for task 46 INFO Executor: Finished task 3.0 in stage 11.0 (TID 46). 2251 bytes result sent to driver 20/08/08 15:33:53.630 Executor task launch worker for task 43 DEBUG BlockManager: Getting local block rdd_25_0 20/08/08 15:33:53.630 Executor task launch worker for task 43 DEBUG BlockManager: Level for block rdd_25_0 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:53.630 Executor task launch worker for task 43 INFO BlockManager: Found block rdd_25_0 locally 20/08/08 15:33:53.631 Executor task launch worker for task 43 DEBUG BlockManager: Getting local block rdd_24_0 20/08/08 15:33:53.631 Executor task launch worker for task 43 DEBUG BlockManager: Level for block rdd_24_0 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:53.631 Executor task launch worker for task 43 INFO BlockManager: Found block rdd_24_0 locally 20/08/08 15:33:53.631 Executor task launch worker for task 43 DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 4 20/08/08 15:33:53.631 Executor task launch worker for task 43 DEBUG MapOutputTrackerMaster: Convert map statuses for shuffle 4, mappers 0-10, partitions 0-1 20/08/08 15:33:53.631 Executor task launch worker for task 43 DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 20/08/08 15:33:53.632 Executor task launch worker for task 9 DEBUG CodeGenerator: /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificSafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificSafeProjection extends org.apache.spark.sql.catalyst.expressions.codegen.BaseProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private InternalRow mutableRow; /* 009 */ private boolean resultIsNull_0; /* 010 */ private UTF8String value_MapObject_lambda_variable_1; /* 011 */ private boolean isNull_MapObject_lambda_variable_1; /* 012 */ private boolean globalIsNull_0; /* 013 */ private java.lang.Object[] mutableStateArray_0 = new java.lang.Object[1]; /* 014 */ /* 015 */ public SpecificSafeProjection(Object[] references) { /* 016 */ this.references = references; /* 017 */ mutableRow = (InternalRow) references[references.length - 1]; /* 018 */ /* 019 */ /* 020 */ } /* 021 */ /* 022 */ public void initialize(int partitionIndex) { /* 023 */ /* 024 */ } /* 025 */ /* 026 */ public java.lang.Object apply(java.lang.Object _i) { /* 027 */ InternalRow i = (InternalRow) _i; /* 028 */ org.apache.spark.sql.Row value_7 = CreateExternalRow_0(i); /* 029 */ if (false) { /* 030 */ mutableRow.setNullAt(0); /* 031 */ } else { /* 032 */ /* 033 */ mutableRow.update(0, value_7); /* 034 */ } /* 035 */ /* 036 */ return mutableRow; /* 037 */ } /* 038 */ /* 039 */ /* 040 */ private org.apache.spark.sql.Row CreateExternalRow_0(InternalRow i) { /* 041 */ Object[] values_0 = new Object[1]; /* 042 */ /* 043 */ resultIsNull_0 = false; /* 044 */ if (!resultIsNull_0) { /* 045 */ ArrayData value_6 = MapObjects_0(i); /* 046 */ boolean isNull_2 = true; /* 047 */ java.lang.Object value_2 = null; /* 048 */ if (!globalIsNull_0) { /* 049 */ /* 050 */ isNull_2 = false; /* 051 */ if (!isNull_2) { /* 052 */ /* 053 */ Object funcResult_1 = null; /* 054 */ funcResult_1 = value_6.array(); /* 055 */ value_2 = (java.lang.Object) funcResult_1; /* 056 */ /* 057 */ } /* 058 */ } /* 059 */ resultIsNull_0 = isNull_2; /* 060 */ mutableStateArray_0[0] = value_2; /* 061 */ } /* 062 */ /* 063 */ boolean isNull_1 = resultIsNull_0; /* 064 */ scala.collection.Seq value_1 = null; /* 065 */ if (!resultIsNull_0) { /* 066 */ value_1 = scala.collection.mutable.WrappedArray.make(mutableStateArray_0[0]); /* 067 */ } /* 068 */ if (isNull_1) { /* 069 */ values_0[0] = null; /* 070 */ } else { /* 071 */ values_0[0] = value_1; /* 072 */ } /* 073 */ /* 074 */ final org.apache.spark.sql.Row value_0 = new org.apache.spark.sql.catalyst.expressions.GenericRowWithSchema(values_0, ((org.apache.spark.sql.types.StructType) references[0] /* schema */)); /* 075 */ /* 076 */ return value_0; /* 077 */ } /* 078 */ /* 079 */ /* 080 */ private ArrayData MapObjects_0(InternalRow i) { /* 081 */ boolean isNull_4 = i.isNullAt(0); /* 082 */ ArrayData value_4 = isNull_4 ? /* 083 */ null : (i.getArray(0)); /* 084 */ ArrayData value_3 = null; /* 085 */ /* 086 */ if (!isNull_4) { /* 087 */ /* 088 */ int dataLength_0 = value_4.numElements(); /* 089 */ /* 090 */ java.lang.String[] convertedArray_0 = null; /* 091 */ convertedArray_0 = new java.lang.String[dataLength_0]; /* 092 */ /* 093 */ /* 094 */ int loopIndex_0 = 0; /* 095 */ /* 096 */ while (loopIndex_0 < dataLength_0) { /* 097 */ value_MapObject_lambda_variable_1 = (UTF8String) (value_4.getUTF8String(loopIndex_0)); /* 098 */ isNull_MapObject_lambda_variable_1 = value_4.isNullAt(loopIndex_0); /* 099 */ /* 100 */ boolean isNull_5 = true; /* 101 */ java.lang.String value_5 = null; /* 102 */ if (!isNull_MapObject_lambda_variable_1) { /* 103 */ /* 104 */ isNull_5 = false; /* 105 */ if (!isNull_5) { /* 106 */ /* 107 */ Object funcResult_0 = null; /* 108 */ funcResult_0 = value_MapObject_lambda_variable_1.toString(); /* 109 */ value_5 = (java.lang.String) funcResult_0; /* 110 */ /* 111 */ } /* 112 */ } /* 113 */ if (isNull_5) { /* 114 */ convertedArray_0[loopIndex_0] = null; /* 115 */ } else { /* 116 */ convertedArray_0[loopIndex_0] = value_5; /* 117 */ } /* 118 */ /* 119 */ loopIndex_0 += 1; /* 120 */ } /* 121 */ /* 122 */ value_3 = new org.apache.spark.sql.catalyst.util.GenericArrayData(convertedArray_0); /* 123 */ } /* 124 */ globalIsNull_0 = isNull_4; /* 125 */ return value_3; /* 126 */ } /* 127 */ /* 128 */ } 20/08/08 15:33:53.641 Executor task launch worker for task 43 INFO ShuffleBlockFetcherIterator: Getting 3 (684.0 B) non-empty blocks including 3 (684.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 20/08/08 15:33:53.641 Executor task launch worker for task 43 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 10 ms 20/08/08 15:33:53.642 Executor task launch worker for task 43 DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_4_33_0,0), (shuffle_4_34_0,1), (shuffle_4_35_0,2) 20/08/08 15:33:53.642 Executor task launch worker for task 43 INFO BlockManager: Getting local shuffle block shuffle_4_33_0 20/08/08 15:33:53.642 Executor task launch worker for task 43 INFO BlockManager: Getting local shuffle block shuffle_4_34_0 20/08/08 15:33:53.642 Executor task launch worker for task 43 INFO BlockManager: Getting local shuffle block shuffle_4_35_0 20/08/08 15:33:53.642 Executor task launch worker for task 43 DEBUG ShuffleBlockFetcherIterator: Got local blocks in 10 ms 20/08/08 15:33:53.643 dispatcher-event-loop-2 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_11.0, runningTasks: 3 20/08/08 15:33:53.643 dispatcher-event-loop-2 INFO TaskSetManager: Starting task 4.0 in stage 11.0 (TID 47, 192.168.1.225, executor driver, partition 4, PROCESS_LOCAL, 7422 bytes) taskResourceAssignments Map() 20/08/08 15:33:53.644 Executor task launch worker for task 47 INFO Executor: Running task 4.0 in stage 11.0 (TID 47) 20/08/08 15:33:53.644 Executor task launch worker for task 43 DEBUG TaskMemoryManager: Task 43 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@34b98d92 20/08/08 15:33:53.644 task-result-getter-1 INFO TaskSetManager: Finished task 3.0 in stage 11.0 (TID 46) in 52 ms on 192.168.1.225 (executor driver) (1/10) 20/08/08 15:33:53.645 dag-scheduler-event-loop DEBUG DAGScheduler: ShuffleMapTask finished on driver 20/08/08 15:33:53.654 Executor task launch worker for task 46 DEBUG ExecutorMetricsPoller: stageTCMP: (11, 0) -> 3 20/08/08 15:33:53.654 Executor task launch worker for task 47 DEBUG ExecutorMetricsPoller: stageTCMP: (11, 0) -> 4 20/08/08 15:33:53.668 Executor task launch worker for task 45 WARN LAPACK: Failed to load implementation from: com.github.fommil.netlib.NativeSystemLAPACK 20/08/08 15:33:53.669 Executor task launch worker for task 45 WARN LAPACK: Failed to load implementation from: com.github.fommil.netlib.NativeRefLAPACK 20/08/08 15:33:53.689 Executor task launch worker for task 47 DEBUG BlockManager: Getting local block rdd_25_4 20/08/08 15:33:53.689 Executor task launch worker for task 47 DEBUG BlockManager: Level for block rdd_25_4 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:53.689 Executor task launch worker for task 47 INFO BlockManager: Found block rdd_25_4 locally 20/08/08 15:33:53.689 Executor task launch worker for task 47 DEBUG BlockManager: Getting local block rdd_24_4 20/08/08 15:33:53.689 Executor task launch worker for task 47 DEBUG BlockManager: Level for block rdd_24_4 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:53.689 Executor task launch worker for task 47 INFO BlockManager: Found block rdd_24_4 locally 20/08/08 15:33:53.689 Executor task launch worker for task 47 DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 4 20/08/08 15:33:53.689 Executor task launch worker for task 47 DEBUG MapOutputTrackerMaster: Convert map statuses for shuffle 4, mappers 0-10, partitions 4-5 20/08/08 15:33:53.690 Executor task launch worker for task 47 DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 20/08/08 15:33:53.695 Executor task launch worker for task 9 INFO CodeGenerator: Code generated in 87.024562 ms 20/08/08 15:33:53.701 Executor task launch worker for task 47 INFO ShuffleBlockFetcherIterator: Getting 3 (684.0 B) non-empty blocks including 3 (684.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 20/08/08 15:33:53.701 Executor task launch worker for task 47 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 11 ms 20/08/08 15:33:53.701 Executor task launch worker for task 47 DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_4_33_4,0), (shuffle_4_34_4,1), (shuffle_4_35_4,2) 20/08/08 15:33:53.701 Executor task launch worker for task 47 INFO BlockManager: Getting local shuffle block shuffle_4_33_4 20/08/08 15:33:53.702 Executor task launch worker for task 47 INFO BlockManager: Getting local shuffle block shuffle_4_34_4 20/08/08 15:33:53.702 Executor task launch worker for task 47 INFO BlockManager: Getting local shuffle block shuffle_4_35_4 20/08/08 15:33:53.702 Executor task launch worker for task 47 DEBUG ShuffleBlockFetcherIterator: Got local blocks in 12 ms 20/08/08 15:33:53.704 Executor task launch worker for task 47 DEBUG TaskMemoryManager: Task 47 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@3f53d9f5 20/08/08 15:33:53.704 Executor task launch worker for task 47 DEBUG TaskMemoryManager: Task 47 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@4b27d874 20/08/08 15:33:53.705 Executor task launch worker for task 47 DEBUG TaskMemoryManager: Task 47 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@67ce9f53 20/08/08 15:33:53.706 Executor task launch worker for task 47 INFO Executor: Finished task 4.0 in stage 11.0 (TID 47). 2251 bytes result sent to driver 20/08/08 15:33:53.706 dispatcher-event-loop-3 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_11.0, runningTasks: 3 20/08/08 15:33:53.706 dispatcher-event-loop-3 INFO TaskSetManager: Starting task 5.0 in stage 11.0 (TID 48, 192.168.1.225, executor driver, partition 5, PROCESS_LOCAL, 7422 bytes) taskResourceAssignments Map() 20/08/08 15:33:53.707 task-result-getter-3 INFO TaskSetManager: Finished task 4.0 in stage 11.0 (TID 47) in 64 ms on 192.168.1.225 (executor driver) (2/10) 20/08/08 15:33:53.707 Executor task launch worker for task 47 DEBUG ExecutorMetricsPoller: stageTCMP: (11, 0) -> 3 20/08/08 15:33:53.707 Executor task launch worker for task 48 INFO Executor: Running task 5.0 in stage 11.0 (TID 48) 20/08/08 15:33:53.708 Executor task launch worker for task 48 DEBUG ExecutorMetricsPoller: stageTCMP: (11, 0) -> 4 20/08/08 15:33:53.708 dag-scheduler-event-loop DEBUG DAGScheduler: ShuffleMapTask finished on driver 20/08/08 15:33:53.714 Executor task launch worker for task 48 DEBUG BlockManager: Getting local block rdd_25_5 20/08/08 15:33:53.715 Executor task launch worker for task 48 DEBUG BlockManager: Level for block rdd_25_5 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:53.715 Executor task launch worker for task 48 INFO BlockManager: Found block rdd_25_5 locally 20/08/08 15:33:53.715 Executor task launch worker for task 48 DEBUG BlockManager: Getting local block rdd_24_5 20/08/08 15:33:53.715 Executor task launch worker for task 48 DEBUG BlockManager: Level for block rdd_24_5 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:53.715 Executor task launch worker for task 48 INFO BlockManager: Found block rdd_24_5 locally 20/08/08 15:33:53.715 Executor task launch worker for task 48 DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 4 20/08/08 15:33:53.715 Executor task launch worker for task 48 DEBUG MapOutputTrackerMaster: Convert map statuses for shuffle 4, mappers 0-10, partitions 5-6 20/08/08 15:33:53.715 Executor task launch worker for task 48 DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 20/08/08 15:33:53.716 Executor task launch worker for task 48 INFO ShuffleBlockFetcherIterator: Getting 3 (684.0 B) non-empty blocks including 3 (684.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 20/08/08 15:33:53.716 Executor task launch worker for task 48 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 20/08/08 15:33:53.716 Executor task launch worker for task 48 DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_4_33_5,0), (shuffle_4_34_5,1), (shuffle_4_35_5,2) 20/08/08 15:33:53.716 Executor task launch worker for task 48 INFO BlockManager: Getting local shuffle block shuffle_4_33_5 20/08/08 15:33:53.716 Executor task launch worker for task 48 INFO BlockManager: Getting local shuffle block shuffle_4_34_5 20/08/08 15:33:53.716 Executor task launch worker for task 48 INFO BlockManager: Getting local shuffle block shuffle_4_35_5 20/08/08 15:33:53.717 Executor task launch worker for task 48 DEBUG ShuffleBlockFetcherIterator: Got local blocks in 1 ms 20/08/08 15:33:53.718 Executor task launch worker for task 48 DEBUG TaskMemoryManager: Task 48 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@1fa0d3d1 20/08/08 15:33:53.719 Executor task launch worker for task 48 DEBUG TaskMemoryManager: Task 48 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@6eb08c76 20/08/08 15:33:53.719 Executor task launch worker for task 48 DEBUG TaskMemoryManager: Task 48 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@4251a045 20/08/08 15:33:53.720 Executor task launch worker for task 48 INFO Executor: Finished task 5.0 in stage 11.0 (TID 48). 2251 bytes result sent to driver 20/08/08 15:33:53.723 dispatcher-event-loop-2 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_11.0, runningTasks: 3 20/08/08 15:33:53.723 dispatcher-event-loop-2 INFO TaskSetManager: Starting task 6.0 in stage 11.0 (TID 49, 192.168.1.225, executor driver, partition 6, PROCESS_LOCAL, 7422 bytes) taskResourceAssignments Map() 20/08/08 15:33:53.724 Executor task launch worker for task 49 INFO Executor: Running task 6.0 in stage 11.0 (TID 49) 20/08/08 15:33:53.724 task-result-getter-2 INFO TaskSetManager: Finished task 5.0 in stage 11.0 (TID 48) in 18 ms on 192.168.1.225 (executor driver) (3/10) 20/08/08 15:33:53.724 Executor task launch worker for task 48 DEBUG ExecutorMetricsPoller: stageTCMP: (11, 0) -> 3 20/08/08 15:33:53.725 Executor task launch worker for task 49 DEBUG ExecutorMetricsPoller: stageTCMP: (11, 0) -> 4 20/08/08 15:33:53.725 dag-scheduler-event-loop DEBUG DAGScheduler: ShuffleMapTask finished on driver 20/08/08 15:33:53.731 Executor task launch worker for task 49 DEBUG BlockManager: Getting local block rdd_25_6 20/08/08 15:33:53.730 Executor task launch worker for task 10 INFO PythonRunner: Times: total = 48, boot = -752, init = 800, finish = 0 20/08/08 15:33:53.731 Executor task launch worker for task 49 DEBUG BlockManager: Level for block rdd_25_6 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:53.731 Executor task launch worker for task 49 INFO BlockManager: Found block rdd_25_6 locally 20/08/08 15:33:53.731 Executor task launch worker for task 49 DEBUG BlockManager: Getting local block rdd_24_6 20/08/08 15:33:53.731 Executor task launch worker for task 49 DEBUG BlockManager: Level for block rdd_24_6 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:53.731 Executor task launch worker for task 49 INFO BlockManager: Found block rdd_24_6 locally 20/08/08 15:33:53.731 Executor task launch worker for task 49 DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 4 20/08/08 15:33:53.731 Executor task launch worker for task 49 DEBUG MapOutputTrackerMaster: Convert map statuses for shuffle 4, mappers 0-10, partitions 6-7 20/08/08 15:33:53.731 Executor task launch worker for task 11 INFO PythonRunner: Times: total = 82, boot = -815, init = 897, finish = 0 20/08/08 15:33:53.732 Executor task launch worker for task 49 DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 20/08/08 15:33:53.732 Executor task launch worker for task 49 INFO ShuffleBlockFetcherIterator: Getting 3 (684.0 B) non-empty blocks including 3 (684.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 20/08/08 15:33:53.732 Executor task launch worker for task 49 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 20/08/08 15:33:53.732 Executor task launch worker for task 49 DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_4_33_6,0), (shuffle_4_34_6,1), (shuffle_4_35_6,2) 20/08/08 15:33:53.732 Executor task launch worker for task 49 INFO BlockManager: Getting local shuffle block shuffle_4_33_6 20/08/08 15:33:53.732 Executor task launch worker for task 49 INFO BlockManager: Getting local shuffle block shuffle_4_34_6 20/08/08 15:33:53.733 Executor task launch worker for task 49 INFO BlockManager: Getting local shuffle block shuffle_4_35_6 20/08/08 15:33:53.733 Executor task launch worker for task 49 DEBUG ShuffleBlockFetcherIterator: Got local blocks in 1 ms 20/08/08 15:33:53.734 Executor task launch worker for task 9 INFO PythonRunner: Times: total = 61, boot = -855, init = 916, finish = 0 20/08/08 15:33:53.734 Executor task launch worker for task 49 DEBUG TaskMemoryManager: Task 49 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@6c28ebe9 20/08/08 15:33:53.735 Executor task launch worker for task 8 INFO PythonRunner: Times: total = 47, boot = -841, init = 888, finish = 0 20/08/08 15:33:53.735 Executor task launch worker for task 49 DEBUG TaskMemoryManager: Task 49 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@7ca803f4 20/08/08 15:33:53.735 Executor task launch worker for task 49 DEBUG TaskMemoryManager: Task 49 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@49c1533f 20/08/08 15:33:53.736 Executor task launch worker for task 49 INFO Executor: Finished task 6.0 in stage 11.0 (TID 49). 2251 bytes result sent to driver 20/08/08 15:33:53.737 dispatcher-event-loop-3 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_11.0, runningTasks: 3 20/08/08 15:33:53.738 dispatcher-event-loop-3 INFO TaskSetManager: Starting task 7.0 in stage 11.0 (TID 50, 192.168.1.225, executor driver, partition 7, PROCESS_LOCAL, 7422 bytes) taskResourceAssignments Map() 20/08/08 15:33:53.738 Executor task launch worker for task 50 INFO Executor: Running task 7.0 in stage 11.0 (TID 50) 20/08/08 15:33:53.738 task-result-getter-0 INFO TaskSetManager: Finished task 6.0 in stage 11.0 (TID 49) in 15 ms on 192.168.1.225 (executor driver) (4/10) 20/08/08 15:33:53.738 Executor task launch worker for task 49 DEBUG ExecutorMetricsPoller: stageTCMP: (11, 0) -> 3 20/08/08 15:33:53.739 Executor task launch worker for task 50 DEBUG ExecutorMetricsPoller: stageTCMP: (11, 0) -> 4 20/08/08 15:33:53.739 dag-scheduler-event-loop DEBUG DAGScheduler: ShuffleMapTask finished on driver 20/08/08 15:33:53.754 Executor task launch worker for task 50 DEBUG BlockManager: Getting local block rdd_25_7 20/08/08 15:33:53.754 Executor task launch worker for task 50 DEBUG BlockManager: Level for block rdd_25_7 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:53.754 Executor task launch worker for task 50 INFO BlockManager: Found block rdd_25_7 locally 20/08/08 15:33:53.754 Executor task launch worker for task 50 DEBUG BlockManager: Getting local block rdd_24_7 20/08/08 15:33:53.754 Executor task launch worker for task 50 DEBUG BlockManager: Level for block rdd_24_7 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:53.754 Executor task launch worker for task 50 INFO BlockManager: Found block rdd_24_7 locally 20/08/08 15:33:53.755 Executor task launch worker for task 50 DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 4 20/08/08 15:33:53.755 Executor task launch worker for task 50 DEBUG MapOutputTrackerMaster: Convert map statuses for shuffle 4, mappers 0-10, partitions 7-8 20/08/08 15:33:53.755 Executor task launch worker for task 50 DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 20/08/08 15:33:53.756 Executor task launch worker for task 50 INFO ShuffleBlockFetcherIterator: Getting 3 (684.0 B) non-empty blocks including 3 (684.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 20/08/08 15:33:53.756 Executor task launch worker for task 50 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 20/08/08 15:33:53.756 Executor task launch worker for task 50 DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_4_33_7,0), (shuffle_4_34_7,1), (shuffle_4_35_7,2) 20/08/08 15:33:53.756 Executor task launch worker for task 50 INFO BlockManager: Getting local shuffle block shuffle_4_33_7 20/08/08 15:33:53.756 Executor task launch worker for task 50 INFO BlockManager: Getting local shuffle block shuffle_4_34_7 20/08/08 15:33:53.756 Executor task launch worker for task 50 INFO BlockManager: Getting local shuffle block shuffle_4_35_7 20/08/08 15:33:53.756 Executor task launch worker for task 50 DEBUG ShuffleBlockFetcherIterator: Got local blocks in 1 ms 20/08/08 15:33:53.758 Executor task launch worker for task 50 DEBUG TaskMemoryManager: Task 50 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@25a55a0e 20/08/08 15:33:53.758 Executor task launch worker for task 50 DEBUG TaskMemoryManager: Task 50 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@6ff7c65b 20/08/08 15:33:53.759 Executor task launch worker for task 50 DEBUG TaskMemoryManager: Task 50 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@36f78504 20/08/08 15:33:53.760 Executor task launch worker for task 50 INFO Executor: Finished task 7.0 in stage 11.0 (TID 50). 2251 bytes result sent to driver 20/08/08 15:33:53.760 Executor task launch worker for task 50 DEBUG ExecutorMetricsPoller: stageTCMP: (11, 0) -> 3 20/08/08 15:33:53.760 dispatcher-event-loop-2 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_11.0, runningTasks: 3 20/08/08 15:33:53.761 dispatcher-event-loop-2 INFO TaskSetManager: Starting task 8.0 in stage 11.0 (TID 51, 192.168.1.225, executor driver, partition 8, PROCESS_LOCAL, 7422 bytes) taskResourceAssignments Map() 20/08/08 15:33:53.762 task-result-getter-1 INFO TaskSetManager: Finished task 7.0 in stage 11.0 (TID 50) in 25 ms on 192.168.1.225 (executor driver) (5/10) 20/08/08 15:33:53.763 dag-scheduler-event-loop DEBUG DAGScheduler: ShuffleMapTask finished on driver 20/08/08 15:33:53.765 Executor task launch worker for task 51 INFO Executor: Running task 8.0 in stage 11.0 (TID 51) 20/08/08 15:33:53.765 Executor task launch worker for task 51 DEBUG ExecutorMetricsPoller: stageTCMP: (11, 0) -> 4 20/08/08 15:33:53.774 Executor task launch worker for task 8 DEBUG TaskMemoryManager: Task 8 release 0.0 B from org.apache.spark.util.collection.ExternalSorter@16bd6ad3 20/08/08 15:33:53.782 Executor task launch worker for task 51 DEBUG BlockManager: Getting local block rdd_25_8 20/08/08 15:33:53.782 Executor task launch worker for task 51 DEBUG BlockManager: Level for block rdd_25_8 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:53.782 Executor task launch worker for task 51 INFO BlockManager: Found block rdd_25_8 locally 20/08/08 15:33:53.782 Executor task launch worker for task 51 DEBUG BlockManager: Getting local block rdd_24_8 20/08/08 15:33:53.782 Executor task launch worker for task 51 DEBUG BlockManager: Level for block rdd_24_8 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:53.783 Executor task launch worker for task 51 INFO BlockManager: Found block rdd_24_8 locally 20/08/08 15:33:53.783 Executor task launch worker for task 51 DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 4 20/08/08 15:33:53.783 Executor task launch worker for task 51 DEBUG MapOutputTrackerMaster: Convert map statuses for shuffle 4, mappers 0-10, partitions 8-9 20/08/08 15:33:53.783 Executor task launch worker for task 51 DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 20/08/08 15:33:53.784 Executor task launch worker for task 51 INFO ShuffleBlockFetcherIterator: Getting 3 (684.0 B) non-empty blocks including 3 (684.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 20/08/08 15:33:53.784 Executor task launch worker for task 51 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 20/08/08 15:33:53.784 Executor task launch worker for task 51 DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_4_33_8,0), (shuffle_4_34_8,1), (shuffle_4_35_8,2) 20/08/08 15:33:53.784 Executor task launch worker for task 51 INFO BlockManager: Getting local shuffle block shuffle_4_33_8 20/08/08 15:33:53.784 Executor task launch worker for task 51 INFO BlockManager: Getting local shuffle block shuffle_4_34_8 20/08/08 15:33:53.784 Executor task launch worker for task 51 INFO BlockManager: Getting local shuffle block shuffle_4_35_8 20/08/08 15:33:53.784 Executor task launch worker for task 51 DEBUG ShuffleBlockFetcherIterator: Got local blocks in 1 ms 20/08/08 15:33:53.786 Executor task launch worker for task 51 DEBUG TaskMemoryManager: Task 51 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@7a727e81 20/08/08 15:33:53.787 Executor task launch worker for task 51 DEBUG TaskMemoryManager: Task 51 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@5d5047c0 20/08/08 15:33:53.787 Executor task launch worker for task 51 DEBUG TaskMemoryManager: Task 51 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@73837794 20/08/08 15:33:53.788 Executor task launch worker for task 51 INFO Executor: Finished task 8.0 in stage 11.0 (TID 51). 2251 bytes result sent to driver 20/08/08 15:33:53.788 Executor task launch worker for task 51 DEBUG ExecutorMetricsPoller: stageTCMP: (11, 0) -> 3 20/08/08 15:33:53.789 dispatcher-event-loop-3 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_11.0, runningTasks: 3 20/08/08 15:33:53.789 dispatcher-event-loop-3 INFO TaskSetManager: Starting task 9.0 in stage 11.0 (TID 52, 192.168.1.225, executor driver, partition 9, PROCESS_LOCAL, 7422 bytes) taskResourceAssignments Map() 20/08/08 15:33:53.790 task-result-getter-3 INFO TaskSetManager: Finished task 8.0 in stage 11.0 (TID 51) in 30 ms on 192.168.1.225 (executor driver) (6/10) 20/08/08 15:33:53.791 dag-scheduler-event-loop DEBUG DAGScheduler: ShuffleMapTask finished on driver 20/08/08 15:33:53.791 Executor task launch worker for task 52 INFO Executor: Running task 9.0 in stage 11.0 (TID 52) 20/08/08 15:33:53.792 Executor task launch worker for task 52 DEBUG ExecutorMetricsPoller: stageTCMP: (11, 0) -> 4 20/08/08 15:33:53.797 Executor task launch worker for task 11 DEBUG TaskMemoryManager: Task 11 release 0.0 B from org.apache.spark.util.collection.ExternalSorter@2fd27186 20/08/08 15:33:53.798 Executor task launch worker for task 10 DEBUG TaskMemoryManager: Task 10 release 0.0 B from org.apache.spark.util.collection.ExternalSorter@57ac0a2d 20/08/08 15:33:53.800 Executor task launch worker for task 8 INFO Executor: Finished task 0.0 in stage 2.0 (TID 8). 2073 bytes result sent to driver 20/08/08 15:33:53.800 Executor task launch worker for task 8 DEBUG ExecutorMetricsPoller: stageTCMP: (2, 0) -> 3 20/08/08 15:33:53.775 Executor task launch worker for task 9 DEBUG TaskMemoryManager: Task 9 release 0.0 B from org.apache.spark.util.collection.ExternalSorter@17154411 20/08/08 15:33:53.801 dispatcher-event-loop-1 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_2.0, runningTasks: 3 20/08/08 15:33:53.801 dispatcher-event-loop-1 DEBUG TaskSetManager: No tasks for locality level NO_PREF, so moving to locality level ANY 20/08/08 15:33:53.804 task-result-getter-0 INFO TaskSetManager: Finished task 0.0 in stage 2.0 (TID 8) in 479 ms on 192.168.1.225 (executor driver) (1/4) 20/08/08 15:33:53.804 Executor task launch worker for task 10 INFO Executor: Finished task 2.0 in stage 2.0 (TID 10). 2030 bytes result sent to driver 20/08/08 15:33:53.804 Executor task launch worker for task 10 DEBUG ExecutorMetricsPoller: stageTCMP: (2, 0) -> 2 20/08/08 15:33:53.805 dispatcher-event-loop-3 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_2.0, runningTasks: 2 20/08/08 15:33:53.805 Executor task launch worker for task 43 DEBUG TaskMemoryManager: Task 43 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@2f4098ae 20/08/08 15:33:53.806 task-result-getter-1 INFO TaskSetManager: Finished task 2.0 in stage 2.0 (TID 10) in 480 ms on 192.168.1.225 (executor driver) (2/4) 20/08/08 15:33:53.804 Executor task launch worker for task 9 INFO Executor: Finished task 1.0 in stage 2.0 (TID 9). 2030 bytes result sent to driver 20/08/08 15:33:53.806 Executor task launch worker for task 9 DEBUG ExecutorMetricsPoller: stageTCMP: (2, 0) -> 1 20/08/08 15:33:53.807 dispatcher-event-loop-0 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_2.0, runningTasks: 1 20/08/08 15:33:53.808 Executor task launch worker for task 43 DEBUG TaskMemoryManager: Task 43 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@22e10b 20/08/08 15:33:53.808 task-result-getter-2 INFO TaskSetManager: Finished task 1.0 in stage 2.0 (TID 9) in 482 ms on 192.168.1.225 (executor driver) (3/4) 20/08/08 15:33:53.809 Executor task launch worker for task 44 DEBUG TaskMemoryManager: Task 44 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@5d8600eb 20/08/08 15:33:53.811 Executor task launch worker for task 44 DEBUG TaskMemoryManager: Task 44 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@7c7a1b2f 20/08/08 15:33:53.804 Executor task launch worker for task 11 INFO Executor: Finished task 3.0 in stage 2.0 (TID 11). 2030 bytes result sent to driver 20/08/08 15:33:53.813 Executor task launch worker for task 11 DEBUG ExecutorMetricsPoller: removing (2, 0) from stageTCMP 20/08/08 15:33:53.813 dispatcher-event-loop-2 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_2.0, runningTasks: 0 20/08/08 15:33:53.814 Executor task launch worker for task 44 INFO Executor: Finished task 1.0 in stage 11.0 (TID 44). 2423 bytes result sent to driver 20/08/08 15:33:53.814 task-result-getter-3 INFO TaskSetManager: Finished task 3.0 in stage 2.0 (TID 11) in 487 ms on 192.168.1.225 (executor driver) (4/4) 20/08/08 15:33:53.814 dispatcher-event-loop-2 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_11.0, runningTasks: 3 20/08/08 15:33:53.814 task-result-getter-3 INFO TaskSchedulerImpl: Removed TaskSet 2.0, whose tasks have all completed, from pool 20/08/08 15:33:53.815 dispatcher-event-loop-2 DEBUG TaskSetManager: No tasks for locality level PROCESS_LOCAL, so moving to locality level NODE_LOCAL 20/08/08 15:33:53.815 dispatcher-event-loop-2 DEBUG TaskSetManager: No tasks for locality level NODE_LOCAL, so moving to locality level ANY 20/08/08 15:33:53.815 Executor task launch worker for task 44 DEBUG ExecutorMetricsPoller: stageTCMP: (11, 0) -> 3 20/08/08 15:33:53.815 task-result-getter-2 INFO TaskSetManager: Finished task 1.0 in stage 11.0 (TID 44) in 223 ms on 192.168.1.225 (executor driver) (7/10) 20/08/08 15:33:53.816 dag-scheduler-event-loop DEBUG DAGScheduler: ShuffleMapTask finished on driver 20/08/08 15:33:53.816 dag-scheduler-event-loop DEBUG DAGScheduler: ShuffleMapTask finished on driver 20/08/08 15:33:53.818 dag-scheduler-event-loop DEBUG DAGScheduler: ShuffleMapTask finished on driver 20/08/08 15:33:53.819 dag-scheduler-event-loop DEBUG DAGScheduler: ShuffleMapTask finished on driver 20/08/08 15:33:53.820 dag-scheduler-event-loop DEBUG DAGScheduler: ShuffleMapTask finished on driver 20/08/08 15:33:53.820 dag-scheduler-event-loop INFO DAGScheduler: ShuffleMapStage 2 (flatMap at CountVectorizer.scala:212) finished in 0.573 s 20/08/08 15:33:53.821 dag-scheduler-event-loop INFO DAGScheduler: looking for newly runnable stages 20/08/08 15:33:53.822 dag-scheduler-event-loop INFO DAGScheduler: running: Set() 20/08/08 15:33:53.822 dag-scheduler-event-loop INFO DAGScheduler: waiting: Set(ResultStage 3) 20/08/08 15:33:53.822 dag-scheduler-event-loop INFO DAGScheduler: failed: Set() 20/08/08 15:33:53.823 dag-scheduler-event-loop DEBUG MapOutputTrackerMaster: Increasing epoch to 1 20/08/08 15:33:53.823 Executor task launch worker for task 52 DEBUG BlockManager: Getting local block rdd_25_9 20/08/08 15:33:53.823 Executor task launch worker for task 52 DEBUG BlockManager: Level for block rdd_25_9 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:53.823 Executor task launch worker for task 52 INFO BlockManager: Found block rdd_25_9 locally 20/08/08 15:33:53.823 Executor task launch worker for task 52 DEBUG BlockManager: Getting local block rdd_24_9 20/08/08 15:33:53.824 Executor task launch worker for task 52 DEBUG BlockManager: Level for block rdd_24_9 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:53.824 Executor task launch worker for task 52 INFO BlockManager: Found block rdd_24_9 locally 20/08/08 15:33:53.824 Executor task launch worker for task 52 DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 4 20/08/08 15:33:53.824 Executor task launch worker for task 52 DEBUG MapOutputTrackerMaster: Convert map statuses for shuffle 4, mappers 0-10, partitions 9-10 20/08/08 15:33:53.824 Executor task launch worker for task 43 INFO Executor: Finished task 0.0 in stage 11.0 (TID 43). 2423 bytes result sent to driver 20/08/08 15:33:53.824 Executor task launch worker for task 43 DEBUG ExecutorMetricsPoller: stageTCMP: (11, 0) -> 2 20/08/08 15:33:53.824 Executor task launch worker for task 52 DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 20/08/08 15:33:53.824 dispatcher-event-loop-1 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_11.0, runningTasks: 2 20/08/08 15:33:53.824 Executor task launch worker for task 45 DEBUG TaskMemoryManager: Task 45 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@6e522b9f 20/08/08 15:33:53.825 Executor task launch worker for task 52 INFO ShuffleBlockFetcherIterator: Getting 3 (684.0 B) non-empty blocks including 3 (684.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 20/08/08 15:33:53.825 Executor task launch worker for task 52 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 20/08/08 15:33:53.825 Executor task launch worker for task 52 DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_4_33_9,0), (shuffle_4_34_9,1), (shuffle_4_35_9,2) 20/08/08 15:33:53.825 Executor task launch worker for task 52 INFO BlockManager: Getting local shuffle block shuffle_4_33_9 20/08/08 15:33:53.825 Executor task launch worker for task 52 INFO BlockManager: Getting local shuffle block shuffle_4_34_9 20/08/08 15:33:53.825 task-result-getter-0 INFO TaskSetManager: Finished task 0.0 in stage 11.0 (TID 43) in 234 ms on 192.168.1.225 (executor driver) (8/10) 20/08/08 15:33:53.826 dag-scheduler-event-loop DEBUG DAGScheduler: ShuffleMapTask finished on driver 20/08/08 15:33:53.829 Executor task launch worker for task 45 DEBUG TaskMemoryManager: Task 45 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@53d6b38c 20/08/08 15:33:53.830 dag-scheduler-event-loop DEBUG DAGScheduler: submitStage(ResultStage 3 (name=count at CountVectorizer.scala:233;jobs=2)) 20/08/08 15:33:53.831 dag-scheduler-event-loop DEBUG DAGScheduler: missing: List() 20/08/08 15:33:53.831 dag-scheduler-event-loop INFO DAGScheduler: Submitting ResultStage 3 (MapPartitionsRDD[21] at map at CountVectorizer.scala:230), which has no missing parents 20/08/08 15:33:53.831 dag-scheduler-event-loop DEBUG DAGScheduler: submitMissingTasks(ResultStage 3) 20/08/08 15:33:53.825 Executor task launch worker for task 52 INFO BlockManager: Getting local shuffle block shuffle_4_35_9 20/08/08 15:33:53.843 Thread-4 DEBUG WholeStageCodegenExec: /* 001 */ public Object generate(Object[] references) { /* 002 */ return new GeneratedIteratorForCodegenStage2(references); /* 003 */ } /* 004 */ /* 005 */ // codegenStageId=2 /* 006 */ final class GeneratedIteratorForCodegenStage2 extends org.apache.spark.sql.execution.BufferedRowIterator { /* 007 */ private Object[] references; /* 008 */ private scala.collection.Iterator[] inputs; /* 009 */ private boolean agg_initAgg_0; /* 010 */ private boolean agg_bufIsNull_0; /* 011 */ private double agg_bufValue_0; /* 012 */ private boolean agg_bufIsNull_1; /* 013 */ private double agg_bufValue_1; /* 014 */ private scala.collection.Iterator inputadapter_input_0; /* 015 */ private boolean agg_agg_isNull_7_0; /* 016 */ private boolean agg_agg_isNull_9_0; /* 017 */ private boolean agg_agg_isNull_14_0; /* 018 */ private boolean agg_agg_isNull_16_0; /* 019 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[] agg_mutableStateArray_0 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[1]; /* 020 */ /* 021 */ public GeneratedIteratorForCodegenStage2(Object[] references) { /* 022 */ this.references = references; /* 023 */ } /* 024 */ /* 025 */ public void init(int index, scala.collection.Iterator[] inputs) { /* 026 */ partitionIndex = index; /* 027 */ this.inputs = inputs; /* 028 */ /* 029 */ inputadapter_input_0 = inputs[0]; /* 030 */ agg_mutableStateArray_0[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(1, 0); /* 031 */ /* 032 */ } /* 033 */ /* 034 */ private void agg_doAggregate_sum_0(double agg_expr_0_0, boolean agg_exprIsNull_0_0) throws java.io.IOException { /* 035 */ agg_agg_isNull_7_0 = true; /* 036 */ double agg_value_7 = -1.0; /* 037 */ do { /* 038 */ boolean agg_isNull_8 = true; /* 039 */ double agg_value_8 = -1.0; /* 040 */ agg_agg_isNull_9_0 = true; /* 041 */ double agg_value_9 = -1.0; /* 042 */ do { /* 043 */ if (!agg_bufIsNull_0) { /* 044 */ agg_agg_isNull_9_0 = false; /* 045 */ agg_value_9 = agg_bufValue_0; /* 046 */ continue; /* 047 */ } /* 048 */ /* 049 */ if (!false) { /* 050 */ agg_agg_isNull_9_0 = false; /* 051 */ agg_value_9 = 0.0D; /* 052 */ continue; /* 053 */ } /* 054 */ /* 055 */ } while (false); /* 056 */ /* 057 */ if (!agg_exprIsNull_0_0) { /* 058 */ agg_isNull_8 = false; // resultCode could change nullability. /* 059 */ /* 060 */ agg_value_8 = agg_value_9 + agg_expr_0_0; /* 061 */ /* 062 */ } /* 063 */ if (!agg_isNull_8) { /* 064 */ agg_agg_isNull_7_0 = false; /* 065 */ agg_value_7 = agg_value_8; /* 066 */ continue; /* 067 */ } /* 068 */ /* 069 */ if (!agg_bufIsNull_0) { /* 070 */ agg_agg_isNull_7_0 = false; /* 071 */ agg_value_7 = agg_bufValue_0; /* 072 */ continue; /* 073 */ } /* 074 */ /* 075 */ } while (false); /* 076 */ /* 077 */ agg_bufIsNull_0 = agg_agg_isNull_7_0; /* 078 */ agg_bufValue_0 = agg_value_7; /* 079 */ } /* 080 */ /* 081 */ private void agg_doAggregateWithoutKey_0() throws java.io.IOException { /* 082 */ // initialize aggregation buffer /* 083 */ agg_bufIsNull_0 = true; /* 084 */ agg_bufValue_0 = -1.0; /* 085 */ agg_bufIsNull_1 = true; /* 086 */ agg_bufValue_1 = -1.0; /* 087 */ /* 088 */ while ( inputadapter_input_0.hasNext()) { /* 089 */ InternalRow inputadapter_row_0 = (InternalRow) inputadapter_input_0.next(); /* 090 */ /* 091 */ boolean inputadapter_isNull_0 = inputadapter_row_0.isNullAt(0); /* 092 */ double inputadapter_value_0 = inputadapter_isNull_0 ? /* 093 */ -1.0 : (inputadapter_row_0.getDouble(0)); /* 094 */ boolean inputadapter_isNull_1 = inputadapter_row_0.isNullAt(1); /* 095 */ double inputadapter_value_1 = inputadapter_isNull_1 ? /* 096 */ -1.0 : (inputadapter_row_0.getDouble(1)); /* 097 */ /* 098 */ agg_doConsume_0(inputadapter_row_0, inputadapter_value_0, inputadapter_isNull_0, inputadapter_value_1, inputadapter_isNull_1); /* 099 */ // shouldStop check is eliminated /* 100 */ } /* 101 */ /* 102 */ } /* 103 */ /* 104 */ private void agg_doConsume_0(InternalRow inputadapter_row_0, double agg_expr_0_0, boolean agg_exprIsNull_0_0, double agg_expr_1_0, boolean agg_exprIsNull_1_0) throws java.io.IOException { /* 105 */ // do aggregate /* 106 */ // common sub-expressions /* 107 */ /* 108 */ // evaluate aggregate functions and update aggregation buffers /* 109 */ agg_doAggregate_sum_0(agg_expr_0_0, agg_exprIsNull_0_0); /* 110 */ agg_doAggregate_sum_1(agg_exprIsNull_1_0, agg_expr_1_0); /* 111 */ /* 112 */ } /* 113 */ /* 114 */ private void agg_doAggregate_sum_1(boolean agg_exprIsNull_1_0, double agg_expr_1_0) throws java.io.IOException { /* 115 */ agg_agg_isNull_14_0 = true; /* 116 */ double agg_value_14 = -1.0; /* 117 */ do { /* 118 */ boolean agg_isNull_15 = true; /* 119 */ double agg_value_15 = -1.0; /* 120 */ agg_agg_isNull_16_0 = true; /* 121 */ double agg_value_16 = -1.0; /* 122 */ do { /* 123 */ if (!agg_bufIsNull_1) { /* 124 */ agg_agg_isNull_16_0 = false; /* 125 */ agg_value_16 = agg_bufValue_1; /* 126 */ continue; /* 127 */ } /* 128 */ /* 129 */ if (!false) { /* 130 */ agg_agg_isNull_16_0 = false; /* 131 */ agg_value_16 = 0.0D; /* 132 */ continue; /* 133 */ } /* 134 */ /* 135 */ } while (false); /* 136 */ /* 137 */ if (!agg_exprIsNull_1_0) { /* 138 */ agg_isNull_15 = false; // resultCode could change nullability. /* 139 */ /* 140 */ agg_value_15 = agg_value_16 + agg_expr_1_0; /* 141 */ /* 142 */ } /* 143 */ if (!agg_isNull_15) { /* 144 */ agg_agg_isNull_14_0 = false; /* 145 */ agg_value_14 = agg_value_15; /* 146 */ continue; /* 147 */ } /* 148 */ /* 149 */ if (!agg_bufIsNull_1) { /* 150 */ agg_agg_isNull_14_0 = false; /* 151 */ agg_value_14 = agg_bufValue_1; /* 152 */ continue; /* 153 */ } /* 154 */ /* 155 */ } while (false); /* 156 */ /* 157 */ agg_bufIsNull_1 = agg_agg_isNull_14_0; /* 158 */ agg_bufValue_1 = agg_value_14; /* 159 */ } /* 160 */ /* 161 */ protected void processNext() throws java.io.IOException { /* 162 */ while (!agg_initAgg_0) { /* 163 */ agg_initAgg_0 = true; /* 164 */ long agg_beforeAgg_0 = System.nanoTime(); /* 165 */ agg_doAggregateWithoutKey_0(); /* 166 */ ((org.apache.spark.sql.execution.metric.SQLMetric) references[1] /* aggTime */).add((System.nanoTime() - agg_beforeAgg_0) / 1000000); /* 167 */ /* 168 */ // output the result /* 169 */ boolean agg_isNull_4 = false; /* 170 */ double agg_value_4 = -1.0; /* 171 */ if (agg_bufIsNull_1 || agg_bufValue_1 == 0) { /* 172 */ agg_isNull_4 = true; /* 173 */ } else { /* 174 */ if (agg_bufIsNull_0) { /* 175 */ agg_isNull_4 = true; /* 176 */ } else { /* 177 */ agg_value_4 = (double)(agg_bufValue_0 / agg_bufValue_1); /* 178 */ } /* 179 */ } /* 180 */ /* 181 */ ((org.apache.spark.sql.execution.metric.SQLMetric) references[0] /* numOutputRows */).add(1); /* 182 */ agg_mutableStateArray_0[0].reset(); /* 183 */ /* 184 */ agg_mutableStateArray_0[0].zeroOutNullBytes(); /* 185 */ /* 186 */ if (agg_isNull_4) { /* 187 */ agg_mutableStateArray_0[0].setNullAt(0); /* 188 */ } else { /* 189 */ agg_mutableStateArray_0[0].write(0, agg_value_4); /* 190 */ } /* 191 */ append((agg_mutableStateArray_0[0].getRow())); /* 192 */ } /* 193 */ } /* 194 */ /* 195 */ } 20/08/08 15:33:53.846 Thread-4 DEBUG CodeGenerator: /* 001 */ public Object generate(Object[] references) { /* 002 */ return new GeneratedIteratorForCodegenStage2(references); /* 003 */ } /* 004 */ /* 005 */ // codegenStageId=2 /* 006 */ final class GeneratedIteratorForCodegenStage2 extends org.apache.spark.sql.execution.BufferedRowIterator { /* 007 */ private Object[] references; /* 008 */ private scala.collection.Iterator[] inputs; /* 009 */ private boolean agg_initAgg_0; /* 010 */ private boolean agg_bufIsNull_0; /* 011 */ private double agg_bufValue_0; /* 012 */ private boolean agg_bufIsNull_1; /* 013 */ private double agg_bufValue_1; /* 014 */ private scala.collection.Iterator inputadapter_input_0; /* 015 */ private boolean agg_agg_isNull_7_0; /* 016 */ private boolean agg_agg_isNull_9_0; /* 017 */ private boolean agg_agg_isNull_14_0; /* 018 */ private boolean agg_agg_isNull_16_0; /* 019 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[] agg_mutableStateArray_0 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[1]; /* 020 */ /* 021 */ public GeneratedIteratorForCodegenStage2(Object[] references) { /* 022 */ this.references = references; /* 023 */ } /* 024 */ /* 025 */ public void init(int index, scala.collection.Iterator[] inputs) { /* 026 */ partitionIndex = index; /* 027 */ this.inputs = inputs; /* 028 */ /* 029 */ inputadapter_input_0 = inputs[0]; /* 030 */ agg_mutableStateArray_0[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(1, 0); /* 031 */ /* 032 */ } /* 033 */ /* 034 */ private void agg_doAggregate_sum_0(double agg_expr_0_0, boolean agg_exprIsNull_0_0) throws java.io.IOException { /* 035 */ agg_agg_isNull_7_0 = true; /* 036 */ double agg_value_7 = -1.0; /* 037 */ do { /* 038 */ boolean agg_isNull_8 = true; /* 039 */ double agg_value_8 = -1.0; /* 040 */ agg_agg_isNull_9_0 = true; /* 041 */ double agg_value_9 = -1.0; /* 042 */ do { /* 043 */ if (!agg_bufIsNull_0) { /* 044 */ agg_agg_isNull_9_0 = false; /* 045 */ agg_value_9 = agg_bufValue_0; /* 046 */ continue; /* 047 */ } /* 048 */ /* 049 */ if (!false) { /* 050 */ agg_agg_isNull_9_0 = false; /* 051 */ agg_value_9 = 0.0D; /* 052 */ continue; /* 053 */ } /* 054 */ /* 055 */ } while (false); /* 056 */ /* 057 */ if (!agg_exprIsNull_0_0) { /* 058 */ agg_isNull_8 = false; // resultCode could change nullability. /* 059 */ /* 060 */ agg_value_8 = agg_value_9 + agg_expr_0_0; /* 061 */ /* 062 */ } /* 063 */ if (!agg_isNull_8) { /* 064 */ agg_agg_isNull_7_0 = false; /* 065 */ agg_value_7 = agg_value_8; /* 066 */ continue; /* 067 */ } /* 068 */ /* 069 */ if (!agg_bufIsNull_0) { /* 070 */ agg_agg_isNull_7_0 = false; /* 071 */ agg_value_7 = agg_bufValue_0; /* 072 */ continue; /* 073 */ } /* 074 */ /* 075 */ } while (false); /* 076 */ /* 077 */ agg_bufIsNull_0 = agg_agg_isNull_7_0; /* 078 */ agg_bufValue_0 = agg_value_7; /* 079 */ } /* 080 */ /* 081 */ private void agg_doAggregateWithoutKey_0() throws java.io.IOException { /* 082 */ // initialize aggregation buffer /* 083 */ agg_bufIsNull_0 = true; /* 084 */ agg_bufValue_0 = -1.0; /* 085 */ agg_bufIsNull_1 = true; /* 086 */ agg_bufValue_1 = -1.0; /* 087 */ /* 088 */ while ( inputadapter_input_0.hasNext()) { /* 089 */ InternalRow inputadapter_row_0 = (InternalRow) inputadapter_input_0.next(); /* 090 */ /* 091 */ boolean inputadapter_isNull_0 = inputadapter_row_0.isNullAt(0); /* 092 */ double inputadapter_value_0 = inputadapter_isNull_0 ? /* 093 */ -1.0 : (inputadapter_row_0.getDouble(0)); /* 094 */ boolean inputadapter_isNull_1 = inputadapter_row_0.isNullAt(1); /* 095 */ double inputadapter_value_1 = inputadapter_isNull_1 ? /* 096 */ -1.0 : (inputadapter_row_0.getDouble(1)); /* 097 */ /* 098 */ agg_doConsume_0(inputadapter_row_0, inputadapter_value_0, inputadapter_isNull_0, inputadapter_value_1, inputadapter_isNull_1); /* 099 */ // shouldStop check is eliminated /* 100 */ } /* 101 */ /* 102 */ } /* 103 */ /* 104 */ private void agg_doConsume_0(InternalRow inputadapter_row_0, double agg_expr_0_0, boolean agg_exprIsNull_0_0, double agg_expr_1_0, boolean agg_exprIsNull_1_0) throws java.io.IOException { /* 105 */ // do aggregate /* 106 */ // common sub-expressions /* 107 */ /* 108 */ // evaluate aggregate functions and update aggregation buffers /* 109 */ agg_doAggregate_sum_0(agg_expr_0_0, agg_exprIsNull_0_0); /* 110 */ agg_doAggregate_sum_1(agg_exprIsNull_1_0, agg_expr_1_0); /* 111 */ /* 112 */ } /* 113 */ /* 114 */ private void agg_doAggregate_sum_1(boolean agg_exprIsNull_1_0, double agg_expr_1_0) throws java.io.IOException { /* 115 */ agg_agg_isNull_14_0 = true; /* 116 */ double agg_value_14 = -1.0; /* 117 */ do { /* 118 */ boolean agg_isNull_15 = true; /* 119 */ double agg_value_15 = -1.0; /* 120 */ agg_agg_isNull_16_0 = true; /* 121 */ double agg_value_16 = -1.0; /* 122 */ do { /* 123 */ if (!agg_bufIsNull_1) { /* 124 */ agg_agg_isNull_16_0 = false; /* 125 */ agg_value_16 = agg_bufValue_1; /* 126 */ continue; /* 127 */ } /* 128 */ /* 129 */ if (!false) { /* 130 */ agg_agg_isNull_16_0 = false; /* 131 */ agg_value_16 = 0.0D; /* 132 */ continue; /* 133 */ } /* 134 */ /* 135 */ } while (false); /* 136 */ /* 137 */ if (!agg_exprIsNull_1_0) { /* 138 */ agg_isNull_15 = false; // resultCode could change nullability. /* 139 */ /* 140 */ agg_value_15 = agg_value_16 + agg_expr_1_0; /* 141 */ /* 142 */ } /* 143 */ if (!agg_isNull_15) { /* 144 */ agg_agg_isNull_14_0 = false; /* 145 */ agg_value_14 = agg_value_15; /* 146 */ continue; /* 147 */ } /* 148 */ /* 149 */ if (!agg_bufIsNull_1) { /* 150 */ agg_agg_isNull_14_0 = false; /* 151 */ agg_value_14 = agg_bufValue_1; /* 152 */ continue; /* 153 */ } /* 154 */ /* 155 */ } while (false); /* 156 */ /* 157 */ agg_bufIsNull_1 = agg_agg_isNull_14_0; /* 158 */ agg_bufValue_1 = agg_value_14; /* 159 */ } /* 160 */ /* 161 */ protected void processNext() throws java.io.IOException { /* 162 */ while (!agg_initAgg_0) { /* 163 */ agg_initAgg_0 = true; /* 164 */ long agg_beforeAgg_0 = System.nanoTime(); /* 165 */ agg_doAggregateWithoutKey_0(); /* 166 */ ((org.apache.spark.sql.execution.metric.SQLMetric) references[1] /* aggTime */).add((System.nanoTime() - agg_beforeAgg_0) / 1000000); /* 167 */ /* 168 */ // output the result /* 169 */ boolean agg_isNull_4 = false; /* 170 */ double agg_value_4 = -1.0; /* 171 */ if (agg_bufIsNull_1 || agg_bufValue_1 == 0) { /* 172 */ agg_isNull_4 = true; /* 173 */ } else { /* 174 */ if (agg_bufIsNull_0) { /* 175 */ agg_isNull_4 = true; /* 176 */ } else { /* 177 */ agg_value_4 = (double)(agg_bufValue_0 / agg_bufValue_1); /* 178 */ } /* 179 */ } /* 180 */ /* 181 */ ((org.apache.spark.sql.execution.metric.SQLMetric) references[0] /* numOutputRows */).add(1); /* 182 */ agg_mutableStateArray_0[0].reset(); /* 183 */ /* 184 */ agg_mutableStateArray_0[0].zeroOutNullBytes(); /* 185 */ /* 186 */ if (agg_isNull_4) { /* 187 */ agg_mutableStateArray_0[0].setNullAt(0); /* 188 */ } else { /* 189 */ agg_mutableStateArray_0[0].write(0, agg_value_4); /* 190 */ } /* 191 */ append((agg_mutableStateArray_0[0].getRow())); /* 192 */ } /* 193 */ } /* 194 */ /* 195 */ } 20/08/08 15:33:53.855 Executor task launch worker for task 45 INFO Executor: Finished task 2.0 in stage 11.0 (TID 45). 2423 bytes result sent to driver 20/08/08 15:33:53.855 Executor task launch worker for task 45 DEBUG ExecutorMetricsPoller: stageTCMP: (11, 0) -> 1 20/08/08 15:33:53.856 Executor task launch worker for task 52 DEBUG ShuffleBlockFetcherIterator: Got local blocks in 31 ms 20/08/08 15:33:53.856 dispatcher-event-loop-3 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_11.0, runningTasks: 1 20/08/08 15:33:53.857 task-result-getter-1 INFO TaskSetManager: Finished task 2.0 in stage 11.0 (TID 45) in 265 ms on 192.168.1.225 (executor driver) (9/10) 20/08/08 15:33:53.857 Executor task launch worker for task 52 DEBUG TaskMemoryManager: Task 52 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@1be44777 20/08/08 15:33:53.858 dag-scheduler-event-loop DEBUG DAGScheduler: ShuffleMapTask finished on driver 20/08/08 15:33:53.858 Executor task launch worker for task 52 DEBUG TaskMemoryManager: Task 52 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@269f0108 20/08/08 15:33:53.858 Executor task launch worker for task 52 DEBUG TaskMemoryManager: Task 52 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@6bb6f9ab 20/08/08 15:33:53.859 Executor task launch worker for task 52 INFO Executor: Finished task 9.0 in stage 11.0 (TID 52). 2251 bytes result sent to driver 20/08/08 15:33:53.859 Executor task launch worker for task 52 DEBUG ExecutorMetricsPoller: removing (11, 0) from stageTCMP 20/08/08 15:33:53.860 dispatcher-event-loop-0 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_11.0, runningTasks: 0 20/08/08 15:33:53.860 task-result-getter-3 INFO TaskSetManager: Finished task 9.0 in stage 11.0 (TID 52) in 71 ms on 192.168.1.225 (executor driver) (10/10) 20/08/08 15:33:53.861 task-result-getter-3 INFO TaskSchedulerImpl: Removed TaskSet 11.0, whose tasks have all completed, from pool 20/08/08 15:33:53.861 dag-scheduler-event-loop DEBUG DAGScheduler: ShuffleMapTask finished on driver 20/08/08 15:33:53.862 dag-scheduler-event-loop INFO DAGScheduler: ShuffleMapStage 11 (flatMap at ALS.scala:1688) finished in 0.295 s 20/08/08 15:33:53.862 dag-scheduler-event-loop INFO DAGScheduler: looking for newly runnable stages 20/08/08 15:33:53.862 dag-scheduler-event-loop INFO DAGScheduler: running: Set() 20/08/08 15:33:53.862 dag-scheduler-event-loop INFO DAGScheduler: waiting: Set(ResultStage 12) 20/08/08 15:33:53.862 dag-scheduler-event-loop INFO DAGScheduler: failed: Set() 20/08/08 15:33:53.862 dag-scheduler-event-loop DEBUG MapOutputTrackerMaster: Increasing epoch to 5 20/08/08 15:33:53.862 dag-scheduler-event-loop DEBUG DAGScheduler: submitStage(ResultStage 12 (name=count at ALS.scala:1062;jobs=3)) 20/08/08 15:33:53.862 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_5 stored as values in memory (estimated size 4.9 KiB, free 366.2 MiB) 20/08/08 15:33:53.863 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_5 locally took 1 ms 20/08/08 15:33:53.863 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_5 without replication took 1 ms 20/08/08 15:33:53.863 dag-scheduler-event-loop DEBUG DAGScheduler: missing: List() 20/08/08 15:33:53.864 dag-scheduler-event-loop INFO DAGScheduler: Submitting ResultStage 12 (userFactors MapPartitionsRDD[50] at mapPartitions at ALS.scala:1042), which has no missing parents 20/08/08 15:33:53.864 dag-scheduler-event-loop DEBUG DAGScheduler: submitMissingTasks(ResultStage 12) 20/08/08 15:33:53.864 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_5_piece0 stored as bytes in memory (estimated size 2.7 KiB, free 366.2 MiB) 20/08/08 15:33:53.864 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master broadcast_5_piece0 for BlockManagerId(driver, 192.168.1.225, 36433, None) 20/08/08 15:33:53.865 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added broadcast_5_piece0 in memory on 192.168.1.225:36433 (size: 2.7 KiB, free: 366.3 MiB) 20/08/08 15:33:53.873 dag-scheduler-event-loop DEBUG BlockManagerMaster: Updated info of block broadcast_5_piece0 20/08/08 15:33:53.874 dag-scheduler-event-loop DEBUG BlockManager: Told master about block broadcast_5_piece0 20/08/08 15:33:53.874 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_5_piece0 locally took 10 ms 20/08/08 15:33:53.874 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_5_piece0 without replication took 10 ms 20/08/08 15:33:53.874 dag-scheduler-event-loop INFO SparkContext: Created broadcast 5 from broadcast at DAGScheduler.scala:1348 20/08/08 15:33:53.875 dag-scheduler-event-loop INFO DAGScheduler: Submitting 4 missing tasks from ResultStage 3 (MapPartitionsRDD[21] at map at CountVectorizer.scala:230) (first 15 tasks are for partitions Vector(0, 1, 2, 3)) 20/08/08 15:33:53.875 dag-scheduler-event-loop INFO TaskSchedulerImpl: Adding task set 3.0 with 4 tasks resource profile 0 20/08/08 15:33:53.875 dag-scheduler-event-loop DEBUG TaskSetManager: Epoch for TaskSet 3.0: 1 20/08/08 15:33:53.877 dag-scheduler-event-loop DEBUG TaskSetManager: Adding pending tasks took 1 ms 20/08/08 15:33:53.884 Thread-4 INFO CodeGenerator: Code generated in 40.1192 ms 20/08/08 15:33:53.888 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_8 stored as values in memory (estimated size 39.5 KiB, free 365.9 MiB) 20/08/08 15:33:53.888 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_8 locally took 0 ms 20/08/08 15:33:53.888 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_8 without replication took 1 ms 20/08/08 15:33:53.893 dag-scheduler-event-loop DEBUG TaskSetManager: Valid locality levels for TaskSet 3.0: NODE_LOCAL, NO_PREF, ANY 20/08/08 15:33:53.894 dispatcher-event-loop-1 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_3.0, runningTasks: 0 20/08/08 15:33:53.896 dispatcher-event-loop-1 INFO TaskSetManager: Starting task 1.0 in stage 3.0 (TID 12, 192.168.1.225, executor driver, partition 1, NODE_LOCAL, 7143 bytes) taskResourceAssignments Map() 20/08/08 15:33:53.897 dispatcher-event-loop-1 INFO TaskSetManager: Starting task 2.0 in stage 3.0 (TID 13, 192.168.1.225, executor driver, partition 2, NODE_LOCAL, 7143 bytes) taskResourceAssignments Map() 20/08/08 15:33:53.897 dispatcher-event-loop-1 INFO TaskSetManager: Starting task 3.0 in stage 3.0 (TID 14, 192.168.1.225, executor driver, partition 3, NODE_LOCAL, 7143 bytes) taskResourceAssignments Map() 20/08/08 15:33:53.897 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_8_piece0 stored as bytes in memory (estimated size 17.4 KiB, free 365.9 MiB) 20/08/08 15:33:53.897 dispatcher-event-loop-1 DEBUG TaskSetManager: No tasks for locality level NODE_LOCAL, so moving to locality level NO_PREF 20/08/08 15:33:53.897 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master broadcast_8_piece0 for BlockManagerId(driver, 192.168.1.225, 38159, None) 20/08/08 15:33:53.898 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added broadcast_8_piece0 in memory on 192.168.1.225:38159 (size: 17.4 KiB, free: 366.2 MiB) 20/08/08 15:33:53.898 dispatcher-event-loop-1 DEBUG TaskSetManager: Moving to ANY after waiting for 0ms 20/08/08 15:33:53.898 dag-scheduler-event-loop DEBUG BlockManagerMaster: Updated info of block broadcast_8_piece0 20/08/08 15:33:53.898 dag-scheduler-event-loop DEBUG BlockManager: Told master about block broadcast_8_piece0 20/08/08 15:33:53.898 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_8_piece0 locally took 9 ms 20/08/08 15:33:53.898 dispatcher-event-loop-1 INFO TaskSetManager: Starting task 0.0 in stage 3.0 (TID 15, 192.168.1.225, executor driver, partition 0, PROCESS_LOCAL, 7143 bytes) taskResourceAssignments Map() 20/08/08 15:33:53.898 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_8_piece0 without replication took 9 ms 20/08/08 15:33:53.898 dag-scheduler-event-loop INFO SparkContext: Created broadcast 8 from broadcast at DAGScheduler.scala:1348 20/08/08 15:33:53.899 Executor task launch worker for task 13 INFO Executor: Running task 2.0 in stage 3.0 (TID 13) 20/08/08 15:33:53.899 Executor task launch worker for task 15 INFO Executor: Running task 0.0 in stage 3.0 (TID 15) 20/08/08 15:33:53.899 Executor task launch worker for task 14 INFO Executor: Running task 3.0 in stage 3.0 (TID 14) 20/08/08 15:33:53.899 dag-scheduler-event-loop INFO DAGScheduler: Submitting 10 missing tasks from ResultStage 12 (userFactors MapPartitionsRDD[50] at mapPartitions at ALS.scala:1042) (first 15 tasks are for partitions Vector(0, 1, 2, 3, 4, 5, 6, 7, 8, 9)) 20/08/08 15:33:53.899 dag-scheduler-event-loop INFO TaskSchedulerImpl: Adding task set 12.0 with 10 tasks resource profile 0 20/08/08 15:33:53.899 Executor task launch worker for task 12 INFO Executor: Running task 1.0 in stage 3.0 (TID 12) 20/08/08 15:33:53.899 Executor task launch worker for task 14 DEBUG ExecutorMetricsPoller: stageTCMP: (3, 0) -> 1 20/08/08 15:33:53.900 dag-scheduler-event-loop DEBUG TaskSetManager: Epoch for TaskSet 12.0: 5 20/08/08 15:33:53.900 Executor task launch worker for task 15 DEBUG ExecutorMetricsPoller: stageTCMP: (3, 0) -> 2 20/08/08 15:33:53.900 dag-scheduler-event-loop DEBUG TaskSetManager: Adding pending tasks took 0 ms 20/08/08 15:33:53.900 dag-scheduler-event-loop DEBUG TaskSetManager: Valid locality levels for TaskSet 12.0: PROCESS_LOCAL, NODE_LOCAL, ANY 20/08/08 15:33:53.900 Executor task launch worker for task 12 DEBUG ExecutorMetricsPoller: stageTCMP: (3, 0) -> 3 20/08/08 15:33:53.900 dispatcher-event-loop-2 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_12.0, runningTasks: 0 20/08/08 15:33:53.900 Executor task launch worker for task 13 DEBUG ExecutorMetricsPoller: stageTCMP: (3, 0) -> 4 20/08/08 15:33:53.901 Executor task launch worker for task 15 DEBUG BlockManager: Getting local block broadcast_5 20/08/08 15:33:53.901 Executor task launch worker for task 15 DEBUG BlockManager: Level for block broadcast_5 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:53.901 dispatcher-event-loop-2 INFO TaskSetManager: Starting task 0.0 in stage 12.0 (TID 53, 192.168.1.225, executor driver, partition 0, PROCESS_LOCAL, 7433 bytes) taskResourceAssignments Map() 20/08/08 15:33:53.901 dispatcher-event-loop-2 INFO TaskSetManager: Starting task 1.0 in stage 12.0 (TID 54, 192.168.1.225, executor driver, partition 1, PROCESS_LOCAL, 7433 bytes) taskResourceAssignments Map() 20/08/08 15:33:53.901 dispatcher-event-loop-2 INFO TaskSetManager: Starting task 2.0 in stage 12.0 (TID 55, 192.168.1.225, executor driver, partition 2, PROCESS_LOCAL, 7433 bytes) taskResourceAssignments Map() 20/08/08 15:33:53.902 dispatcher-event-loop-2 INFO TaskSetManager: Starting task 3.0 in stage 12.0 (TID 56, 192.168.1.225, executor driver, partition 3, PROCESS_LOCAL, 7433 bytes) taskResourceAssignments Map() 20/08/08 15:33:53.902 Executor task launch worker for task 53 INFO Executor: Running task 0.0 in stage 12.0 (TID 53) 20/08/08 15:33:53.902 Executor task launch worker for task 56 INFO Executor: Running task 3.0 in stage 12.0 (TID 56) 20/08/08 15:33:53.902 Executor task launch worker for task 55 INFO Executor: Running task 2.0 in stage 12.0 (TID 55) 20/08/08 15:33:53.903 Executor task launch worker for task 55 DEBUG ExecutorMetricsPoller: stageTCMP: (12, 0) -> 1 20/08/08 15:33:53.903 Executor task launch worker for task 55 DEBUG BlockManager: Getting local block broadcast_8 20/08/08 15:33:53.904 Executor task launch worker for task 55 DEBUG BlockManager: Level for block broadcast_8 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:53.905 Executor task launch worker for task 14 DEBUG BlockManager: Getting local block rdd_21_3 20/08/08 15:33:53.905 Executor task launch worker for task 15 DEBUG BlockManager: Getting local block rdd_21_0 20/08/08 15:33:53.906 Executor task launch worker for task 15 DEBUG BlockManager: Block rdd_21_0 was not found 20/08/08 15:33:53.902 Executor task launch worker for task 54 INFO Executor: Running task 1.0 in stage 12.0 (TID 54) 20/08/08 15:33:53.906 Executor task launch worker for task 54 DEBUG ExecutorMetricsPoller: stageTCMP: (12, 0) -> 2 20/08/08 15:33:53.907 Executor task launch worker for task 15 DEBUG BlockManager: Getting remote block rdd_21_0 20/08/08 15:33:53.906 Executor task launch worker for task 14 DEBUG BlockManager: Block rdd_21_3 was not found 20/08/08 15:33:53.907 Executor task launch worker for task 14 DEBUG BlockManager: Getting remote block rdd_21_3 20/08/08 15:33:53.909 Executor task launch worker for task 13 DEBUG BlockManager: Getting local block rdd_21_2 20/08/08 15:33:53.909 Executor task launch worker for task 13 DEBUG BlockManager: Block rdd_21_2 was not found 20/08/08 15:33:53.909 Executor task launch worker for task 13 DEBUG BlockManager: Getting remote block rdd_21_2 20/08/08 15:33:53.909 Executor task launch worker for task 56 DEBUG ExecutorMetricsPoller: stageTCMP: (12, 0) -> 3 20/08/08 15:33:53.909 Executor task launch worker for task 12 DEBUG BlockManager: Getting local block rdd_21_1 20/08/08 15:33:53.910 Executor task launch worker for task 12 DEBUG BlockManager: Block rdd_21_1 was not found 20/08/08 15:33:53.910 Executor task launch worker for task 12 DEBUG BlockManager: Getting remote block rdd_21_1 20/08/08 15:33:53.910 Executor task launch worker for task 14 DEBUG BlockManager: Block rdd_21_3 is unknown by block manager master 20/08/08 15:33:53.910 Executor task launch worker for task 13 DEBUG BlockManager: Block rdd_21_2 is unknown by block manager master 20/08/08 15:33:53.910 Executor task launch worker for task 12 DEBUG BlockManager: Block rdd_21_1 is unknown by block manager master 20/08/08 15:33:53.911 Executor task launch worker for task 15 DEBUG BlockManager: Block rdd_21_0 is unknown by block manager master 20/08/08 15:33:53.911 Executor task launch worker for task 53 DEBUG ExecutorMetricsPoller: stageTCMP: (12, 0) -> 4 20/08/08 15:33:53.911 Executor task launch worker for task 15 DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 0 20/08/08 15:33:53.911 Executor task launch worker for task 13 DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 0 20/08/08 15:33:53.914 Executor task launch worker for task 12 DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 0 20/08/08 15:33:53.914 Executor task launch worker for task 13 DEBUG MapOutputTrackerMaster: Convert map statuses for shuffle 0, mappers 0-4, partitions 2-3 20/08/08 15:33:53.914 Executor task launch worker for task 14 DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 0 20/08/08 15:33:53.914 Executor task launch worker for task 14 DEBUG MapOutputTrackerMaster: Convert map statuses for shuffle 0, mappers 0-4, partitions 3-4 20/08/08 15:33:53.914 Executor task launch worker for task 12 DEBUG MapOutputTrackerMaster: Convert map statuses for shuffle 0, mappers 0-4, partitions 1-2 20/08/08 15:33:53.915 Executor task launch worker for task 55 DEBUG BlockManager: Getting local block rdd_50_2 20/08/08 15:33:53.915 Executor task launch worker for task 55 DEBUG BlockManager: Block rdd_50_2 was not found 20/08/08 15:33:53.915 Executor task launch worker for task 55 DEBUG BlockManager: Getting remote block rdd_50_2 20/08/08 15:33:53.915 Executor task launch worker for task 15 DEBUG MapOutputTrackerMaster: Convert map statuses for shuffle 0, mappers 0-4, partitions 0-1 20/08/08 15:33:53.917 Executor task launch worker for task 56 DEBUG BlockManager: Getting local block rdd_50_3 20/08/08 15:33:53.917 Executor task launch worker for task 56 DEBUG BlockManager: Block rdd_50_3 was not found 20/08/08 15:33:53.917 Executor task launch worker for task 56 DEBUG BlockManager: Getting remote block rdd_50_3 20/08/08 15:33:53.917 Executor task launch worker for task 56 DEBUG BlockManager: Block rdd_50_3 is unknown by block manager master 20/08/08 15:33:53.918 Executor task launch worker for task 56 DEBUG BlockManager: Getting local block rdd_19_3 20/08/08 15:33:53.918 Executor task launch worker for task 56 DEBUG BlockManager: Level for block rdd_19_3 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:53.918 Executor task launch worker for task 56 INFO BlockManager: Found block rdd_19_3 locally 20/08/08 15:33:53.918 Executor task launch worker for task 56 DEBUG BlockManager: Getting local block rdd_19_3 20/08/08 15:33:53.918 Executor task launch worker for task 56 DEBUG BlockManager: Level for block rdd_19_3 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:53.918 Executor task launch worker for task 56 INFO BlockManager: Found block rdd_19_3 locally 20/08/08 15:33:53.918 Executor task launch worker for task 56 DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 3 20/08/08 15:33:53.918 Executor task launch worker for task 56 DEBUG MapOutputTrackerMaster: Convert map statuses for shuffle 3, mappers 0-10, partitions 3-4 20/08/08 15:33:53.919 Executor task launch worker for task 56 DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 20/08/08 15:33:53.919 Executor task launch worker for task 54 DEBUG BlockManager: Getting local block rdd_50_1 20/08/08 15:33:53.919 Executor task launch worker for task 54 DEBUG BlockManager: Block rdd_50_1 was not found 20/08/08 15:33:53.919 Executor task launch worker for task 54 DEBUG BlockManager: Getting remote block rdd_50_1 20/08/08 15:33:53.919 Executor task launch worker for task 55 DEBUG BlockManager: Block rdd_50_2 is unknown by block manager master 20/08/08 15:33:53.919 Executor task launch worker for task 55 DEBUG BlockManager: Getting local block rdd_19_2 20/08/08 15:33:53.919 Executor task launch worker for task 55 DEBUG BlockManager: Level for block rdd_19_2 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:53.920 Executor task launch worker for task 55 INFO BlockManager: Found block rdd_19_2 locally 20/08/08 15:33:53.920 Executor task launch worker for task 55 DEBUG BlockManager: Getting local block rdd_19_2 20/08/08 15:33:53.919 Executor task launch worker for task 56 INFO ShuffleBlockFetcherIterator: Getting 3 (684.0 B) non-empty blocks including 3 (684.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 20/08/08 15:33:53.920 Executor task launch worker for task 53 DEBUG BlockManager: Getting local block rdd_50_0 20/08/08 15:33:53.920 Executor task launch worker for task 53 DEBUG BlockManager: Block rdd_50_0 was not found 20/08/08 15:33:53.920 Executor task launch worker for task 53 DEBUG BlockManager: Getting remote block rdd_50_0 20/08/08 15:33:53.920 Executor task launch worker for task 55 DEBUG BlockManager: Level for block rdd_19_2 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:53.920 Executor task launch worker for task 55 INFO BlockManager: Found block rdd_19_2 locally 20/08/08 15:33:53.920 Executor task launch worker for task 55 DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 3 20/08/08 15:33:53.920 Executor task launch worker for task 55 DEBUG MapOutputTrackerMaster: Convert map statuses for shuffle 3, mappers 0-10, partitions 2-3 20/08/08 15:33:53.920 Executor task launch worker for task 53 DEBUG BlockManager: Block rdd_50_0 is unknown by block manager master 20/08/08 15:33:53.920 Executor task launch worker for task 53 DEBUG BlockManager: Getting local block rdd_19_0 20/08/08 15:33:53.921 Executor task launch worker for task 55 DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 20/08/08 15:33:53.921 Executor task launch worker for task 54 DEBUG BlockManager: Block rdd_50_1 is unknown by block manager master 20/08/08 15:33:53.921 Executor task launch worker for task 54 DEBUG BlockManager: Getting local block rdd_19_1 20/08/08 15:33:53.921 Executor task launch worker for task 55 INFO ShuffleBlockFetcherIterator: Getting 3 (707.0 B) non-empty blocks including 3 (707.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 20/08/08 15:33:53.921 Executor task launch worker for task 55 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 20/08/08 15:33:53.921 Executor task launch worker for task 55 DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_3_43_2,0), (shuffle_3_44_2,1), (shuffle_3_45_2,2) 20/08/08 15:33:53.921 Executor task launch worker for task 55 INFO BlockManager: Getting local shuffle block shuffle_3_43_2 20/08/08 15:33:53.921 Executor task launch worker for task 55 INFO BlockManager: Getting local shuffle block shuffle_3_44_2 20/08/08 15:33:53.922 Executor task launch worker for task 55 INFO BlockManager: Getting local shuffle block shuffle_3_45_2 20/08/08 15:33:53.922 Executor task launch worker for task 55 DEBUG ShuffleBlockFetcherIterator: Got local blocks in 1 ms 20/08/08 15:33:53.922 Thread-4 DEBUG WholeStageCodegenExec: /* 001 */ public Object generate(Object[] references) { /* 002 */ return new GeneratedIteratorForCodegenStage1(references); /* 003 */ } /* 004 */ /* 005 */ // codegenStageId=1 /* 006 */ final class GeneratedIteratorForCodegenStage1 extends org.apache.spark.sql.execution.BufferedRowIterator { /* 007 */ private Object[] references; /* 008 */ private scala.collection.Iterator[] inputs; /* 009 */ private boolean agg_initAgg_0; /* 010 */ private boolean agg_bufIsNull_0; /* 011 */ private double agg_bufValue_0; /* 012 */ private boolean agg_bufIsNull_1; /* 013 */ private double agg_bufValue_1; /* 014 */ private scala.collection.Iterator rdd_input_0; /* 015 */ private boolean agg_agg_isNull_4_0; /* 016 */ private boolean agg_agg_isNull_6_0; /* 017 */ private boolean agg_agg_isNull_22_0; /* 018 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[] rdd_mutableStateArray_0 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[7]; /* 019 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[] rdd_mutableStateArray_1 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[6]; /* 020 */ /* 021 */ public GeneratedIteratorForCodegenStage1(Object[] references) { /* 022 */ this.references = references; /* 023 */ } /* 024 */ /* 025 */ public void init(int index, scala.collection.Iterator[] inputs) { /* 026 */ partitionIndex = index; /* 027 */ this.inputs = inputs; /* 028 */ wholestagecodegen_init_0_0(); /* 029 */ wholestagecodegen_init_0_1(); /* 030 */ /* 031 */ } /* 032 */ /* 033 */ private void agg_doAggregate_sum_0(boolean agg_exprIsNull_1_0, org.apache.spark.sql.catalyst.InternalRow agg_expr_1_0, double agg_expr_0_0, boolean agg_exprIsNull_0_0) throws java.io.IOException { /* 034 */ agg_agg_isNull_4_0 = true; /* 035 */ double agg_value_4 = -1.0; /* 036 */ do { /* 037 */ boolean agg_isNull_5 = true; /* 038 */ double agg_value_5 = -1.0; /* 039 */ agg_agg_isNull_6_0 = true; /* 040 */ double agg_value_6 = -1.0; /* 041 */ do { /* 042 */ if (!agg_bufIsNull_0) { /* 043 */ agg_agg_isNull_6_0 = false; /* 044 */ agg_value_6 = agg_bufValue_0; /* 045 */ continue; /* 046 */ } /* 047 */ /* 048 */ if (!false) { /* 049 */ agg_agg_isNull_6_0 = false; /* 050 */ agg_value_6 = 0.0D; /* 051 */ continue; /* 052 */ } /* 053 */ /* 054 */ } while (false); /* 055 */ boolean agg_isNull_9 = true; /* 056 */ double agg_value_9 = -1.0; /* 057 */ boolean agg_isNull_10 = false; /* 058 */ double agg_value_10 = -1.0; /* 059 */ if (!false && agg_exprIsNull_0_0) { /* 060 */ agg_isNull_10 = true; /* 061 */ agg_value_10 = -1.0; /* 062 */ } else { /* 063 */ Object agg_arg_0 = null; /* 064 */ if (agg_exprIsNull_1_0) { /* 065 */ agg_arg_0 = ((scala.Function1[]) references[4] /* converters */)[0].apply(null); /* 066 */ } else { /* 067 */ agg_arg_0 = ((scala.Function1[]) references[4] /* converters */)[0].apply(agg_expr_1_0); /* 068 */ } /* 069 */ /* 070 */ Double agg_conv_0 = agg_expr_0_0; /* 071 */ Object agg_arg_1 = false ? null : agg_conv_0; /* 072 */ /* 073 */ Double agg_conv_1 = 1.0D; /* 074 */ Object agg_arg_2 = false ? null : agg_conv_1; /* 075 */ /* 076 */ Double agg_result_0 = null; /* 077 */ try { /* 078 */ agg_result_0 = (Double)((scala.Function3) references[6] /* udf */).apply(agg_arg_0, agg_arg_1, agg_arg_2); /* 079 */ } catch (Exception e) { /* 080 */ throw new org.apache.spark.SparkException(((java.lang.String) references[5] /* errMsg */), e); /* 081 */ } /* 082 */ /* 083 */ boolean agg_isNull_14 = agg_result_0 == null; /* 084 */ double agg_value_14 = -1.0; /* 085 */ if (!agg_isNull_14) { /* 086 */ agg_value_14 = agg_result_0; /* 087 */ } /* 088 */ agg_isNull_10 = agg_isNull_14; /* 089 */ agg_value_10 = agg_value_14; /* 090 */ } /* 091 */ if (!agg_isNull_10) { /* 092 */ agg_isNull_9 = false; // resultCode could change nullability. /* 093 */ /* 094 */ agg_value_9 = agg_value_10 * 1.0D; /* 095 */ /* 096 */ } /* 097 */ if (!agg_isNull_9) { /* 098 */ agg_isNull_5 = false; // resultCode could change nullability. /* 099 */ /* 100 */ agg_value_5 = agg_value_6 + agg_value_9; /* 101 */ /* 102 */ } /* 103 */ if (!agg_isNull_5) { /* 104 */ agg_agg_isNull_4_0 = false; /* 105 */ agg_value_4 = agg_value_5; /* 106 */ continue; /* 107 */ } /* 108 */ /* 109 */ if (!agg_bufIsNull_0) { /* 110 */ agg_agg_isNull_4_0 = false; /* 111 */ agg_value_4 = agg_bufValue_0; /* 112 */ continue; /* 113 */ } /* 114 */ /* 115 */ } while (false); /* 116 */ /* 117 */ agg_bufIsNull_0 = agg_agg_isNull_4_0; /* 118 */ agg_bufValue_0 = agg_value_4; /* 119 */ } /* 120 */ /* 121 */ private void agg_doAggregateWithoutKey_0() throws java.io.IOException { /* 122 */ // initialize aggregation buffer /* 123 */ agg_bufIsNull_0 = true; /* 124 */ agg_bufValue_0 = -1.0; /* 125 */ agg_bufIsNull_1 = true; /* 126 */ agg_bufValue_1 = -1.0; /* 127 */ /* 128 */ while ( rdd_input_0.hasNext()) { /* 129 */ InternalRow rdd_row_0 = (InternalRow) rdd_input_0.next(); /* 130 */ ((org.apache.spark.sql.execution.metric.SQLMetric) references[0] /* numOutputRows */).add(1); /* 131 */ boolean rdd_isNull_1 = rdd_row_0.isNullAt(1); /* 132 */ double rdd_value_1 = rdd_isNull_1 ? /* 133 */ -1.0 : (rdd_row_0.getDouble(1)); /* 134 */ boolean rdd_isNull_0 = rdd_row_0.isNullAt(0); /* 135 */ InternalRow rdd_value_0 = rdd_isNull_0 ? /* 136 */ null : (rdd_row_0.getStruct(0, 4)); /* 137 */ /* 138 */ Object project_arg_0 = null; /* 139 */ if (rdd_isNull_0) { /* 140 */ project_arg_0 = ((scala.Function1[]) references[1] /* converters */)[0].apply(null); /* 141 */ } else { /* 142 */ project_arg_0 = ((scala.Function1[]) references[1] /* converters */)[0].apply(rdd_value_0); /* 143 */ } /* 144 */ /* 145 */ InternalRow project_result_0 = null; /* 146 */ try { /* 147 */ project_result_0 = (InternalRow)((scala.Function1[]) references[1] /* converters */)[1].apply(((scala.Function1) references[3] /* udf */).apply(project_arg_0)); /* 148 */ } catch (Exception e) { /* 149 */ throw new org.apache.spark.SparkException(((java.lang.String) references[2] /* errMsg */), e); /* 150 */ } /* 151 */ /* 152 */ boolean project_isNull_1 = project_result_0 == null; /* 153 */ InternalRow project_value_1 = null; /* 154 */ if (!project_isNull_1) { /* 155 */ project_value_1 = project_result_0; /* 156 */ } /* 157 */ /* 158 */ agg_doConsume_0(rdd_value_1, rdd_isNull_1, project_value_1, project_isNull_1); /* 159 */ // shouldStop check is eliminated /* 160 */ } /* 161 */ /* 162 */ } /* 163 */ /* 164 */ private void project_writeFields_0_1(InternalRow project_tmpInput_0) { /* 165 */ if ((project_tmpInput_0.isNullAt(3))) { /* 166 */ rdd_mutableStateArray_0[3].setNullAt(3); /* 167 */ } else { /* 168 */ // Remember the current cursor so that we can calculate how many bytes are /* 169 */ // written later. /* 170 */ final int project_previousCursor_2 = rdd_mutableStateArray_0[3].cursor(); /* 171 */ /* 172 */ final ArrayData project_tmpInput_2 = (project_tmpInput_0.getArray(3)); /* 173 */ if (project_tmpInput_2 instanceof UnsafeArrayData) { /* 174 */ rdd_mutableStateArray_0[3].write((UnsafeArrayData) project_tmpInput_2); /* 175 */ } else { /* 176 */ final int project_numElements_1 = project_tmpInput_2.numElements(); /* 177 */ rdd_mutableStateArray_1[3].initialize(project_numElements_1); /* 178 */ /* 179 */ for (int project_index_1 = 0; project_index_1 < project_numElements_1; project_index_1++) { /* 180 */ rdd_mutableStateArray_1[3].write(project_index_1, project_tmpInput_2.getDouble(project_index_1)); /* 181 */ } /* 182 */ } /* 183 */ /* 184 */ rdd_mutableStateArray_0[3].setOffsetAndSizeFromPreviousCursor(3, project_previousCursor_2); /* 185 */ } /* 186 */ /* 187 */ } /* 188 */ /* 189 */ private void wholestagecodegen_init_0_1() { /* 190 */ rdd_mutableStateArray_0[4] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(2, 32); /* 191 */ rdd_mutableStateArray_0[5] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(rdd_mutableStateArray_0[4], 4); /* 192 */ rdd_mutableStateArray_1[4] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[5], 4); /* 193 */ rdd_mutableStateArray_1[5] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[5], 8); /* 194 */ rdd_mutableStateArray_0[6] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(2, 0); /* 195 */ /* 196 */ } /* 197 */ /* 198 */ private void agg_doConsume_0(double agg_expr_0_0, boolean agg_exprIsNull_0_0, InternalRow agg_expr_1_0, boolean agg_exprIsNull_1_0) throws java.io.IOException { /* 199 */ // do aggregate /* 200 */ // common sub-expressions /* 201 */ /* 202 */ // evaluate aggregate functions and update aggregation buffers /* 203 */ agg_doAggregate_sum_0(agg_exprIsNull_1_0, agg_expr_1_0, agg_expr_0_0, agg_exprIsNull_0_0); /* 204 */ agg_doAggregate_sum_1(); /* 205 */ /* 206 */ } /* 207 */ /* 208 */ private void agg_doAggregate_sum_1() throws java.io.IOException { /* 209 */ agg_agg_isNull_22_0 = true; /* 210 */ double agg_value_22 = -1.0; /* 211 */ do { /* 212 */ if (!agg_bufIsNull_1) { /* 213 */ agg_agg_isNull_22_0 = false; /* 214 */ agg_value_22 = agg_bufValue_1; /* 215 */ continue; /* 216 */ } /* 217 */ /* 218 */ if (!false) { /* 219 */ agg_agg_isNull_22_0 = false; /* 220 */ agg_value_22 = 0.0D; /* 221 */ continue; /* 222 */ } /* 223 */ /* 224 */ } while (false); /* 225 */ /* 226 */ double agg_value_21 = -1.0; /* 227 */ /* 228 */ agg_value_21 = agg_value_22 + 1.0D; /* 229 */ /* 230 */ agg_bufIsNull_1 = false; /* 231 */ agg_bufValue_1 = agg_value_21; /* 232 */ } /* 233 */ /* 234 */ private void project_writeFields_0_0(InternalRow project_tmpInput_0) { /* 235 */ rdd_mutableStateArray_0[3].write(0, (project_tmpInput_0.getByte(0))); /* 236 */ /* 237 */ if ((project_tmpInput_0.isNullAt(1))) { /* 238 */ rdd_mutableStateArray_0[3].setNullAt(1); /* 239 */ } else { /* 240 */ rdd_mutableStateArray_0[3].write(1, (project_tmpInput_0.getInt(1))); /* 241 */ } /* 242 */ /* 243 */ if ((project_tmpInput_0.isNullAt(2))) { /* 244 */ rdd_mutableStateArray_0[3].setNullAt(2); /* 245 */ } else { /* 246 */ // Remember the current cursor so that we can calculate how many bytes are /* 247 */ // written later. /* 248 */ final int project_previousCursor_1 = rdd_mutableStateArray_0[3].cursor(); /* 249 */ /* 250 */ final ArrayData project_tmpInput_1 = (project_tmpInput_0.getArray(2)); /* 251 */ if (project_tmpInput_1 instanceof UnsafeArrayData) { /* 252 */ rdd_mutableStateArray_0[3].write((UnsafeArrayData) project_tmpInput_1); /* 253 */ } else { /* 254 */ final int project_numElements_0 = project_tmpInput_1.numElements(); /* 255 */ rdd_mutableStateArray_1[2].initialize(project_numElements_0); /* 256 */ /* 257 */ for (int project_index_0 = 0; project_index_0 < project_numElements_0; project_index_0++) { /* 258 */ rdd_mutableStateArray_1[2].write(project_index_0, project_tmpInput_1.getInt(project_index_0)); /* 259 */ } /* 260 */ } /* 261 */ /* 262 */ rdd_mutableStateArray_0[3].setOffsetAndSizeFromPreviousCursor(2, project_previousCursor_1); /* 263 */ } /* 264 */ /* 265 */ } /* 266 */ /* 267 */ protected void processNext() throws java.io.IOException { /* 268 */ while (!agg_initAgg_0) { /* 269 */ agg_initAgg_0 = true; /* 270 */ long agg_beforeAgg_0 = System.nanoTime(); /* 271 */ agg_doAggregateWithoutKey_0(); /* 272 */ ((org.apache.spark.sql.execution.metric.SQLMetric) references[8] /* aggTime */).add((System.nanoTime() - agg_beforeAgg_0) / 1000000); /* 273 */ /* 274 */ // output the result /* 275 */ /* 276 */ ((org.apache.spark.sql.execution.metric.SQLMetric) references[7] /* numOutputRows */).add(1); /* 277 */ rdd_mutableStateArray_0[6].reset(); /* 278 */ /* 279 */ rdd_mutableStateArray_0[6].zeroOutNullBytes(); /* 280 */ /* 281 */ if (agg_bufIsNull_0) { /* 282 */ rdd_mutableStateArray_0[6].setNullAt(0); /* 283 */ } else { /* 284 */ rdd_mutableStateArray_0[6].write(0, agg_bufValue_0); /* 285 */ } /* 286 */ /* 287 */ if (agg_bufIsNull_1) { /* 288 */ rdd_mutableStateArray_0[6].setNullAt(1); /* 289 */ } else { /* 290 */ rdd_mutableStateArray_0[6].write(1, agg_bufValue_1); /* 291 */ } /* 292 */ append((rdd_mutableStateArray_0[6].getRow())); /* 293 */ } /* 294 */ } /* 295 */ /* 296 */ private void wholestagecodegen_init_0_0() { /* 297 */ rdd_input_0 = inputs[0]; /* 298 */ rdd_mutableStateArray_0[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(2, 32); /* 299 */ rdd_mutableStateArray_0[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(rdd_mutableStateArray_0[0], 4); /* 300 */ rdd_mutableStateArray_1[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[1], 4); /* 301 */ rdd_mutableStateArray_1[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[1], 8); /* 302 */ rdd_mutableStateArray_0[2] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(2, 32); /* 303 */ rdd_mutableStateArray_0[3] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(rdd_mutableStateArray_0[2], 4); /* 304 */ rdd_mutableStateArray_1[2] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[3], 4); /* 305 */ rdd_mutableStateArray_1[3] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[3], 8); /* 306 */ /* 307 */ } /* 308 */ /* 309 */ } 20/08/08 15:33:53.920 Executor task launch worker for task 56 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 1 ms 20/08/08 15:33:53.921 Executor task launch worker for task 54 DEBUG BlockManager: Level for block rdd_19_1 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:53.923 Executor task launch worker for task 54 INFO BlockManager: Found block rdd_19_1 locally 20/08/08 15:33:53.923 Executor task launch worker for task 54 DEBUG BlockManager: Getting local block rdd_19_1 20/08/08 15:33:53.923 Executor task launch worker for task 54 DEBUG BlockManager: Level for block rdd_19_1 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:53.923 Executor task launch worker for task 54 INFO BlockManager: Found block rdd_19_1 locally 20/08/08 15:33:53.923 Executor task launch worker for task 54 DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 3 20/08/08 15:33:53.923 Executor task launch worker for task 54 DEBUG MapOutputTrackerMaster: Convert map statuses for shuffle 3, mappers 0-10, partitions 1-2 20/08/08 15:33:53.924 Executor task launch worker for task 55 DEBUG TaskMemoryManager: Task 55 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@5d4cfbad 20/08/08 15:33:53.924 Executor task launch worker for task 54 DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 20/08/08 15:33:53.921 Executor task launch worker for task 53 DEBUG BlockManager: Level for block rdd_19_0 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:53.924 Executor task launch worker for task 53 INFO BlockManager: Found block rdd_19_0 locally 20/08/08 15:33:53.924 Executor task launch worker for task 53 DEBUG BlockManager: Getting local block rdd_19_0 20/08/08 15:33:53.924 Executor task launch worker for task 53 DEBUG BlockManager: Level for block rdd_19_0 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:53.924 Executor task launch worker for task 53 INFO BlockManager: Found block rdd_19_0 locally 20/08/08 15:33:53.925 Executor task launch worker for task 53 DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 3 20/08/08 15:33:53.925 Executor task launch worker for task 55 DEBUG TaskMemoryManager: Task 55 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@1bd3dd2f 20/08/08 15:33:53.926 Executor task launch worker for task 55 DEBUG TaskMemoryManager: Task 55 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@37dc49d3 20/08/08 15:33:53.927 Executor task launch worker for task 55 INFO MemoryStore: Block rdd_50_2 stored as values in memory (estimated size 88.0 B, free 365.9 MiB) 20/08/08 15:33:53.927 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_50_2 for BlockManagerId(driver, 192.168.1.225, 38159, None) 20/08/08 15:33:53.927 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_50_2 in memory on 192.168.1.225:38159 (size: 88.0 B, free: 366.2 MiB) 20/08/08 15:33:53.923 Executor task launch worker for task 56 DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_3_43_3,0), (shuffle_3_44_3,1), (shuffle_3_45_3,2) 20/08/08 15:33:53.928 Thread-4 DEBUG CodeGenerator: /* 001 */ public Object generate(Object[] references) { /* 002 */ return new GeneratedIteratorForCodegenStage1(references); /* 003 */ } /* 004 */ /* 005 */ // codegenStageId=1 /* 006 */ final class GeneratedIteratorForCodegenStage1 extends org.apache.spark.sql.execution.BufferedRowIterator { /* 007 */ private Object[] references; /* 008 */ private scala.collection.Iterator[] inputs; /* 009 */ private boolean agg_initAgg_0; /* 010 */ private boolean agg_bufIsNull_0; /* 011 */ private double agg_bufValue_0; /* 012 */ private boolean agg_bufIsNull_1; /* 013 */ private double agg_bufValue_1; /* 014 */ private scala.collection.Iterator rdd_input_0; /* 015 */ private boolean agg_agg_isNull_4_0; /* 016 */ private boolean agg_agg_isNull_6_0; /* 017 */ private boolean agg_agg_isNull_22_0; /* 018 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[] rdd_mutableStateArray_0 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[7]; /* 019 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[] rdd_mutableStateArray_1 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[6]; /* 020 */ /* 021 */ public GeneratedIteratorForCodegenStage1(Object[] references) { /* 022 */ this.references = references; /* 023 */ } /* 024 */ /* 025 */ public void init(int index, scala.collection.Iterator[] inputs) { /* 026 */ partitionIndex = index; /* 027 */ this.inputs = inputs; /* 028 */ wholestagecodegen_init_0_0(); /* 029 */ wholestagecodegen_init_0_1(); /* 030 */ /* 031 */ } /* 032 */ /* 033 */ private void agg_doAggregate_sum_0(boolean agg_exprIsNull_1_0, org.apache.spark.sql.catalyst.InternalRow agg_expr_1_0, double agg_expr_0_0, boolean agg_exprIsNull_0_0) throws java.io.IOException { /* 034 */ agg_agg_isNull_4_0 = true; /* 035 */ double agg_value_4 = -1.0; /* 036 */ do { /* 037 */ boolean agg_isNull_5 = true; /* 038 */ double agg_value_5 = -1.0; /* 039 */ agg_agg_isNull_6_0 = true; /* 040 */ double agg_value_6 = -1.0; /* 041 */ do { /* 042 */ if (!agg_bufIsNull_0) { /* 043 */ agg_agg_isNull_6_0 = false; /* 044 */ agg_value_6 = agg_bufValue_0; /* 045 */ continue; /* 046 */ } /* 047 */ /* 048 */ if (!false) { /* 049 */ agg_agg_isNull_6_0 = false; /* 050 */ agg_value_6 = 0.0D; /* 051 */ continue; /* 052 */ } /* 053 */ /* 054 */ } while (false); /* 055 */ boolean agg_isNull_9 = true; /* 056 */ double agg_value_9 = -1.0; /* 057 */ boolean agg_isNull_10 = false; /* 058 */ double agg_value_10 = -1.0; /* 059 */ if (!false && agg_exprIsNull_0_0) { /* 060 */ agg_isNull_10 = true; /* 061 */ agg_value_10 = -1.0; /* 062 */ } else { /* 063 */ Object agg_arg_0 = null; /* 064 */ if (agg_exprIsNull_1_0) { /* 065 */ agg_arg_0 = ((scala.Function1[]) references[4] /* converters */)[0].apply(null); /* 066 */ } else { /* 067 */ agg_arg_0 = ((scala.Function1[]) references[4] /* converters */)[0].apply(agg_expr_1_0); /* 068 */ } /* 069 */ /* 070 */ Double agg_conv_0 = agg_expr_0_0; /* 071 */ Object agg_arg_1 = false ? null : agg_conv_0; /* 072 */ /* 073 */ Double agg_conv_1 = 1.0D; /* 074 */ Object agg_arg_2 = false ? null : agg_conv_1; /* 075 */ /* 076 */ Double agg_result_0 = null; /* 077 */ try { /* 078 */ agg_result_0 = (Double)((scala.Function3) references[6] /* udf */).apply(agg_arg_0, agg_arg_1, agg_arg_2); /* 079 */ } catch (Exception e) { /* 080 */ throw new org.apache.spark.SparkException(((java.lang.String) references[5] /* errMsg */), e); /* 081 */ } /* 082 */ /* 083 */ boolean agg_isNull_14 = agg_result_0 == null; /* 084 */ double agg_value_14 = -1.0; /* 085 */ if (!agg_isNull_14) { /* 086 */ agg_value_14 = agg_result_0; /* 087 */ } /* 088 */ agg_isNull_10 = agg_isNull_14; /* 089 */ agg_value_10 = agg_value_14; /* 090 */ } /* 091 */ if (!agg_isNull_10) { /* 092 */ agg_isNull_9 = false; // resultCode could change nullability. /* 093 */ /* 094 */ agg_value_9 = agg_value_10 * 1.0D; /* 095 */ /* 096 */ } /* 097 */ if (!agg_isNull_9) { /* 098 */ agg_isNull_5 = false; // resultCode could change nullability. /* 099 */ /* 100 */ agg_value_5 = agg_value_6 + agg_value_9; /* 101 */ /* 102 */ } /* 103 */ if (!agg_isNull_5) { /* 104 */ agg_agg_isNull_4_0 = false; /* 105 */ agg_value_4 = agg_value_5; /* 106 */ continue; /* 107 */ } /* 108 */ /* 109 */ if (!agg_bufIsNull_0) { /* 110 */ agg_agg_isNull_4_0 = false; /* 111 */ agg_value_4 = agg_bufValue_0; /* 112 */ continue; /* 113 */ } /* 114 */ /* 115 */ } while (false); /* 116 */ /* 117 */ agg_bufIsNull_0 = agg_agg_isNull_4_0; /* 118 */ agg_bufValue_0 = agg_value_4; /* 119 */ } /* 120 */ /* 121 */ private void agg_doAggregateWithoutKey_0() throws java.io.IOException { /* 122 */ // initialize aggregation buffer /* 123 */ agg_bufIsNull_0 = true; /* 124 */ agg_bufValue_0 = -1.0; /* 125 */ agg_bufIsNull_1 = true; /* 126 */ agg_bufValue_1 = -1.0; /* 127 */ /* 128 */ while ( rdd_input_0.hasNext()) { /* 129 */ InternalRow rdd_row_0 = (InternalRow) rdd_input_0.next(); /* 130 */ ((org.apache.spark.sql.execution.metric.SQLMetric) references[0] /* numOutputRows */).add(1); /* 131 */ boolean rdd_isNull_1 = rdd_row_0.isNullAt(1); /* 132 */ double rdd_value_1 = rdd_isNull_1 ? /* 133 */ -1.0 : (rdd_row_0.getDouble(1)); /* 134 */ boolean rdd_isNull_0 = rdd_row_0.isNullAt(0); /* 135 */ InternalRow rdd_value_0 = rdd_isNull_0 ? /* 136 */ null : (rdd_row_0.getStruct(0, 4)); /* 137 */ /* 138 */ Object project_arg_0 = null; /* 139 */ if (rdd_isNull_0) { /* 140 */ project_arg_0 = ((scala.Function1[]) references[1] /* converters */)[0].apply(null); /* 141 */ } else { /* 142 */ project_arg_0 = ((scala.Function1[]) references[1] /* converters */)[0].apply(rdd_value_0); /* 143 */ } /* 144 */ /* 145 */ InternalRow project_result_0 = null; /* 146 */ try { /* 147 */ project_result_0 = (InternalRow)((scala.Function1[]) references[1] /* converters */)[1].apply(((scala.Function1) references[3] /* udf */).apply(project_arg_0)); /* 148 */ } catch (Exception e) { /* 149 */ throw new org.apache.spark.SparkException(((java.lang.String) references[2] /* errMsg */), e); /* 150 */ } /* 151 */ /* 152 */ boolean project_isNull_1 = project_result_0 == null; /* 153 */ InternalRow project_value_1 = null; /* 154 */ if (!project_isNull_1) { /* 155 */ project_value_1 = project_result_0; /* 156 */ } /* 157 */ /* 158 */ agg_doConsume_0(rdd_value_1, rdd_isNull_1, project_value_1, project_isNull_1); /* 159 */ // shouldStop check is eliminated /* 160 */ } /* 161 */ /* 162 */ } /* 163 */ /* 164 */ private void project_writeFields_0_1(InternalRow project_tmpInput_0) { /* 165 */ if ((project_tmpInput_0.isNullAt(3))) { /* 166 */ rdd_mutableStateArray_0[3].setNullAt(3); /* 167 */ } else { /* 168 */ // Remember the current cursor so that we can calculate how many bytes are /* 169 */ // written later. /* 170 */ final int project_previousCursor_2 = rdd_mutableStateArray_0[3].cursor(); /* 171 */ /* 172 */ final ArrayData project_tmpInput_2 = (project_tmpInput_0.getArray(3)); /* 173 */ if (project_tmpInput_2 instanceof UnsafeArrayData) { /* 174 */ rdd_mutableStateArray_0[3].write((UnsafeArrayData) project_tmpInput_2); /* 175 */ } else { /* 176 */ final int project_numElements_1 = project_tmpInput_2.numElements(); /* 177 */ rdd_mutableStateArray_1[3].initialize(project_numElements_1); /* 178 */ /* 179 */ 20/08/08 15:33:53.928 Executor task launch worker for task 56 INFO BlockManager: Getting local shuffle block shuffle_3_43_3 for (int project_index_1 = 0; project_index_1 < project_numElements_1; project_index_1++) { /* 180 */ rdd_mutableStateArray_1[3].write(project_index_1, project_tmpInput_2.getDouble(project_index_1)); /* 181 */ } /* 182 */ } /* 183 */ /* 184 */ rdd_mutableStateArray_0[3].setOffsetAndSizeFromPreviousCursor(3, project_previousCursor_2); /* 185 */ } /* 186 */ /* 187 */ } /* 188 */ /* 189 */ private void wholestagecodegen_init_0_1() { /* 190 */ rdd_mutableStateArray_0[4] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(2, 32); /* 191 */ rdd_mutableStateArray_0[5] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(rdd_mutableStateArray_0[4], 4); /* 192 */ rdd_mutableStateArray_1[4] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[5], 4); /* 193 */ rdd_mutableStateArray_1[5] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[5], 8); /* 194 */ rdd_mutableStateArray_0[6] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(2, 0); /* 195 */ /* 196 */ } /* 197 */ /* 198 */ private void agg_doConsume_0(double agg_expr_0_0, boolean agg_exprIsNull_0_0, InternalRow agg_expr_1_0, boolean agg_exprIsNull_1_0) throws java.io.IOException { /* 199 */ // do aggregate /* 200 */ // common sub-expressions /* 201 */ /* 202 */ // evaluate aggregate functions and update aggregation buffers /* 203 */ agg_doAggregate_sum_0(agg_exprIsNull_1_0, agg_expr_1_0, agg_expr_0_0, agg_exprIsNull_0_0); /* 204 */ agg_doAggregate_sum_1(); /* 205 */ /* 206 */ } /* 207 */ /* 208 */ private void agg_doAggregate_sum_1() throws java.io.IOException { /* 209 */ agg_agg_isNull_22_0 = true; /* 210 */ double agg_value_22 = -1.0; /* 211 */ do { /* 212 */ if (!agg_bufIsNull_1) { /* 213 */ agg_agg_isNull_22_0 = false; /* 214 */ agg_value_22 = agg_bufValue_1; /* 215 */ continue; /* 216 */ } /* 217 */ /* 218 */ if (!false) { /* 219 */ agg_agg_isNull_22_0 = false; /* 220 */ agg_value_22 = 0.0D; /* 221 */ continue; /* 222 */ } /* 223 */ /* 224 */ } while (false); /* 225 */ /* 226 */ double agg_value_21 = -1.0; /* 227 */ /* 228 */ agg_value_21 = agg_value_22 + 1.0D; /* 229 */ /* 230 */ agg_bufIsNull_1 = false; /* 231 */ agg_bufValue_1 = agg_value_21; /* 232 */ } /* 233 */ /* 234 */ private void project_writeFields_0_0(InternalRow project_tmpInput_0) { /* 235 */ rdd_mutableStateArray_0[3].write(0, (project_tmpInput_0.getByte(0))); /* 236 */ /* 237 */ if ((project_tmpInput_0.isNullAt(1))) { /* 238 */ rdd_mutableStateArray_0[3].setNullAt(1); /* 239 */ } else { /* 240 */ rdd_mutableStateArray_0[3].write(1, (project_tmpInput_0.getInt(1))); /* 241 */ } /* 242 */ /* 243 */ if ((project_tmpInput_0.isNullAt(2))) { /* 244 */ rdd_mutableStateArray_0[3].setNullAt(2); /* 245 */ } else { /* 246 */ // Remember the current cursor so that we can calculate how many bytes are /* 247 */ // written later. /* 248 */ final int project_previousCursor_1 = rdd_mutableStateArray_0[3].cursor(); /* 249 */ /* 250 */ final ArrayData project_tmpInput_1 = (project_tmpInput_0.getArray(2)); /* 251 */ if (project_tmpInput_1 instanceof UnsafeArrayData) { /* 252 */ rdd_mutableStateArray_0[3].write((UnsafeArrayData) project_tmpInput_1); /* 253 */ } else { /* 254 */ final int project_numElements_0 = project_tmpInput_1.numElements(); /* 255 */ rdd_mutableStateArray_1[2].initialize(project_numElements_0); /* 256 */ /* 257 */ for (int project_index_0 = 0; project_index_0 < project_numElements_0; project_index_0++) { /* 258 */ rdd_mutableStateArray_1[2].write(project_index_0, project_tmpInput_1.getInt(project_index_0)); /* 259 */ } /* 260 */ } /* 261 */ /* 262 */ rdd_mutableStateArray_0[3].setOffsetAndSizeFromPreviousCursor(2, project_previousCursor_1); /* 263 */ } /* 264 */ /* 265 */ } /* 266 */ /* 267 */ protected void processNext() throws java.io.IOException { /* 268 */ while (!agg_initAgg_0) { /* 269 */ agg_initAgg_0 = true; /* 270 */ long agg_beforeAgg_0 = System.nanoTime(); /* 271 */ agg_doAggregateWithoutKey_0(); /* 272 */ ((org.apache.spark.sql.execution.metric.SQLMetric) references[8] /* aggTime */).add((System.nanoTime() - agg_beforeAgg_0) / 1000000); /* 273 */ /* 274 */ // output the result /* 275 */ /* 276 */ ((org.apache.spark.sql.execution.metric.SQLMetric) references[7] /* numOutputRows */).add(1); /* 277 */ rdd_mutableStateArray_0[6].reset(); /* 278 */ /* 279 */ rdd_mutableStateArray_0[6].zeroOutNullBytes(); /* 280 */ /* 281 */ if (agg_bufIsNull_0) { /* 282 */ rdd_mutableStateArray_0[6].setNullAt(0); /* 283 */ } else { /* 284 */ rdd_mutableStateArray_0[6].write(0, agg_bufValue_0); /* 285 */ } /* 286 */ /* 287 */ if (agg_bufIsNull_1) { /* 288 */ rdd_mutableStateArray_0[6].setNullAt(1); /* 289 */ } else { /* 290 */ rdd_mutableStateArray_0[6].write(1, agg_bufValue_1); /* 291 */ } /* 292 */ append((rdd_mutableStateArray_0[6].getRow())); /* 293 */ } /* 294 */ } /* 295 */ /* 296 */ private void wholestagecodegen_init_0_0() { /* 297 */ rdd_input_0 = inputs[0]; /* 298 */ rdd_mutableStateArray_0[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(2, 32); /* 299 */ rdd_mutableStateArray_0[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(rdd_mutableStateArray_0[0], 4); /* 300 */ rdd_mutableStateArray_1[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[1], 4); /* 301 */ rdd_mutableStateArray_1[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[1], 8); /* 302 */ rdd_mutableStateArray_0[2] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(2, 32); /* 303 */ rdd_mutableStateArray_0[3] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(rdd_mutableStateArray_0[2], 4); /* 304 */ rdd_mutableStateArray_1[2] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[3], 4); /* 305 */ rdd_mutableStateArray_1[3] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[3], 8); /* 306 */ /* 307 */ } /* 308 */ /* 309 */ } 20/08/08 15:33:53.928 Executor task launch worker for task 56 INFO BlockManager: Getting local shuffle block shuffle_3_44_3 20/08/08 15:33:53.928 Executor task launch worker for task 56 INFO BlockManager: Getting local shuffle block shuffle_3_45_3 20/08/08 15:33:53.929 Executor task launch worker for task 56 DEBUG ShuffleBlockFetcherIterator: Got local blocks in 9 ms 20/08/08 15:33:53.930 Executor task launch worker for task 56 DEBUG TaskMemoryManager: Task 56 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@ec8fa9b 20/08/08 15:33:53.931 Executor task launch worker for task 56 DEBUG TaskMemoryManager: Task 56 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@6f69b36d 20/08/08 15:33:53.931 Executor task launch worker for task 54 INFO ShuffleBlockFetcherIterator: Getting 3 (707.0 B) non-empty blocks including 3 (707.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 20/08/08 15:33:53.931 Executor task launch worker for task 54 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 7 ms 20/08/08 15:33:53.931 Executor task launch worker for task 54 DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_3_43_1,0), (shuffle_3_44_1,1), (shuffle_3_45_1,2) 20/08/08 15:33:53.925 Executor task launch worker for task 53 DEBUG MapOutputTrackerMaster: Convert map statuses for shuffle 3, mappers 0-10, partitions 0-1 20/08/08 15:33:53.931 Executor task launch worker for task 54 INFO BlockManager: Getting local shuffle block shuffle_3_43_1 20/08/08 15:33:53.931 Executor task launch worker for task 54 INFO BlockManager: Getting local shuffle block shuffle_3_44_1 20/08/08 15:33:53.932 Executor task launch worker for task 53 DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 20/08/08 15:33:53.932 Executor task launch worker for task 53 INFO ShuffleBlockFetcherIterator: Getting 3 (707.0 B) non-empty blocks including 3 (707.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 20/08/08 15:33:53.932 Executor task launch worker for task 53 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 20/08/08 15:33:53.932 Executor task launch worker for task 53 DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_3_43_0,0), (shuffle_3_44_0,1), (shuffle_3_45_0,2) 20/08/08 15:33:53.932 Executor task launch worker for task 53 INFO BlockManager: Getting local shuffle block shuffle_3_43_0 20/08/08 15:33:53.932 Executor task launch worker for task 53 INFO BlockManager: Getting local shuffle block shuffle_3_44_0 20/08/08 15:33:53.932 Executor task launch worker for task 53 INFO BlockManager: Getting local shuffle block shuffle_3_45_0 20/08/08 15:33:53.933 Executor task launch worker for task 53 DEBUG ShuffleBlockFetcherIterator: Got local blocks in 1 ms 20/08/08 15:33:53.933 Executor task launch worker for task 55 DEBUG BlockManagerMaster: Updated info of block rdd_50_2 20/08/08 15:33:53.933 Executor task launch worker for task 55 DEBUG BlockManager: Told master about block rdd_50_2 20/08/08 15:33:53.933 Executor task launch worker for task 55 DEBUG BlockManager: Put block rdd_50_2 locally took 13 ms 20/08/08 15:33:53.933 Executor task launch worker for task 55 DEBUG BlockManager: Putting block rdd_50_2 without replication took 13 ms 20/08/08 15:33:53.933 Executor task launch worker for task 55 DEBUG BlockManager: Getting local block rdd_50_2 20/08/08 15:33:53.933 Executor task launch worker for task 55 DEBUG BlockManager: Level for block rdd_50_2 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:53.934 Executor task launch worker for task 55 INFO Executor: Finished task 2.0 in stage 12.0 (TID 55). 2256 bytes result sent to driver 20/08/08 15:33:53.935 dispatcher-event-loop-1 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_12.0, runningTasks: 3 20/08/08 15:33:53.935 dispatcher-event-loop-1 INFO TaskSetManager: Starting task 4.0 in stage 12.0 (TID 57, 192.168.1.225, executor driver, partition 4, PROCESS_LOCAL, 7433 bytes) taskResourceAssignments Map() 20/08/08 15:33:53.935 Executor task launch worker for task 57 INFO Executor: Running task 4.0 in stage 12.0 (TID 57) 20/08/08 15:33:53.936 Executor task launch worker for task 55 DEBUG ExecutorMetricsPoller: stageTCMP: (12, 0) -> 3 20/08/08 15:33:53.931 Executor task launch worker for task 56 DEBUG TaskMemoryManager: Task 56 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@15ec0e20 20/08/08 15:33:53.936 task-result-getter-2 INFO TaskSetManager: Finished task 2.0 in stage 12.0 (TID 55) in 34 ms on 192.168.1.225 (executor driver) (1/10) 20/08/08 15:33:53.934 Executor task launch worker for task 53 DEBUG TaskMemoryManager: Task 53 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@596033a0 20/08/08 15:33:53.932 Executor task launch worker for task 54 INFO BlockManager: Getting local shuffle block shuffle_3_45_1 20/08/08 15:33:53.936 Executor task launch worker for task 57 DEBUG ExecutorMetricsPoller: stageTCMP: (12, 0) -> 4 20/08/08 15:33:53.936 Executor task launch worker for task 54 DEBUG ShuffleBlockFetcherIterator: Got local blocks in 12 ms 20/08/08 15:33:53.937 Executor task launch worker for task 53 DEBUG TaskMemoryManager: Task 53 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@6b23069f 20/08/08 15:33:53.938 Executor task launch worker for task 56 INFO MemoryStore: Block rdd_50_3 stored as values in memory (estimated size 16.0 B, free 365.9 MiB) 20/08/08 15:33:53.938 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_50_3 for BlockManagerId(driver, 192.168.1.225, 38159, None) 20/08/08 15:33:53.938 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_50_3 in memory on 192.168.1.225:38159 (size: 16.0 B, free: 366.2 MiB) 20/08/08 15:33:53.939 Executor task launch worker for task 53 DEBUG TaskMemoryManager: Task 53 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@2ed4e11e 20/08/08 15:33:53.939 Executor task launch worker for task 53 INFO MemoryStore: Block rdd_50_0 stored as values in memory (estimated size 88.0 B, free 365.9 MiB) 20/08/08 15:33:53.939 Executor task launch worker for task 56 DEBUG BlockManagerMaster: Updated info of block rdd_50_3 20/08/08 15:33:53.939 Executor task launch worker for task 56 DEBUG BlockManager: Told master about block rdd_50_3 20/08/08 15:33:53.939 Executor task launch worker for task 56 DEBUG BlockManager: Put block rdd_50_3 locally took 21 ms 20/08/08 15:33:53.940 Executor task launch worker for task 56 DEBUG BlockManager: Putting block rdd_50_3 without replication took 21 ms 20/08/08 15:33:53.940 Executor task launch worker for task 56 DEBUG BlockManager: Getting local block rdd_50_3 20/08/08 15:33:53.940 Executor task launch worker for task 54 DEBUG TaskMemoryManager: Task 54 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@6a74edcf 20/08/08 15:33:53.940 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_50_0 for BlockManagerId(driver, 192.168.1.225, 38159, None) 20/08/08 15:33:53.940 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_50_0 in memory on 192.168.1.225:38159 (size: 88.0 B, free: 366.2 MiB) 20/08/08 15:33:53.940 Executor task launch worker for task 54 DEBUG TaskMemoryManager: Task 54 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@3fd047d 20/08/08 15:33:53.940 Executor task launch worker for task 53 DEBUG BlockManagerMaster: Updated info of block rdd_50_0 20/08/08 15:33:53.940 Executor task launch worker for task 53 DEBUG BlockManager: Told master about block rdd_50_0 20/08/08 15:33:53.940 Executor task launch worker for task 53 DEBUG BlockManager: Put block rdd_50_0 locally took 20 ms 20/08/08 15:33:53.941 Executor task launch worker for task 53 DEBUG BlockManager: Putting block rdd_50_0 without replication took 20 ms 20/08/08 15:33:53.941 Executor task launch worker for task 53 DEBUG BlockManager: Getting local block rdd_50_0 20/08/08 15:33:53.941 Executor task launch worker for task 53 DEBUG BlockManager: Level for block rdd_50_0 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:53.941 Executor task launch worker for task 54 DEBUG TaskMemoryManager: Task 54 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@f4501d5 20/08/08 15:33:53.941 Executor task launch worker for task 54 INFO MemoryStore: Block rdd_50_1 stored as values in memory (estimated size 88.0 B, free 365.9 MiB) 20/08/08 15:33:53.941 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_50_1 for BlockManagerId(driver, 192.168.1.225, 38159, None) 20/08/08 15:33:53.942 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_50_1 in memory on 192.168.1.225:38159 (size: 88.0 B, free: 366.2 MiB) 20/08/08 15:33:53.942 Executor task launch worker for task 54 DEBUG BlockManagerMaster: Updated info of block rdd_50_1 20/08/08 15:33:53.942 Executor task launch worker for task 54 DEBUG BlockManager: Told master about block rdd_50_1 20/08/08 15:33:53.942 Executor task launch worker for task 54 DEBUG BlockManager: Put block rdd_50_1 locally took 21 ms 20/08/08 15:33:53.942 Executor task launch worker for task 53 INFO Executor: Finished task 0.0 in stage 12.0 (TID 53). 2256 bytes result sent to driver 20/08/08 15:33:53.942 Executor task launch worker for task 54 DEBUG BlockManager: Putting block rdd_50_1 without replication took 21 ms 20/08/08 15:33:53.942 Executor task launch worker for task 54 DEBUG BlockManager: Getting local block rdd_50_1 20/08/08 15:33:53.942 Executor task launch worker for task 54 DEBUG BlockManager: Level for block rdd_50_1 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:53.943 dispatcher-event-loop-0 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_12.0, runningTasks: 3 20/08/08 15:33:53.943 Executor task launch worker for task 12 DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 20/08/08 15:33:53.943 dispatcher-event-loop-0 INFO TaskSetManager: Starting task 5.0 in stage 12.0 (TID 58, 192.168.1.225, executor driver, partition 5, PROCESS_LOCAL, 7433 bytes) taskResourceAssignments Map() 20/08/08 15:33:53.943 Executor task launch worker for task 54 INFO Executor: Finished task 1.0 in stage 12.0 (TID 54). 2256 bytes result sent to driver 20/08/08 15:33:53.943 Executor task launch worker for task 13 DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 20/08/08 15:33:53.943 task-result-getter-0 INFO TaskSetManager: Finished task 0.0 in stage 12.0 (TID 53) in 42 ms on 192.168.1.225 (executor driver) (2/10) 20/08/08 15:33:53.943 Executor task launch worker for task 14 DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 20/08/08 15:33:53.943 Executor task launch worker for task 53 DEBUG ExecutorMetricsPoller: stageTCMP: (12, 0) -> 3 20/08/08 15:33:53.940 Executor task launch worker for task 56 DEBUG BlockManager: Level for block rdd_50_3 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:53.944 Executor task launch worker for task 54 DEBUG ExecutorMetricsPoller: stageTCMP: (12, 0) -> 2 20/08/08 15:33:53.943 Executor task launch worker for task 58 INFO Executor: Running task 5.0 in stage 12.0 (TID 58) 20/08/08 15:33:53.942 Executor task launch worker for task 57 DEBUG BlockManager: Getting local block rdd_50_4 20/08/08 15:33:53.944 Executor task launch worker for task 57 DEBUG BlockManager: Block rdd_50_4 was not found 20/08/08 15:33:53.944 Executor task launch worker for task 57 DEBUG BlockManager: Getting remote block rdd_50_4 20/08/08 15:33:53.944 dispatcher-event-loop-2 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_12.0, runningTasks: 3 20/08/08 15:33:53.945 dispatcher-event-loop-2 INFO TaskSetManager: Starting task 6.0 in stage 12.0 (TID 59, 192.168.1.225, executor driver, partition 6, PROCESS_LOCAL, 7433 bytes) taskResourceAssignments Map() 20/08/08 15:33:53.945 task-result-getter-1 INFO TaskSetManager: Finished task 1.0 in stage 12.0 (TID 54) in 44 ms on 192.168.1.225 (executor driver) (3/10) 20/08/08 15:33:53.945 Executor task launch worker for task 57 DEBUG BlockManager: Block rdd_50_4 is unknown by block manager master 20/08/08 15:33:53.945 Executor task launch worker for task 57 DEBUG BlockManager: Getting local block rdd_19_4 20/08/08 15:33:53.945 Executor task launch worker for task 57 DEBUG BlockManager: Level for block rdd_19_4 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:53.945 Executor task launch worker for task 57 INFO BlockManager: Found block rdd_19_4 locally 20/08/08 15:33:53.945 Executor task launch worker for task 57 DEBUG BlockManager: Getting local block rdd_19_4 20/08/08 15:33:53.945 Executor task launch worker for task 57 DEBUG BlockManager: Level for block rdd_19_4 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:53.945 Executor task launch worker for task 59 INFO Executor: Running task 6.0 in stage 12.0 (TID 59) 20/08/08 15:33:53.946 Executor task launch worker for task 56 INFO Executor: Finished task 3.0 in stage 12.0 (TID 56). 2213 bytes result sent to driver 20/08/08 15:33:53.946 Executor task launch worker for task 59 DEBUG ExecutorMetricsPoller: stageTCMP: (12, 0) -> 4 20/08/08 15:33:53.946 dispatcher-event-loop-0 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_12.0, runningTasks: 3 20/08/08 15:33:53.946 Executor task launch worker for task 58 DEBUG ExecutorMetricsPoller: stageTCMP: (12, 0) -> 3 20/08/08 15:33:53.946 dispatcher-event-loop-0 INFO TaskSetManager: Starting task 7.0 in stage 12.0 (TID 60, 192.168.1.225, executor driver, partition 7, PROCESS_LOCAL, 7433 bytes) taskResourceAssignments Map() 20/08/08 15:33:53.947 task-result-getter-3 INFO TaskSetManager: Finished task 3.0 in stage 12.0 (TID 56) in 44 ms on 192.168.1.225 (executor driver) (4/10) 20/08/08 15:33:53.947 Executor task launch worker for task 60 INFO Executor: Running task 7.0 in stage 12.0 (TID 60) 20/08/08 15:33:53.947 Executor task launch worker for task 56 DEBUG ExecutorMetricsPoller: stageTCMP: (12, 0) -> 3 20/08/08 15:33:53.947 Executor task launch worker for task 60 DEBUG ExecutorMetricsPoller: stageTCMP: (12, 0) -> 4 20/08/08 15:33:53.949 Executor task launch worker for task 15 DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 20/08/08 15:33:53.945 Executor task launch worker for task 57 INFO BlockManager: Found block rdd_19_4 locally 20/08/08 15:33:53.949 Executor task launch worker for task 57 DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 3 20/08/08 15:33:53.949 Executor task launch worker for task 57 DEBUG MapOutputTrackerMaster: Convert map statuses for shuffle 3, mappers 0-10, partitions 4-5 20/08/08 15:33:53.950 Executor task launch worker for task 57 DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 20/08/08 15:33:53.950 Executor task launch worker for task 57 INFO ShuffleBlockFetcherIterator: Getting 3 (684.0 B) non-empty blocks including 3 (684.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 20/08/08 15:33:53.950 Executor task launch worker for task 57 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 20/08/08 15:33:53.950 Executor task launch worker for task 57 DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_3_43_4,0), (shuffle_3_44_4,1), (shuffle_3_45_4,2) 20/08/08 15:33:53.950 Executor task launch worker for task 57 INFO BlockManager: Getting local shuffle block shuffle_3_43_4 20/08/08 15:33:53.950 Executor task launch worker for task 57 INFO BlockManager: Getting local shuffle block shuffle_3_44_4 20/08/08 15:33:53.951 Executor task launch worker for task 57 INFO BlockManager: Getting local shuffle block shuffle_3_45_4 20/08/08 15:33:53.951 Executor task launch worker for task 57 DEBUG ShuffleBlockFetcherIterator: Got local blocks in 1 ms 20/08/08 15:33:53.952 Executor task launch worker for task 57 DEBUG TaskMemoryManager: Task 57 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@6fac35b0 20/08/08 15:33:53.953 Executor task launch worker for task 58 DEBUG BlockManager: Getting local block rdd_50_5 20/08/08 15:33:53.953 Executor task launch worker for task 58 DEBUG BlockManager: Block rdd_50_5 was not found 20/08/08 15:33:53.953 Executor task launch worker for task 58 DEBUG BlockManager: Getting remote block rdd_50_5 20/08/08 15:33:53.953 Executor task launch worker for task 57 DEBUG TaskMemoryManager: Task 57 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@4da30203 20/08/08 15:33:53.953 Executor task launch worker for task 57 DEBUG TaskMemoryManager: Task 57 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@6a5115d6 20/08/08 15:33:53.954 Executor task launch worker for task 57 INFO MemoryStore: Block rdd_50_4 stored as values in memory (estimated size 16.0 B, free 365.9 MiB) 20/08/08 15:33:53.954 Executor task launch worker for task 12 INFO ShuffleBlockFetcherIterator: Getting 3 (567.0 B) non-empty blocks including 3 (567.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 20/08/08 15:33:53.954 Executor task launch worker for task 60 DEBUG BlockManager: Getting local block rdd_50_7 20/08/08 15:33:53.954 Executor task launch worker for task 60 DEBUG BlockManager: Block rdd_50_7 was not found 20/08/08 15:33:53.954 Executor task launch worker for task 60 DEBUG BlockManager: Getting remote block rdd_50_7 20/08/08 15:33:53.954 Executor task launch worker for task 15 INFO ShuffleBlockFetcherIterator: Getting 0 (0.0 B) non-empty blocks including 0 (0.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 20/08/08 15:33:53.955 Executor task launch worker for task 14 INFO ShuffleBlockFetcherIterator: Getting 2 (378.0 B) non-empty blocks including 2 (378.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 20/08/08 15:33:53.955 Executor task launch worker for task 13 INFO ShuffleBlockFetcherIterator: Getting 2 (378.0 B) non-empty blocks including 2 (378.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 20/08/08 15:33:53.956 Executor task launch worker for task 14 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 14 ms 20/08/08 15:33:53.957 Executor task launch worker for task 14 DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_0_8_3,0), (shuffle_0_11_3,3) 20/08/08 15:33:53.957 Executor task launch worker for task 13 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 15 ms 20/08/08 15:33:53.957 Executor task launch worker for task 13 DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_0_8_2,0), (shuffle_0_10_2,2) 20/08/08 15:33:53.958 Executor task launch worker for task 14 INFO BlockManager: Getting local shuffle block shuffle_0_8_3 20/08/08 15:33:53.958 Executor task launch worker for task 13 INFO BlockManager: Getting local shuffle block shuffle_0_8_2 20/08/08 15:33:53.958 Executor task launch worker for task 15 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 9 ms 20/08/08 15:33:53.958 Executor task launch worker for task 15 DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: 20/08/08 15:33:53.958 Executor task launch worker for task 15 DEBUG ShuffleBlockFetcherIterator: Got local blocks in 9 ms 20/08/08 15:33:53.959 Executor task launch worker for task 59 DEBUG BlockManager: Getting local block rdd_50_6 20/08/08 15:33:53.959 Executor task launch worker for task 59 DEBUG BlockManager: Block rdd_50_6 was not found 20/08/08 15:33:53.959 Executor task launch worker for task 59 DEBUG BlockManager: Getting remote block rdd_50_6 20/08/08 15:33:53.959 Executor task launch worker for task 12 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 17 ms 20/08/08 15:33:53.959 Executor task launch worker for task 12 DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_0_8_1,0), (shuffle_0_9_1,1), (shuffle_0_10_1,2) 20/08/08 15:33:53.959 Executor task launch worker for task 12 INFO BlockManager: Getting local shuffle block shuffle_0_8_1 20/08/08 15:33:53.961 Executor task launch worker for task 58 DEBUG BlockManager: Block rdd_50_5 is unknown by block manager master 20/08/08 15:33:53.961 Executor task launch worker for task 12 INFO BlockManager: Getting local shuffle block shuffle_0_9_1 20/08/08 15:33:53.961 Executor task launch worker for task 58 DEBUG BlockManager: Getting local block rdd_19_5 20/08/08 15:33:53.961 Executor task launch worker for task 58 DEBUG BlockManager: Level for block rdd_19_5 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:53.961 Executor task launch worker for task 13 INFO BlockManager: Getting local shuffle block shuffle_0_10_2 20/08/08 15:33:53.961 Executor task launch worker for task 58 INFO BlockManager: Found block rdd_19_5 locally 20/08/08 15:33:53.961 Executor task launch worker for task 12 INFO BlockManager: Getting local shuffle block shuffle_0_10_1 20/08/08 15:33:53.962 Executor task launch worker for task 58 DEBUG BlockManager: Getting local block rdd_19_5 20/08/08 15:33:53.962 Executor task launch worker for task 13 DEBUG ShuffleBlockFetcherIterator: Got local blocks in 19 ms 20/08/08 15:33:53.962 Executor task launch worker for task 58 DEBUG BlockManager: Level for block rdd_19_5 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:53.962 Executor task launch worker for task 58 INFO BlockManager: Found block rdd_19_5 locally 20/08/08 15:33:53.962 Executor task launch worker for task 12 DEBUG ShuffleBlockFetcherIterator: Got local blocks in 20 ms 20/08/08 15:33:53.962 Executor task launch worker for task 58 DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 3 20/08/08 15:33:53.962 Executor task launch worker for task 58 DEBUG MapOutputTrackerMaster: Convert map statuses for shuffle 3, mappers 0-10, partitions 5-6 20/08/08 15:33:53.962 Executor task launch worker for task 58 DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 20/08/08 15:33:53.962 Executor task launch worker for task 14 INFO BlockManager: Getting local shuffle block shuffle_0_11_3 20/08/08 15:33:53.963 Executor task launch worker for task 58 INFO ShuffleBlockFetcherIterator: Getting 3 (684.0 B) non-empty blocks including 3 (684.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 20/08/08 15:33:53.963 Executor task launch worker for task 58 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 20/08/08 15:33:53.963 Executor task launch worker for task 14 DEBUG ShuffleBlockFetcherIterator: Got local blocks in 20 ms 20/08/08 15:33:53.963 Executor task launch worker for task 58 DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_3_43_5,0), (shuffle_3_44_5,1), (shuffle_3_45_5,2) 20/08/08 15:33:53.963 Executor task launch worker for task 58 INFO BlockManager: Getting local shuffle block shuffle_3_43_5 20/08/08 15:33:53.963 Executor task launch worker for task 58 INFO BlockManager: Getting local shuffle block shuffle_3_44_5 20/08/08 15:33:53.963 Executor task launch worker for task 58 INFO BlockManager: Getting local shuffle block shuffle_3_45_5 20/08/08 15:33:53.963 Executor task launch worker for task 58 DEBUG ShuffleBlockFetcherIterator: Got local blocks in 1 ms 20/08/08 15:33:53.964 Executor task launch worker for task 59 DEBUG BlockManager: Block rdd_50_6 is unknown by block manager master 20/08/08 15:33:53.964 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_50_4 for BlockManagerId(driver, 192.168.1.225, 38159, None) 20/08/08 15:33:53.964 Executor task launch worker for task 59 DEBUG BlockManager: Getting local block rdd_19_6 20/08/08 15:33:53.964 Executor task launch worker for task 59 DEBUG BlockManager: Level for block rdd_19_6 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:53.964 Executor task launch worker for task 59 INFO BlockManager: Found block rdd_19_6 locally 20/08/08 15:33:53.964 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_50_4 in memory on 192.168.1.225:38159 (size: 16.0 B, free: 366.2 MiB) 20/08/08 15:33:53.965 Executor task launch worker for task 57 DEBUG BlockManagerMaster: Updated info of block rdd_50_4 20/08/08 15:33:53.965 Executor task launch worker for task 57 DEBUG BlockManager: Told master about block rdd_50_4 20/08/08 15:33:53.965 Executor task launch worker for task 57 DEBUG BlockManager: Put block rdd_50_4 locally took 19 ms 20/08/08 15:33:53.965 Executor task launch worker for task 57 DEBUG BlockManager: Putting block rdd_50_4 without replication took 19 ms 20/08/08 15:33:53.965 Executor task launch worker for task 57 DEBUG BlockManager: Getting local block rdd_50_4 20/08/08 15:33:53.965 Executor task launch worker for task 57 DEBUG BlockManager: Level for block rdd_50_4 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:53.965 Executor task launch worker for task 60 DEBUG BlockManager: Block rdd_50_7 is unknown by block manager master 20/08/08 15:33:53.965 Executor task launch worker for task 60 DEBUG BlockManager: Getting local block rdd_19_7 20/08/08 15:33:53.965 Executor task launch worker for task 60 DEBUG BlockManager: Level for block rdd_19_7 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:53.965 Executor task launch worker for task 60 INFO BlockManager: Found block rdd_19_7 locally 20/08/08 15:33:53.966 Executor task launch worker for task 60 DEBUG BlockManager: Getting local block rdd_19_7 20/08/08 15:33:53.966 Executor task launch worker for task 60 DEBUG BlockManager: Level for block rdd_19_7 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:53.966 Executor task launch worker for task 60 INFO BlockManager: Found block rdd_19_7 locally 20/08/08 15:33:53.966 Executor task launch worker for task 60 DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 3 20/08/08 15:33:53.966 Executor task launch worker for task 60 DEBUG MapOutputTrackerMaster: Convert map statuses for shuffle 3, mappers 0-10, partitions 7-8 20/08/08 15:33:53.964 Executor task launch worker for task 59 DEBUG BlockManager: Getting local block rdd_19_6 20/08/08 15:33:53.966 Executor task launch worker for task 60 DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 20/08/08 15:33:53.966 Executor task launch worker for task 59 DEBUG BlockManager: Level for block rdd_19_6 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:53.966 Executor task launch worker for task 57 INFO Executor: Finished task 4.0 in stage 12.0 (TID 57). 2213 bytes result sent to driver 20/08/08 15:33:53.966 Executor task launch worker for task 59 INFO BlockManager: Found block rdd_19_6 locally 20/08/08 15:33:53.966 Executor task launch worker for task 59 DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 3 20/08/08 15:33:53.966 Executor task launch worker for task 57 DEBUG ExecutorMetricsPoller: stageTCMP: (12, 0) -> 3 20/08/08 15:33:53.967 dispatcher-event-loop-2 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_12.0, runningTasks: 3 20/08/08 15:33:53.966 Executor task launch worker for task 59 DEBUG MapOutputTrackerMaster: Convert map statuses for shuffle 3, mappers 0-10, partitions 6-7 20/08/08 15:33:53.967 Executor task launch worker for task 58 DEBUG TaskMemoryManager: Task 58 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@3802ef98 20/08/08 15:33:53.967 dispatcher-event-loop-2 INFO TaskSetManager: Starting task 8.0 in stage 12.0 (TID 61, 192.168.1.225, executor driver, partition 8, PROCESS_LOCAL, 7433 bytes) taskResourceAssignments Map() 20/08/08 15:33:53.967 Executor task launch worker for task 59 DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 20/08/08 15:33:53.968 Executor task launch worker for task 61 INFO Executor: Running task 8.0 in stage 12.0 (TID 61) 20/08/08 15:33:53.968 Executor task launch worker for task 59 INFO ShuffleBlockFetcherIterator: Getting 3 (684.0 B) non-empty blocks including 3 (684.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 20/08/08 15:33:53.968 Executor task launch worker for task 59 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 20/08/08 15:33:53.968 Executor task launch worker for task 59 DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_3_43_6,0), (shuffle_3_44_6,1), (shuffle_3_45_6,2) 20/08/08 15:33:53.968 Executor task launch worker for task 59 INFO BlockManager: Getting local shuffle block shuffle_3_43_6 20/08/08 15:33:53.968 task-result-getter-2 INFO TaskSetManager: Finished task 4.0 in stage 12.0 (TID 57) in 33 ms on 192.168.1.225 (executor driver) (5/10) 20/08/08 15:33:53.968 Executor task launch worker for task 61 DEBUG ExecutorMetricsPoller: stageTCMP: (12, 0) -> 4 20/08/08 15:33:53.969 Executor task launch worker for task 60 INFO ShuffleBlockFetcherIterator: Getting 3 (684.0 B) non-empty blocks including 3 (684.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 20/08/08 15:33:53.969 Executor task launch worker for task 60 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 3 ms 20/08/08 15:33:53.969 Executor task launch worker for task 60 DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_3_43_7,0), (shuffle_3_44_7,1), (shuffle_3_45_7,2) 20/08/08 15:33:53.969 Executor task launch worker for task 60 INFO BlockManager: Getting local shuffle block shuffle_3_43_7 20/08/08 15:33:53.969 Executor task launch worker for task 60 INFO BlockManager: Getting local shuffle block shuffle_3_44_7 20/08/08 15:33:53.970 Executor task launch worker for task 60 INFO BlockManager: Getting local shuffle block shuffle_3_45_7 20/08/08 15:33:53.970 Executor task launch worker for task 60 DEBUG ShuffleBlockFetcherIterator: Got local blocks in 3 ms 20/08/08 15:33:53.968 Executor task launch worker for task 59 INFO BlockManager: Getting local shuffle block shuffle_3_44_6 20/08/08 15:33:53.970 Executor task launch worker for task 59 INFO BlockManager: Getting local shuffle block shuffle_3_45_6 20/08/08 15:33:53.971 Executor task launch worker for task 59 DEBUG ShuffleBlockFetcherIterator: Got local blocks in 3 ms 20/08/08 15:33:53.972 Executor task launch worker for task 59 DEBUG TaskMemoryManager: Task 59 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@72948cce 20/08/08 15:33:53.973 Executor task launch worker for task 59 DEBUG TaskMemoryManager: Task 59 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@59218d99 20/08/08 15:33:53.973 Executor task launch worker for task 59 DEBUG TaskMemoryManager: Task 59 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@3d1e0f23 20/08/08 15:33:53.973 Executor task launch worker for task 59 INFO MemoryStore: Block rdd_50_6 stored as values in memory (estimated size 16.0 B, free 365.9 MiB) 20/08/08 15:33:53.975 Executor task launch worker for task 61 DEBUG BlockManager: Getting local block rdd_50_8 20/08/08 15:33:53.975 Executor task launch worker for task 61 DEBUG BlockManager: Block rdd_50_8 was not found 20/08/08 15:33:53.975 Executor task launch worker for task 61 DEBUG BlockManager: Getting remote block rdd_50_8 20/08/08 15:33:53.975 Executor task launch worker for task 58 DEBUG TaskMemoryManager: Task 58 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@18789fb5 20/08/08 15:33:53.975 Executor task launch worker for task 58 DEBUG TaskMemoryManager: Task 58 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@2959975e 20/08/08 15:33:53.976 Executor task launch worker for task 58 INFO MemoryStore: Block rdd_50_5 stored as values in memory (estimated size 16.0 B, free 365.9 MiB) 20/08/08 15:33:53.981 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_50_6 for BlockManagerId(driver, 192.168.1.225, 38159, None) 20/08/08 15:33:53.981 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_50_6 in memory on 192.168.1.225:38159 (size: 16.0 B, free: 366.2 MiB) 20/08/08 15:33:53.983 Executor task launch worker for task 60 DEBUG TaskMemoryManager: Task 60 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@7eb5a441 20/08/08 15:33:53.983 Executor task launch worker for task 60 DEBUG TaskMemoryManager: Task 60 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@388b4399 20/08/08 15:33:53.984 Executor task launch worker for task 60 DEBUG TaskMemoryManager: Task 60 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@75c3f69a 20/08/08 15:33:53.984 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_50_5 for BlockManagerId(driver, 192.168.1.225, 38159, None) 20/08/08 15:33:53.984 Executor task launch worker for task 60 INFO MemoryStore: Block rdd_50_7 stored as values in memory (estimated size 16.0 B, free 365.9 MiB) 20/08/08 15:33:53.984 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_50_5 in memory on 192.168.1.225:38159 (size: 16.0 B, free: 366.2 MiB) 20/08/08 15:33:53.984 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_50_7 for BlockManagerId(driver, 192.168.1.225, 38159, None) 20/08/08 15:33:53.985 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_50_7 in memory on 192.168.1.225:38159 (size: 16.0 B, free: 366.2 MiB) 20/08/08 15:33:53.985 Executor task launch worker for task 60 DEBUG BlockManagerMaster: Updated info of block rdd_50_7 20/08/08 15:33:53.985 Executor task launch worker for task 60 DEBUG BlockManager: Told master about block rdd_50_7 20/08/08 15:33:53.985 Executor task launch worker for task 60 DEBUG BlockManager: Put block rdd_50_7 locally took 19 ms 20/08/08 15:33:53.985 Executor task launch worker for task 60 DEBUG BlockManager: Putting block rdd_50_7 without replication took 19 ms 20/08/08 15:33:53.985 Executor task launch worker for task 60 DEBUG BlockManager: Getting local block rdd_50_7 20/08/08 15:33:53.985 Executor task launch worker for task 60 DEBUG BlockManager: Level for block rdd_50_7 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:53.986 Executor task launch worker for task 60 INFO Executor: Finished task 7.0 in stage 12.0 (TID 60). 2213 bytes result sent to driver 20/08/08 15:33:53.986 Executor task launch worker for task 60 DEBUG ExecutorMetricsPoller: stageTCMP: (12, 0) -> 3 20/08/08 15:33:53.986 Executor task launch worker for task 14 DEBUG TaskMemoryManager: Task 14 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@563cc510 20/08/08 15:33:53.987 Executor task launch worker for task 14 INFO MemoryStore: Block rdd_21_3 stored as values in memory (estimated size 120.0 B, free 366.2 MiB) 20/08/08 15:33:53.984 Executor task launch worker for task 59 DEBUG BlockManagerMaster: Updated info of block rdd_50_6 20/08/08 15:33:53.987 dispatcher-event-loop-0 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_12.0, runningTasks: 3 20/08/08 15:33:53.985 Executor task launch worker for task 61 DEBUG BlockManager: Block rdd_50_8 is unknown by block manager master 20/08/08 15:33:53.987 Executor task launch worker for task 61 DEBUG BlockManager: Getting local block rdd_19_8 20/08/08 15:33:53.987 Executor task launch worker for task 61 DEBUG BlockManager: Level for block rdd_19_8 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:53.987 dispatcher-event-loop-0 INFO TaskSetManager: Starting task 9.0 in stage 12.0 (TID 62, 192.168.1.225, executor driver, partition 9, PROCESS_LOCAL, 7433 bytes) taskResourceAssignments Map() 20/08/08 15:33:53.987 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_21_3 for BlockManagerId(driver, 192.168.1.225, 36433, None) 20/08/08 15:33:53.985 Executor task launch worker for task 58 DEBUG BlockManagerMaster: Updated info of block rdd_50_5 20/08/08 15:33:53.987 Executor task launch worker for task 58 DEBUG BlockManager: Told master about block rdd_50_5 20/08/08 15:33:53.987 Executor task launch worker for task 58 DEBUG BlockManager: Put block rdd_50_5 locally took 26 ms 20/08/08 15:33:53.987 Executor task launch worker for task 58 DEBUG BlockManager: Putting block rdd_50_5 without replication took 26 ms 20/08/08 15:33:53.987 Executor task launch worker for task 58 DEBUG BlockManager: Getting local block rdd_50_5 20/08/08 15:33:53.988 Executor task launch worker for task 58 DEBUG BlockManager: Level for block rdd_50_5 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:53.988 task-result-getter-0 INFO TaskSetManager: Finished task 7.0 in stage 12.0 (TID 60) in 42 ms on 192.168.1.225 (executor driver) (6/10) 20/08/08 15:33:53.988 Executor task launch worker for task 62 INFO Executor: Running task 9.0 in stage 12.0 (TID 62) 20/08/08 15:33:53.988 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_21_3 in memory on 192.168.1.225:36433 (size: 120.0 B, free: 366.3 MiB) 20/08/08 15:33:53.988 Executor task launch worker for task 62 DEBUG ExecutorMetricsPoller: stageTCMP: (12, 0) -> 4 20/08/08 15:33:53.988 Executor task launch worker for task 58 INFO Executor: Finished task 5.0 in stage 12.0 (TID 58). 2213 bytes result sent to driver 20/08/08 15:33:53.988 Executor task launch worker for task 58 DEBUG ExecutorMetricsPoller: stageTCMP: (12, 0) -> 3 20/08/08 15:33:53.989 Executor task launch worker for task 12 DEBUG TaskMemoryManager: Task 12 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@15fff408 20/08/08 15:33:53.987 Executor task launch worker for task 61 INFO BlockManager: Found block rdd_19_8 locally 20/08/08 15:33:53.990 Executor task launch worker for task 61 DEBUG BlockManager: Getting local block rdd_19_8 20/08/08 15:33:53.990 Executor task launch worker for task 12 INFO MemoryStore: Block rdd_21_1 stored as values in memory (estimated size 120.0 B, free 366.2 MiB) 20/08/08 15:33:53.990 Executor task launch worker for task 61 DEBUG BlockManager: Level for block rdd_19_8 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:53.990 Executor task launch worker for task 61 INFO BlockManager: Found block rdd_19_8 locally 20/08/08 15:33:53.990 Executor task launch worker for task 61 DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 3 20/08/08 15:33:53.990 Executor task launch worker for task 61 DEBUG MapOutputTrackerMaster: Convert map statuses for shuffle 3, mappers 0-10, partitions 8-9 20/08/08 15:33:53.990 Executor task launch worker for task 14 DEBUG BlockManagerMaster: Updated info of block rdd_21_3 20/08/08 15:33:53.990 Executor task launch worker for task 14 DEBUG BlockManager: Told master about block rdd_21_3 20/08/08 15:33:53.990 Executor task launch worker for task 14 DEBUG BlockManager: Put block rdd_21_3 locally took 79 ms 20/08/08 15:33:53.990 Executor task launch worker for task 61 DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 20/08/08 15:33:53.991 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_21_1 for BlockManagerId(driver, 192.168.1.225, 36433, None) 20/08/08 15:33:53.991 Executor task launch worker for task 61 INFO ShuffleBlockFetcherIterator: Getting 3 (684.0 B) non-empty blocks including 3 (684.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 20/08/08 15:33:53.991 Executor task launch worker for task 61 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 20/08/08 15:33:53.991 Executor task launch worker for task 61 DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_3_43_8,0), (shuffle_3_44_8,1), (shuffle_3_45_8,2) 20/08/08 15:33:53.991 Executor task launch worker for task 61 INFO BlockManager: Getting local shuffle block shuffle_3_43_8 20/08/08 15:33:53.991 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_21_1 in memory on 192.168.1.225:36433 (size: 120.0 B, free: 366.3 MiB) 20/08/08 15:33:53.991 Executor task launch worker for task 61 INFO BlockManager: Getting local shuffle block shuffle_3_44_8 20/08/08 15:33:53.991 Executor task launch worker for task 61 INFO BlockManager: Getting local shuffle block shuffle_3_45_8 20/08/08 15:33:53.991 Executor task launch worker for task 61 DEBUG ShuffleBlockFetcherIterator: Got local blocks in 1 ms 20/08/08 15:33:53.992 Executor task launch worker for task 14 DEBUG BlockManager: Putting block rdd_21_3 without replication took 81 ms 20/08/08 15:33:53.992 Executor task launch worker for task 14 DEBUG BlockManager: Getting local block rdd_21_3 20/08/08 15:33:53.993 Executor task launch worker for task 61 DEBUG TaskMemoryManager: Task 61 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@1e4ef066 20/08/08 15:33:53.993 dispatcher-event-loop-1 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_12.0, runningTasks: 3 20/08/08 15:33:53.993 dispatcher-event-loop-1 DEBUG TaskSetManager: No tasks for locality level PROCESS_LOCAL, so moving to locality level NODE_LOCAL 20/08/08 15:33:53.993 Executor task launch worker for task 61 DEBUG TaskMemoryManager: Task 61 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@3b05e3cf 20/08/08 15:33:53.994 Executor task launch worker for task 61 DEBUG TaskMemoryManager: Task 61 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@69c69654 20/08/08 15:33:53.994 Executor task launch worker for task 14 DEBUG BlockManager: Level for block rdd_21_3 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:53.994 Executor task launch worker for task 61 INFO MemoryStore: Block rdd_50_8 stored as values in memory (estimated size 16.0 B, free 365.9 MiB) 20/08/08 15:33:53.994 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_50_8 for BlockManagerId(driver, 192.168.1.225, 38159, None) 20/08/08 15:33:53.995 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_50_8 in memory on 192.168.1.225:38159 (size: 16.0 B, free: 366.2 MiB) 20/08/08 15:33:53.995 Executor task launch worker for task 61 DEBUG BlockManagerMaster: Updated info of block rdd_50_8 20/08/08 15:33:53.995 Executor task launch worker for task 61 DEBUG BlockManager: Told master about block rdd_50_8 20/08/08 15:33:53.995 Executor task launch worker for task 61 DEBUG BlockManager: Put block rdd_50_8 locally took 7 ms 20/08/08 15:33:53.995 Executor task launch worker for task 61 DEBUG BlockManager: Putting block rdd_50_8 without replication took 8 ms 20/08/08 15:33:53.995 Executor task launch worker for task 61 DEBUG BlockManager: Getting local block rdd_50_8 20/08/08 15:33:53.995 Executor task launch worker for task 61 DEBUG BlockManager: Level for block rdd_50_8 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:53.995 Executor task launch worker for task 12 DEBUG BlockManagerMaster: Updated info of block rdd_21_1 20/08/08 15:33:53.996 Executor task launch worker for task 12 DEBUG BlockManager: Told master about block rdd_21_1 20/08/08 15:33:53.996 Executor task launch worker for task 12 DEBUG BlockManager: Put block rdd_21_1 locally took 85 ms 20/08/08 15:33:53.996 Executor task launch worker for task 62 DEBUG BlockManager: Getting local block rdd_50_9 20/08/08 15:33:53.996 Executor task launch worker for task 62 DEBUG BlockManager: Block rdd_50_9 was not found 20/08/08 15:33:53.996 Executor task launch worker for task 62 DEBUG BlockManager: Getting remote block rdd_50_9 20/08/08 15:33:53.996 Executor task launch worker for task 12 DEBUG BlockManager: Putting block rdd_21_1 without replication took 85 ms 20/08/08 15:33:53.996 Executor task launch worker for task 12 DEBUG BlockManager: Getting local block rdd_21_1 20/08/08 15:33:53.996 Executor task launch worker for task 61 INFO Executor: Finished task 8.0 in stage 12.0 (TID 61). 2213 bytes result sent to driver 20/08/08 15:33:53.996 Executor task launch worker for task 12 DEBUG BlockManager: Level for block rdd_21_1 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:53.996 Executor task launch worker for task 61 DEBUG ExecutorMetricsPoller: stageTCMP: (12, 0) -> 2 20/08/08 15:33:53.996 Executor task launch worker for task 62 DEBUG BlockManager: Block rdd_50_9 is unknown by block manager master 20/08/08 15:33:53.987 Executor task launch worker for task 59 DEBUG BlockManager: Told master about block rdd_50_6 20/08/08 15:33:53.993 dispatcher-event-loop-1 DEBUG TaskSetManager: No tasks for locality level NODE_LOCAL, so moving to locality level ANY 20/08/08 15:33:53.996 Executor task launch worker for task 59 DEBUG BlockManager: Put block rdd_50_6 locally took 32 ms 20/08/08 15:33:53.997 task-result-getter-1 INFO TaskSetManager: Finished task 5.0 in stage 12.0 (TID 58) in 54 ms on 192.168.1.225 (executor driver) (7/10) 20/08/08 15:33:53.997 Executor task launch worker for task 12 INFO Executor: Finished task 1.0 in stage 3.0 (TID 12). 1305 bytes result sent to driver 20/08/08 15:33:53.997 dispatcher-event-loop-1 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_12.0, runningTasks: 2 20/08/08 15:33:53.997 Executor task launch worker for task 62 DEBUG BlockManager: Getting local block rdd_19_9 20/08/08 15:33:53.997 Executor task launch worker for task 62 DEBUG BlockManager: Level for block rdd_19_9 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:53.997 Executor task launch worker for task 62 INFO BlockManager: Found block rdd_19_9 locally 20/08/08 15:33:53.997 dispatcher-event-loop-3 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_3.0, runningTasks: 3 20/08/08 15:33:53.998 Executor task launch worker for task 59 DEBUG BlockManager: Putting block rdd_50_6 without replication took 33 ms 20/08/08 15:33:53.998 dispatcher-event-loop-3 DEBUG TaskSetManager: No tasks for locality level NODE_LOCAL, so moving to locality level NO_PREF 20/08/08 15:33:53.998 Executor task launch worker for task 59 DEBUG BlockManager: Getting local block rdd_50_6 20/08/08 15:33:53.998 dispatcher-event-loop-3 DEBUG TaskSetManager: No tasks for locality level NO_PREF, so moving to locality level ANY 20/08/08 15:33:53.997 task-result-getter-3 INFO TaskSetManager: Finished task 8.0 in stage 12.0 (TID 61) in 30 ms on 192.168.1.225 (executor driver) (8/10) 20/08/08 15:33:53.998 Executor task launch worker for task 12 DEBUG ExecutorMetricsPoller: stageTCMP: (3, 0) -> 3 20/08/08 15:33:53.998 task-result-getter-0 INFO TaskSetManager: Finished task 1.0 in stage 3.0 (TID 12) in 104 ms on 192.168.1.225 (executor driver) (1/4) 20/08/08 15:33:53.998 Executor task launch worker for task 59 DEBUG BlockManager: Level for block rdd_50_6 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:53.998 Executor task launch worker for task 62 DEBUG BlockManager: Getting local block rdd_19_9 20/08/08 15:33:53.999 Executor task launch worker for task 62 DEBUG BlockManager: Level for block rdd_19_9 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:53.999 Executor task launch worker for task 62 INFO BlockManager: Found block rdd_19_9 locally 20/08/08 15:33:53.999 Executor task launch worker for task 62 DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 3 20/08/08 15:33:53.999 Executor task launch worker for task 62 DEBUG MapOutputTrackerMaster: Convert map statuses for shuffle 3, mappers 0-10, partitions 9-10 20/08/08 15:33:53.996 Executor task launch worker for task 15 DEBUG TaskMemoryManager: Task 15 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@520572d4 20/08/08 15:33:53.999 Executor task launch worker for task 59 INFO Executor: Finished task 6.0 in stage 12.0 (TID 59). 2213 bytes result sent to driver 20/08/08 15:33:53.999 Executor task launch worker for task 59 DEBUG ExecutorMetricsPoller: stageTCMP: (12, 0) -> 1 20/08/08 15:33:54.000 Executor task launch worker for task 15 INFO MemoryStore: Block rdd_21_0 stored as values in memory (estimated size 16.0 B, free 366.2 MiB) 20/08/08 15:33:54.000 dispatcher-event-loop-0 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_12.0, runningTasks: 1 20/08/08 15:33:54.000 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_21_0 for BlockManagerId(driver, 192.168.1.225, 36433, None) 20/08/08 15:33:54.000 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_21_0 in memory on 192.168.1.225:36433 (size: 16.0 B, free: 366.3 MiB) 20/08/08 15:33:54.000 task-result-getter-2 INFO TaskSetManager: Finished task 6.0 in stage 12.0 (TID 59) in 56 ms on 192.168.1.225 (executor driver) (9/10) 20/08/08 15:33:54.001 Executor task launch worker for task 62 DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 20/08/08 15:33:54.001 Executor task launch worker for task 15 DEBUG BlockManagerMaster: Updated info of block rdd_21_0 20/08/08 15:33:54.001 Executor task launch worker for task 15 DEBUG BlockManager: Told master about block rdd_21_0 20/08/08 15:33:54.001 Executor task launch worker for task 15 DEBUG BlockManager: Put block rdd_21_0 locally took 90 ms 20/08/08 15:33:54.001 Executor task launch worker for task 15 DEBUG BlockManager: Putting block rdd_21_0 without replication took 90 ms 20/08/08 15:33:54.001 Executor task launch worker for task 62 INFO ShuffleBlockFetcherIterator: Getting 3 (684.0 B) non-empty blocks including 3 (684.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 20/08/08 15:33:54.001 Executor task launch worker for task 15 DEBUG BlockManager: Getting local block rdd_21_0 20/08/08 15:33:54.001 Executor task launch worker for task 62 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 1 ms 20/08/08 15:33:54.001 Executor task launch worker for task 15 DEBUG BlockManager: Level for block rdd_21_0 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:54.002 Executor task launch worker for task 62 DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_3_43_9,0), (shuffle_3_44_9,1), (shuffle_3_45_9,2) 20/08/08 15:33:54.002 Executor task launch worker for task 62 INFO BlockManager: Getting local shuffle block shuffle_3_43_9 20/08/08 15:33:54.002 Executor task launch worker for task 62 INFO BlockManager: Getting local shuffle block shuffle_3_44_9 20/08/08 15:33:54.002 Executor task launch worker for task 62 INFO BlockManager: Getting local shuffle block shuffle_3_45_9 20/08/08 15:33:54.002 Executor task launch worker for task 62 DEBUG ShuffleBlockFetcherIterator: Got local blocks in 2 ms 20/08/08 15:33:54.002 Executor task launch worker for task 15 INFO Executor: Finished task 0.0 in stage 3.0 (TID 15). 1305 bytes result sent to driver 20/08/08 15:33:54.003 dispatcher-event-loop-0 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_3.0, runningTasks: 2 20/08/08 15:33:54.003 task-result-getter-1 INFO TaskSetManager: Finished task 0.0 in stage 3.0 (TID 15) in 105 ms on 192.168.1.225 (executor driver) (2/4) 20/08/08 15:33:54.004 Executor task launch worker for task 62 DEBUG TaskMemoryManager: Task 62 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@4320ee70 20/08/08 15:33:54.004 Executor task launch worker for task 62 DEBUG TaskMemoryManager: Task 62 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@2efd13cc 20/08/08 15:33:54.004 Executor task launch worker for task 15 DEBUG ExecutorMetricsPoller: stageTCMP: (3, 0) -> 2 20/08/08 15:33:54.004 Executor task launch worker for task 62 DEBUG TaskMemoryManager: Task 62 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@4894b8e2 20/08/08 15:33:54.005 Executor task launch worker for task 62 INFO MemoryStore: Block rdd_50_9 stored as values in memory (estimated size 16.0 B, free 365.9 MiB) 20/08/08 15:33:54.007 Executor task launch worker for task 14 INFO Executor: Finished task 3.0 in stage 3.0 (TID 14). 1305 bytes result sent to driver 20/08/08 15:33:54.007 Executor task launch worker for task 14 DEBUG ExecutorMetricsPoller: stageTCMP: (3, 0) -> 1 20/08/08 15:33:54.007 dispatcher-event-loop-2 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_3.0, runningTasks: 1 20/08/08 15:33:54.008 task-result-getter-2 INFO TaskSetManager: Finished task 3.0 in stage 3.0 (TID 14) in 111 ms on 192.168.1.225 (executor driver) (3/4) 20/08/08 15:33:54.009 Executor task launch worker for task 13 DEBUG TaskMemoryManager: Task 13 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@5a2722cf 20/08/08 15:33:54.010 Executor task launch worker for task 13 INFO MemoryStore: Block rdd_21_2 stored as values in memory (estimated size 120.0 B, free 366.2 MiB) 20/08/08 15:33:54.010 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_21_2 for BlockManagerId(driver, 192.168.1.225, 36433, None) 20/08/08 15:33:54.011 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_21_2 in memory on 192.168.1.225:36433 (size: 120.0 B, free: 366.3 MiB) 20/08/08 15:33:54.011 Thread-4 INFO CodeGenerator: Code generated in 87.839827 ms 20/08/08 15:33:54.011 Executor task launch worker for task 13 DEBUG BlockManagerMaster: Updated info of block rdd_21_2 20/08/08 15:33:54.011 Executor task launch worker for task 13 DEBUG BlockManager: Told master about block rdd_21_2 20/08/08 15:33:54.011 Executor task launch worker for task 13 DEBUG BlockManager: Put block rdd_21_2 locally took 100 ms 20/08/08 15:33:54.011 Executor task launch worker for task 13 DEBUG BlockManager: Putting block rdd_21_2 without replication took 100 ms 20/08/08 15:33:54.011 Executor task launch worker for task 13 DEBUG BlockManager: Getting local block rdd_21_2 20/08/08 15:33:54.011 Executor task launch worker for task 13 DEBUG BlockManager: Level for block rdd_21_2 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:54.011 Thread-4 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$doExecute$4$adapted 20/08/08 15:33:54.012 Executor task launch worker for task 13 INFO Executor: Finished task 2.0 in stage 3.0 (TID 13). 1305 bytes result sent to driver 20/08/08 15:33:54.013 dispatcher-event-loop-1 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_3.0, runningTasks: 0 20/08/08 15:33:54.013 Executor task launch worker for task 13 DEBUG ExecutorMetricsPoller: removing (3, 0) from stageTCMP 20/08/08 15:33:54.013 task-result-getter-3 INFO TaskSetManager: Finished task 2.0 in stage 3.0 (TID 13) in 116 ms on 192.168.1.225 (executor driver) (4/4) 20/08/08 15:33:54.014 task-result-getter-3 INFO TaskSchedulerImpl: Removed TaskSet 3.0, whose tasks have all completed, from pool 20/08/08 15:33:54.014 Thread-4 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$doExecute$4$adapted) is now cleaned +++ 20/08/08 15:33:54.014 dag-scheduler-event-loop INFO DAGScheduler: ResultStage 3 (count at CountVectorizer.scala:233) finished in 0.154 s 20/08/08 15:33:54.014 dag-scheduler-event-loop DEBUG DAGScheduler: After removal of stage 2, remaining stages = 1 20/08/08 15:33:54.015 dag-scheduler-event-loop DEBUG DAGScheduler: After removal of stage 3, remaining stages = 0 20/08/08 15:33:54.015 dag-scheduler-event-loop INFO DAGScheduler: Job 2 is finished. Cancelling potential speculative or zombie tasks for this job 20/08/08 15:33:54.015 dag-scheduler-event-loop INFO TaskSchedulerImpl: Killing all running tasks in stage 3: Stage finished 20/08/08 15:33:54.015 Thread-4 INFO DAGScheduler: Job 2 finished: count at CountVectorizer.scala:233, took 0.812001 s 20/08/08 15:33:54.018 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_50_9 for BlockManagerId(driver, 192.168.1.225, 38159, None) 20/08/08 15:33:54.019 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_50_9 in memory on 192.168.1.225:38159 (size: 16.0 B, free: 366.2 MiB) 20/08/08 15:33:54.019 Executor task launch worker for task 62 DEBUG BlockManagerMaster: Updated info of block rdd_50_9 20/08/08 15:33:54.019 Executor task launch worker for task 62 DEBUG BlockManager: Told master about block rdd_50_9 20/08/08 15:33:54.019 Executor task launch worker for task 62 DEBUG BlockManager: Put block rdd_50_9 locally took 22 ms 20/08/08 15:33:54.020 Executor task launch worker for task 62 DEBUG BlockManager: Putting block rdd_50_9 without replication took 22 ms 20/08/08 15:33:54.020 Executor task launch worker for task 62 DEBUG BlockManager: Getting local block rdd_50_9 20/08/08 15:33:54.020 Executor task launch worker for task 62 DEBUG BlockManager: Level for block rdd_50_9 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:54.020 Thread-4 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$takeOrdered$2 20/08/08 15:33:54.021 Executor task launch worker for task 62 INFO Executor: Finished task 9.0 in stage 12.0 (TID 62). 2213 bytes result sent to driver 20/08/08 15:33:54.021 dispatcher-event-loop-2 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_12.0, runningTasks: 0 20/08/08 15:33:54.022 task-result-getter-0 INFO TaskSetManager: Finished task 9.0 in stage 12.0 (TID 62) in 35 ms on 192.168.1.225 (executor driver) (10/10) 20/08/08 15:33:54.022 task-result-getter-0 INFO TaskSchedulerImpl: Removed TaskSet 12.0, whose tasks have all completed, from pool 20/08/08 15:33:54.022 Executor task launch worker for task 62 DEBUG ExecutorMetricsPoller: removing (12, 0) from stageTCMP 20/08/08 15:33:54.023 dag-scheduler-event-loop INFO DAGScheduler: ResultStage 12 (count at ALS.scala:1062) finished in 0.158 s 20/08/08 15:33:54.023 Thread-4 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$takeOrdered$2) is now cleaned +++ 20/08/08 15:33:54.023 dag-scheduler-event-loop DEBUG DAGScheduler: After removal of stage 8, remaining stages = 5 20/08/08 15:33:54.023 dag-scheduler-event-loop DEBUG DAGScheduler: After removal of stage 11, remaining stages = 4 20/08/08 15:33:54.023 dag-scheduler-event-loop DEBUG DAGScheduler: After removal of stage 7, remaining stages = 3 20/08/08 15:33:54.023 dag-scheduler-event-loop DEBUG DAGScheduler: After removal of stage 10, remaining stages = 2 20/08/08 15:33:54.024 dag-scheduler-event-loop DEBUG DAGScheduler: After removal of stage 9, remaining stages = 1 20/08/08 15:33:54.024 dag-scheduler-event-loop DEBUG DAGScheduler: After removal of stage 12, remaining stages = 0 20/08/08 15:33:54.024 dag-scheduler-event-loop INFO DAGScheduler: Job 3 is finished. Cancelling potential speculative or zombie tasks for this job 20/08/08 15:33:54.024 dag-scheduler-event-loop INFO TaskSchedulerImpl: Killing all running tasks in stage 12: Stage finished 20/08/08 15:33:54.024 Thread-4 INFO DAGScheduler: Job 3 finished: count at ALS.scala:1062, took 0.703363 s 20/08/08 15:33:54.027 Thread-4 INFO MapPartitionsRDD: Removing RDD 19 from persistence list 20/08/08 15:33:54.028 Thread-4 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$takeOrdered$3 20/08/08 15:33:54.035 Thread-4 INFO MapPartitionsRDD: Removing RDD 20 from persistence list 20/08/08 15:33:54.039 Thread-4 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$takeOrdered$3) is now cleaned +++ 20/08/08 15:33:54.040 Thread-4 INFO MapPartitionsRDD: Removing RDD 25 from persistence list 20/08/08 15:33:54.041 Thread-4 INFO MapPartitionsRDD: Removing RDD 16 from persistence list 20/08/08 15:33:54.042 Thread-4 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$runJob$6 20/08/08 15:33:54.042 Thread-4 DEBUG CoGroupedRDD: Adding one-to-one dependency with MapPartitionsRDD[51] at mapValues at ALS.scala:1052 20/08/08 15:33:54.042 Thread-4 DEBUG CoGroupedRDD: Adding one-to-one dependency with MapPartitionsRDD[36] at mapValues at ALS.scala:1702 20/08/08 15:33:54.042 Thread-4 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$count$1$adapted 20/08/08 15:33:54.044 Thread-4 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$runJob$6) is now cleaned +++ 20/08/08 15:33:54.044 Thread-4 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$count$1$adapted) is now cleaned +++ 20/08/08 15:33:54.045 Thread-4 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$doExecute$4$adapted 20/08/08 15:33:54.045 Thread-4 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$runJob$5 20/08/08 15:33:54.045 Thread-4 INFO SparkContext: Starting job: top at CountVectorizer.scala:236 20/08/08 15:33:54.046 dag-scheduler-event-loop DEBUG DAGScheduler: Merging stage rdd profiles: Set() 20/08/08 15:33:54.046 Thread-4 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$doExecute$4$adapted) is now cleaned +++ 20/08/08 15:33:54.046 dag-scheduler-event-loop DEBUG DAGScheduler: Merging stage rdd profiles: Set() 20/08/08 15:33:54.047 Thread-4 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$runJob$5) is now cleaned +++ 20/08/08 15:33:54.047 Thread-4 INFO SparkContext: Starting job: count at ALS.scala:1067 20/08/08 15:33:54.047 dag-scheduler-event-loop INFO DAGScheduler: Got job 3 (top at CountVectorizer.scala:236) with 4 output partitions 20/08/08 15:33:54.047 dag-scheduler-event-loop INFO DAGScheduler: Final stage: ResultStage 5 (top at CountVectorizer.scala:236) 20/08/08 15:33:54.047 dag-scheduler-event-loop INFO DAGScheduler: Parents of final stage: List(ShuffleMapStage 4) 20/08/08 15:33:54.048 dag-scheduler-event-loop DEBUG DAGScheduler: Merging stage rdd profiles: Set() 20/08/08 15:33:54.048 dag-scheduler-event-loop DEBUG DAGScheduler: Merging stage rdd profiles: Set() 20/08/08 15:33:54.048 dag-scheduler-event-loop INFO DAGScheduler: Missing parents: List() 20/08/08 15:33:54.048 dag-scheduler-event-loop DEBUG DAGScheduler: submitStage(ResultStage 5 (name=top at CountVectorizer.scala:236;jobs=3)) 20/08/08 15:33:54.049 block-manager-storage-async-thread-pool-3 DEBUG BlockManagerStorageEndpoint: removing RDD 19 20/08/08 15:33:54.049 dag-scheduler-event-loop DEBUG DAGScheduler: missing: List() 20/08/08 15:33:54.049 dag-scheduler-event-loop INFO DAGScheduler: Submitting ResultStage 5 (MapPartitionsRDD[22] at top at CountVectorizer.scala:236), which has no missing parents 20/08/08 15:33:54.049 dag-scheduler-event-loop DEBUG DAGScheduler: Merging stage rdd profiles: Set() 20/08/08 15:33:54.049 dag-scheduler-event-loop DEBUG DAGScheduler: submitMissingTasks(ResultStage 5) 20/08/08 15:33:54.049 block-manager-storage-async-thread-pool-3 INFO BlockManager: Removing RDD 19 20/08/08 15:33:54.049 dag-scheduler-event-loop DEBUG DAGScheduler: Merging stage rdd profiles: Set() 20/08/08 15:33:54.050 dag-scheduler-event-loop DEBUG DAGScheduler: Merging stage rdd profiles: Set() 20/08/08 15:33:54.050 block-manager-storage-async-thread-pool-3 DEBUG BlockManager: Removing block rdd_19_7 20/08/08 15:33:54.051 dag-scheduler-event-loop INFO DAGScheduler: Got job 4 (count at ALS.scala:1067) with 10 output partitions 20/08/08 15:33:54.051 dag-scheduler-event-loop INFO DAGScheduler: Final stage: ResultStage 17 (count at ALS.scala:1067) 20/08/08 15:33:54.051 dag-scheduler-event-loop INFO DAGScheduler: Parents of final stage: List(ShuffleMapStage 16, ShuffleMapStage 14) 20/08/08 15:33:54.051 block-manager-storage-async-thread-pool-3 DEBUG MemoryStore: Block rdd_19_7 of size 16 dropped from memory (free 383677169) 20/08/08 15:33:54.052 dag-scheduler-event-loop INFO DAGScheduler: Missing parents: List() 20/08/08 15:33:54.057 dag-scheduler-event-loop DEBUG DAGScheduler: submitStage(ResultStage 17 (name=count at ALS.scala:1067;jobs=4)) 20/08/08 15:33:54.058 dag-scheduler-event-loop DEBUG DAGScheduler: missing: List() 20/08/08 15:33:54.058 dag-scheduler-event-loop INFO DAGScheduler: Submitting ResultStage 17 (itemFactors MapPartitionsRDD[55] at mapPartitions at ALS.scala:1054), which has no missing parents 20/08/08 15:33:54.058 dag-scheduler-event-loop DEBUG DAGScheduler: submitMissingTasks(ResultStage 17) 20/08/08 15:33:54.060 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_6 stored as values in memory (estimated size 6.0 KiB, free 366.2 MiB) 20/08/08 15:33:54.060 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_6 locally took 1 ms 20/08/08 15:33:54.061 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_6 without replication took 1 ms 20/08/08 15:33:54.061 block-manager-storage-async-thread-pool-3 DEBUG BlockManager: Removing block rdd_19_6 20/08/08 15:33:54.061 block-manager-storage-async-thread-pool-3 DEBUG MemoryStore: Block rdd_19_6 of size 16 dropped from memory (free 383677185) 20/08/08 15:33:54.061 block-manager-storage-async-thread-pool-3 DEBUG BlockManager: Removing block rdd_19_0 20/08/08 15:33:54.062 block-manager-storage-async-thread-pool-3 DEBUG MemoryStore: Block rdd_19_0 of size 192 dropped from memory (free 383677377) 20/08/08 15:33:54.062 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_6_piece0 stored as bytes in memory (estimated size 3.1 KiB, free 366.2 MiB) 20/08/08 15:33:54.062 block-manager-storage-async-thread-pool-3 DEBUG BlockManager: Removing block rdd_19_9 20/08/08 15:33:54.062 block-manager-storage-async-thread-pool-3 DEBUG MemoryStore: Block rdd_19_9 of size 16 dropped from memory (free 383677393) 20/08/08 15:33:54.062 block-manager-storage-async-thread-pool-3 DEBUG BlockManager: Removing block rdd_19_4 20/08/08 15:33:54.062 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master broadcast_6_piece0 for BlockManagerId(driver, 192.168.1.225, 36433, None) 20/08/08 15:33:54.062 block-manager-storage-async-thread-pool-3 DEBUG MemoryStore: Block rdd_19_4 of size 16 dropped from memory (free 383677409) 20/08/08 15:33:54.062 block-manager-storage-async-thread-pool-3 DEBUG BlockManager: Removing block rdd_19_5 20/08/08 15:33:54.062 block-manager-storage-async-thread-pool-3 DEBUG MemoryStore: Block rdd_19_5 of size 16 dropped from memory (free 383677425) 20/08/08 15:33:54.062 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added broadcast_6_piece0 in memory on 192.168.1.225:36433 (size: 3.1 KiB, free: 366.3 MiB) 20/08/08 15:33:54.062 block-manager-storage-async-thread-pool-3 DEBUG BlockManager: Removing block rdd_19_1 20/08/08 15:33:54.062 block-manager-storage-async-thread-pool-3 DEBUG MemoryStore: Block rdd_19_1 of size 192 dropped from memory (free 383677617) 20/08/08 15:33:54.062 block-manager-storage-async-thread-pool-3 DEBUG BlockManager: Removing block rdd_19_2 20/08/08 15:33:54.062 block-manager-storage-async-thread-pool-3 DEBUG MemoryStore: Block rdd_19_2 of size 192 dropped from memory (free 383677809) 20/08/08 15:33:54.063 block-manager-storage-async-thread-pool-3 DEBUG BlockManager: Removing block rdd_19_8 20/08/08 15:33:54.063 block-manager-storage-async-thread-pool-3 DEBUG MemoryStore: Block rdd_19_8 of size 16 dropped from memory (free 383677825) 20/08/08 15:33:54.063 dag-scheduler-event-loop DEBUG BlockManagerMaster: Updated info of block broadcast_6_piece0 20/08/08 15:33:54.063 block-manager-storage-async-thread-pool-3 DEBUG BlockManager: Removing block rdd_19_3 20/08/08 15:33:54.063 dag-scheduler-event-loop DEBUG BlockManager: Told master about block broadcast_6_piece0 20/08/08 15:33:54.063 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_6_piece0 locally took 1 ms 20/08/08 15:33:54.063 block-manager-storage-async-thread-pool-3 DEBUG MemoryStore: Block rdd_19_3 of size 16 dropped from memory (free 383677841) 20/08/08 15:33:54.063 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_6_piece0 without replication took 1 ms 20/08/08 15:33:54.063 dag-scheduler-event-loop INFO SparkContext: Created broadcast 6 from broadcast at DAGScheduler.scala:1348 20/08/08 15:33:54.064 block-manager-storage-async-thread-pool-4 DEBUG BlockManagerStorageEndpoint: removing RDD 20 20/08/08 15:33:54.064 block-manager-storage-async-thread-pool-4 INFO BlockManager: Removing RDD 20 20/08/08 15:33:54.064 block-manager-storage-async-thread-pool-4 DEBUG BlockManager: Removing block rdd_20_6 20/08/08 15:33:54.064 block-manager-storage-async-thread-pool-4 DEBUG MemoryStore: Block rdd_20_6 of size 16 dropped from memory (free 383677857) 20/08/08 15:33:54.064 dag-scheduler-event-loop INFO DAGScheduler: Submitting 4 missing tasks from ResultStage 5 (MapPartitionsRDD[22] at top at CountVectorizer.scala:236) (first 15 tasks are for partitions Vector(0, 1, 2, 3)) 20/08/08 15:33:54.065 dag-scheduler-event-loop INFO TaskSchedulerImpl: Adding task set 5.0 with 4 tasks resource profile 0 20/08/08 15:33:54.065 dag-scheduler-event-loop DEBUG TaskSetManager: Epoch for TaskSet 5.0: 1 20/08/08 15:33:54.065 Thread-4 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$collect$2 20/08/08 15:33:54.066 dag-scheduler-event-loop DEBUG TaskSetManager: Adding pending tasks took 0 ms 20/08/08 15:33:54.066 dag-scheduler-event-loop DEBUG TaskSetManager: Valid locality levels for TaskSet 5.0: PROCESS_LOCAL, NODE_LOCAL, ANY 20/08/08 15:33:54.072 Thread-4 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$collect$2) is now cleaned +++ 20/08/08 15:33:54.080 Thread-4 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$runJob$5 20/08/08 15:33:54.081 dispatcher-event-loop-3 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_5.0, runningTasks: 0 20/08/08 15:33:54.082 dispatcher-event-loop-3 INFO TaskSetManager: Starting task 0.0 in stage 5.0 (TID 16, 192.168.1.225, executor driver, partition 0, PROCESS_LOCAL, 7143 bytes) taskResourceAssignments Map() 20/08/08 15:33:54.082 dispatcher-event-loop-3 INFO TaskSetManager: Starting task 1.0 in stage 5.0 (TID 17, 192.168.1.225, executor driver, partition 1, PROCESS_LOCAL, 7143 bytes) taskResourceAssignments Map() 20/08/08 15:33:54.082 dispatcher-event-loop-3 INFO TaskSetManager: Starting task 2.0 in stage 5.0 (TID 18, 192.168.1.225, executor driver, partition 2, PROCESS_LOCAL, 7143 bytes) taskResourceAssignments Map() 20/08/08 15:33:54.083 dispatcher-event-loop-3 INFO TaskSetManager: Starting task 3.0 in stage 5.0 (TID 19, 192.168.1.225, executor driver, partition 3, PROCESS_LOCAL, 7143 bytes) taskResourceAssignments Map() 20/08/08 15:33:54.083 Executor task launch worker for task 16 INFO Executor: Running task 0.0 in stage 5.0 (TID 16) 20/08/08 15:33:54.083 Executor task launch worker for task 18 INFO Executor: Running task 2.0 in stage 5.0 (TID 18) 20/08/08 15:33:54.084 Executor task launch worker for task 16 DEBUG ExecutorMetricsPoller: stageTCMP: (5, 0) -> 1 20/08/08 15:33:54.084 Executor task launch worker for task 18 DEBUG ExecutorMetricsPoller: stageTCMP: (5, 0) -> 2 20/08/08 15:33:54.085 Executor task launch worker for task 16 DEBUG BlockManager: Getting local block broadcast_6 20/08/08 15:33:54.085 Executor task launch worker for task 19 INFO Executor: Running task 3.0 in stage 5.0 (TID 19) 20/08/08 15:33:54.085 Executor task launch worker for task 16 DEBUG BlockManager: Level for block broadcast_6 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:54.085 Executor task launch worker for task 19 DEBUG ExecutorMetricsPoller: stageTCMP: (5, 0) -> 3 20/08/08 15:33:54.087 Thread-4 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$runJob$5) is now cleaned +++ 20/08/08 15:33:54.088 Thread-4 INFO SparkContext: Starting job: collect at ClusteringMetrics.scala:97 20/08/08 15:33:54.088 Executor task launch worker for task 17 INFO Executor: Running task 1.0 in stage 5.0 (TID 17) 20/08/08 15:33:54.088 dag-scheduler-event-loop DEBUG DAGScheduler: Merging stage rdd profiles: Set() 20/08/08 15:33:54.089 Executor task launch worker for task 17 DEBUG ExecutorMetricsPoller: stageTCMP: (5, 0) -> 4 20/08/08 15:33:54.089 dag-scheduler-event-loop DEBUG DAGScheduler: Merging stage rdd profiles: Set() 20/08/08 15:33:54.090 dag-scheduler-event-loop INFO DAGScheduler: Registering RDD 14 (collect at ClusteringMetrics.scala:97) as input to shuffle 1 20/08/08 15:33:54.090 dag-scheduler-event-loop INFO DAGScheduler: Got job 2 (collect at ClusteringMetrics.scala:97) with 1 output partitions 20/08/08 15:33:54.090 Executor task launch worker for task 16 DEBUG BlockManager: Getting local block rdd_21_0 20/08/08 15:33:54.090 dag-scheduler-event-loop INFO DAGScheduler: Final stage: ResultStage 4 (collect at ClusteringMetrics.scala:97) 20/08/08 15:33:54.090 dag-scheduler-event-loop INFO DAGScheduler: Parents of final stage: List(ShuffleMapStage 3) 20/08/08 15:33:54.090 Executor task launch worker for task 19 DEBUG BlockManager: Getting local block rdd_21_3 20/08/08 15:33:54.090 Executor task launch worker for task 16 DEBUG BlockManager: Level for block rdd_21_0 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:54.090 Executor task launch worker for task 18 DEBUG BlockManager: Getting local block rdd_21_2 20/08/08 15:33:54.090 Executor task launch worker for task 19 DEBUG BlockManager: Level for block rdd_21_3 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:54.090 dag-scheduler-event-loop INFO DAGScheduler: Missing parents: List(ShuffleMapStage 3) 20/08/08 15:33:54.090 Executor task launch worker for task 18 DEBUG BlockManager: Level for block rdd_21_2 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:54.091 dag-scheduler-event-loop DEBUG DAGScheduler: submitStage(ResultStage 4 (name=collect at ClusteringMetrics.scala:97;jobs=2)) 20/08/08 15:33:54.091 Executor task launch worker for task 16 INFO BlockManager: Found block rdd_21_0 locally 20/08/08 15:33:54.091 dag-scheduler-event-loop DEBUG DAGScheduler: missing: List(ShuffleMapStage 3) 20/08/08 15:33:54.091 Executor task launch worker for task 19 INFO BlockManager: Found block rdd_21_3 locally 20/08/08 15:33:54.091 Executor task launch worker for task 18 INFO BlockManager: Found block rdd_21_2 locally 20/08/08 15:33:54.091 dag-scheduler-event-loop DEBUG DAGScheduler: submitStage(ShuffleMapStage 3 (name=collect at ClusteringMetrics.scala:97;jobs=2)) 20/08/08 15:33:54.091 Executor task launch worker for task 17 DEBUG BlockManager: Getting local block rdd_21_1 20/08/08 15:33:54.091 dag-scheduler-event-loop DEBUG DAGScheduler: missing: List() 20/08/08 15:33:54.091 Executor task launch worker for task 17 DEBUG BlockManager: Level for block rdd_21_1 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:54.091 Executor task launch worker for task 17 INFO BlockManager: Found block rdd_21_1 locally 20/08/08 15:33:54.091 dag-scheduler-event-loop INFO DAGScheduler: Submitting ShuffleMapStage 3 (MapPartitionsRDD[14] at collect at ClusteringMetrics.scala:97), which has no missing parents 20/08/08 15:33:54.091 dag-scheduler-event-loop DEBUG DAGScheduler: submitMissingTasks(ShuffleMapStage 3) 20/08/08 15:33:54.092 block-manager-storage-async-thread-pool-4 DEBUG BlockManager: Removing block rdd_20_9 20/08/08 15:33:54.092 block-manager-storage-async-thread-pool-4 DEBUG MemoryStore: Block rdd_20_9 of size 16 dropped from memory (free 383677873) 20/08/08 15:33:54.092 block-manager-storage-async-thread-pool-4 DEBUG BlockManager: Removing block rdd_20_5 20/08/08 15:33:54.093 block-manager-storage-async-thread-pool-4 DEBUG MemoryStore: Block rdd_20_5 of size 16 dropped from memory (free 383677889) 20/08/08 15:33:54.093 block-manager-storage-async-thread-pool-4 DEBUG BlockManager: Removing block rdd_20_3 20/08/08 15:33:54.093 block-manager-storage-async-thread-pool-4 DEBUG MemoryStore: Block rdd_20_3 of size 16 dropped from memory (free 383677905) 20/08/08 15:33:54.093 block-manager-storage-async-thread-pool-4 DEBUG BlockManager: Removing block rdd_20_2 20/08/08 15:33:54.093 block-manager-storage-async-thread-pool-4 DEBUG MemoryStore: Block rdd_20_2 of size 296 dropped from memory (free 383678201) 20/08/08 15:33:54.093 block-manager-storage-async-thread-pool-4 DEBUG BlockManager: Removing block rdd_20_7 20/08/08 15:33:54.093 block-manager-storage-async-thread-pool-4 DEBUG MemoryStore: Block rdd_20_7 of size 16 dropped from memory (free 383678217) 20/08/08 15:33:54.093 block-manager-storage-async-thread-pool-4 DEBUG BlockManager: Removing block rdd_20_1 20/08/08 15:33:54.093 block-manager-storage-async-thread-pool-4 DEBUG MemoryStore: Block rdd_20_1 of size 296 dropped from memory (free 383678513) 20/08/08 15:33:54.093 block-manager-storage-async-thread-pool-4 DEBUG BlockManager: Removing block rdd_20_0 20/08/08 15:33:54.093 block-manager-storage-async-thread-pool-4 DEBUG MemoryStore: Block rdd_20_0 of size 296 dropped from memory (free 383678809) 20/08/08 15:33:54.093 block-manager-storage-async-thread-pool-4 DEBUG BlockManager: Removing block rdd_20_4 20/08/08 15:33:54.093 block-manager-storage-async-thread-pool-4 DEBUG MemoryStore: Block rdd_20_4 of size 16 dropped from memory (free 383678825) 20/08/08 15:33:54.093 block-manager-storage-async-thread-pool-4 DEBUG BlockManager: Removing block rdd_20_8 20/08/08 15:33:54.094 block-manager-storage-async-thread-pool-4 DEBUG MemoryStore: Block rdd_20_8 of size 16 dropped from memory (free 383678841) 20/08/08 15:33:54.105 Executor task launch worker for task 17 INFO Executor: Finished task 1.0 in stage 5.0 (TID 17). 2643 bytes result sent to driver 20/08/08 15:33:54.105 Executor task launch worker for task 18 INFO Executor: Finished task 2.0 in stage 5.0 (TID 18). 2643 bytes result sent to driver 20/08/08 15:33:54.106 Executor task launch worker for task 17 DEBUG ExecutorMetricsPoller: stageTCMP: (5, 0) -> 3 20/08/08 15:33:54.106 Executor task launch worker for task 18 DEBUG ExecutorMetricsPoller: stageTCMP: (5, 0) -> 2 20/08/08 15:33:54.106 dispatcher-event-loop-0 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_5.0, runningTasks: 3 20/08/08 15:33:54.106 dispatcher-event-loop-0 DEBUG TaskSetManager: No tasks for locality level PROCESS_LOCAL, so moving to locality level NODE_LOCAL 20/08/08 15:33:54.106 dispatcher-event-loop-0 DEBUG TaskSetManager: No tasks for locality level NODE_LOCAL, so moving to locality level ANY 20/08/08 15:33:54.106 Executor task launch worker for task 19 INFO Executor: Finished task 3.0 in stage 5.0 (TID 19). 2600 bytes result sent to driver 20/08/08 15:33:54.107 Executor task launch worker for task 19 DEBUG ExecutorMetricsPoller: stageTCMP: (5, 0) -> 1 20/08/08 15:33:54.107 dispatcher-event-loop-0 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_5.0, runningTasks: 2 20/08/08 15:33:54.107 dispatcher-event-loop-0 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_5.0, runningTasks: 1 20/08/08 15:33:54.109 task-result-getter-1 INFO TaskSetManager: Finished task 2.0 in stage 5.0 (TID 18) in 27 ms on 192.168.1.225 (executor driver) (1/4) 20/08/08 15:33:54.109 task-result-getter-0 INFO TaskSetManager: Finished task 1.0 in stage 5.0 (TID 17) in 27 ms on 192.168.1.225 (executor driver) (2/4) 20/08/08 15:33:54.109 task-result-getter-2 INFO TaskSetManager: Finished task 3.0 in stage 5.0 (TID 19) in 26 ms on 192.168.1.225 (executor driver) (3/4) 20/08/08 15:33:54.110 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_4 stored as values in memory (estimated size 34.9 KiB, free 366.2 MiB) 20/08/08 15:33:54.110 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_4 locally took 1 ms 20/08/08 15:33:54.110 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_4 without replication took 1 ms 20/08/08 15:33:54.112 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_4_piece0 stored as bytes in memory (estimated size 14.7 KiB, free 366.2 MiB) 20/08/08 15:33:54.112 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master broadcast_4_piece0 for BlockManagerId(driver, 192.168.1.225, 36391, None) 20/08/08 15:33:54.112 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added broadcast_4_piece0 in memory on 192.168.1.225:36391 (size: 14.7 KiB, free: 366.3 MiB) 20/08/08 15:33:54.114 dag-scheduler-event-loop DEBUG BlockManagerMaster: Updated info of block broadcast_4_piece0 20/08/08 15:33:54.114 dag-scheduler-event-loop DEBUG BlockManager: Told master about block broadcast_4_piece0 20/08/08 15:33:54.114 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_4_piece0 locally took 2 ms 20/08/08 15:33:54.114 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_4_piece0 without replication took 2 ms 20/08/08 15:33:54.114 dag-scheduler-event-loop INFO SparkContext: Created broadcast 4 from broadcast at DAGScheduler.scala:1348 20/08/08 15:33:54.115 dag-scheduler-event-loop INFO DAGScheduler: Submitting 4 missing tasks from ShuffleMapStage 3 (MapPartitionsRDD[14] at collect at ClusteringMetrics.scala:97) (first 15 tasks are for partitions Vector(0, 1, 2, 3)) 20/08/08 15:33:54.115 dag-scheduler-event-loop INFO TaskSchedulerImpl: Adding task set 3.0 with 4 tasks resource profile 0 20/08/08 15:33:54.115 dag-scheduler-event-loop DEBUG TaskSetManager: Epoch for TaskSet 3.0: 1 20/08/08 15:33:54.115 dag-scheduler-event-loop DEBUG TaskSetManager: Adding pending tasks took 0 ms 20/08/08 15:33:54.115 dag-scheduler-event-loop DEBUG TaskSetManager: Valid locality levels for TaskSet 3.0: NO_PREF, ANY 20/08/08 15:33:54.116 dispatcher-event-loop-0 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_3.0, runningTasks: 0 20/08/08 15:33:54.117 dispatcher-event-loop-0 INFO TaskSetManager: Starting task 0.0 in stage 3.0 (TID 9, 192.168.1.225, executor driver, partition 0, PROCESS_LOCAL, 7387 bytes) taskResourceAssignments Map() 20/08/08 15:33:54.117 dispatcher-event-loop-0 INFO TaskSetManager: Starting task 1.0 in stage 3.0 (TID 10, 192.168.1.225, executor driver, partition 1, PROCESS_LOCAL, 7452 bytes) taskResourceAssignments Map() 20/08/08 15:33:54.117 dispatcher-event-loop-0 INFO TaskSetManager: Starting task 2.0 in stage 3.0 (TID 11, 192.168.1.225, executor driver, partition 2, PROCESS_LOCAL, 7387 bytes) taskResourceAssignments Map() 20/08/08 15:33:54.118 dispatcher-event-loop-0 INFO TaskSetManager: Starting task 3.0 in stage 3.0 (TID 12, 192.168.1.225, executor driver, partition 3, PROCESS_LOCAL, 7452 bytes) taskResourceAssignments Map() 20/08/08 15:33:54.118 Executor task launch worker for task 10 INFO Executor: Running task 1.0 in stage 3.0 (TID 10) 20/08/08 15:33:54.118 Executor task launch worker for task 9 INFO Executor: Running task 0.0 in stage 3.0 (TID 9) 20/08/08 15:33:54.118 Executor task launch worker for task 12 INFO Executor: Running task 3.0 in stage 3.0 (TID 12) 20/08/08 15:33:54.119 Executor task launch worker for task 11 INFO Executor: Running task 2.0 in stage 3.0 (TID 11) 20/08/08 15:33:54.119 Executor task launch worker for task 11 DEBUG ExecutorMetricsPoller: stageTCMP: (3, 0) -> 1 20/08/08 15:33:54.120 Executor task launch worker for task 11 DEBUG BlockManager: Getting local block broadcast_4 20/08/08 15:33:54.120 Executor task launch worker for task 11 DEBUG BlockManager: Level for block broadcast_4 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:54.121 Executor task launch worker for task 16 INFO Executor: Finished task 0.0 in stage 5.0 (TID 16). 2426 bytes result sent to driver 20/08/08 15:33:54.122 block-manager-storage-async-thread-pool-5 DEBUG BlockManagerStorageEndpoint: removing RDD 25 20/08/08 15:33:54.122 Executor task launch worker for task 16 DEBUG ExecutorMetricsPoller: removing (5, 0) from stageTCMP 20/08/08 15:33:54.122 block-manager-storage-async-thread-pool-5 INFO BlockManager: Removing RDD 25 20/08/08 15:33:54.122 block-manager-storage-async-thread-pool-8 DEBUG BlockManagerStorageEndpoint: Done removing RDD 19, response is 0 20/08/08 15:33:54.122 block-manager-storage-async-thread-pool-5 DEBUG BlockManager: Removing block rdd_25_8 20/08/08 15:33:54.123 block-manager-storage-async-thread-pool-5 DEBUG MemoryStore: Block rdd_25_8 of size 16 dropped from memory (free 383678857) 20/08/08 15:33:54.123 block-manager-storage-async-thread-pool-5 DEBUG BlockManager: Removing block rdd_25_1 20/08/08 15:33:54.123 block-manager-storage-async-thread-pool-5 DEBUG MemoryStore: Block rdd_25_1 of size 304 dropped from memory (free 383679161) 20/08/08 15:33:54.123 block-manager-storage-async-thread-pool-5 DEBUG BlockManager: Removing block rdd_25_9 20/08/08 15:33:54.123 block-manager-storage-async-thread-pool-5 DEBUG MemoryStore: Block rdd_25_9 of size 16 dropped from memory (free 383679177) 20/08/08 15:33:54.123 block-manager-storage-async-thread-pool-5 DEBUG BlockManager: Removing block rdd_25_4 20/08/08 15:33:54.123 block-manager-storage-async-thread-pool-5 DEBUG MemoryStore: Block rdd_25_4 of size 16 dropped from memory (free 383679193) 20/08/08 15:33:54.123 block-manager-storage-async-thread-pool-5 DEBUG BlockManager: Removing block rdd_25_0 20/08/08 15:33:54.123 block-manager-storage-async-thread-pool-5 DEBUG MemoryStore: Block rdd_25_0 of size 288 dropped from memory (free 383679481) 20/08/08 15:33:54.123 block-manager-storage-async-thread-pool-5 DEBUG BlockManager: Removing block rdd_25_5 20/08/08 15:33:54.123 block-manager-storage-async-thread-pool-5 DEBUG MemoryStore: Block rdd_25_5 of size 16 dropped from memory (free 383679497) 20/08/08 15:33:54.124 block-manager-storage-async-thread-pool-5 DEBUG BlockManager: Removing block rdd_25_3 20/08/08 15:33:54.124 block-manager-storage-async-thread-pool-5 DEBUG MemoryStore: Block rdd_25_3 of size 16 dropped from memory (free 383679513) 20/08/08 15:33:54.124 block-manager-storage-async-thread-pool-5 DEBUG BlockManager: Removing block rdd_25_7 20/08/08 15:33:54.124 block-manager-storage-async-thread-pool-5 DEBUG MemoryStore: Block rdd_25_7 of size 16 dropped from memory (free 383679529) 20/08/08 15:33:54.124 block-manager-storage-async-thread-pool-5 DEBUG BlockManager: Removing block rdd_25_6 20/08/08 15:33:54.124 block-manager-storage-async-thread-pool-5 DEBUG MemoryStore: Block rdd_25_6 of size 16 dropped from memory (free 383679545) 20/08/08 15:33:54.124 block-manager-storage-async-thread-pool-5 DEBUG BlockManager: Removing block rdd_25_2 20/08/08 15:33:54.124 block-manager-storage-async-thread-pool-5 DEBUG MemoryStore: Block rdd_25_2 of size 296 dropped from memory (free 383679841) 20/08/08 15:33:54.125 Executor task launch worker for task 9 DEBUG ExecutorMetricsPoller: stageTCMP: (3, 0) -> 2 20/08/08 15:33:54.128 Executor task launch worker for task 10 DEBUG ExecutorMetricsPoller: stageTCMP: (3, 0) -> 3 20/08/08 15:33:54.132 dispatcher-event-loop-3 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_5.0, runningTasks: 0 20/08/08 15:33:54.133 task-result-getter-3 INFO TaskSetManager: Finished task 0.0 in stage 5.0 (TID 16) in 52 ms on 192.168.1.225 (executor driver) (4/4) 20/08/08 15:33:54.134 task-result-getter-3 INFO TaskSchedulerImpl: Removed TaskSet 5.0, whose tasks have all completed, from pool 20/08/08 15:33:54.134 dag-scheduler-event-loop INFO DAGScheduler: ResultStage 5 (top at CountVectorizer.scala:236) finished in 0.084 s 20/08/08 15:33:54.135 dag-scheduler-event-loop DEBUG DAGScheduler: After removal of stage 5, remaining stages = 1 20/08/08 15:33:54.135 dag-scheduler-event-loop DEBUG DAGScheduler: After removal of stage 4, remaining stages = 0 20/08/08 15:33:54.135 dag-scheduler-event-loop INFO DAGScheduler: Job 3 is finished. Cancelling potential speculative or zombie tasks for this job 20/08/08 15:33:54.135 dag-scheduler-event-loop INFO TaskSchedulerImpl: Killing all running tasks in stage 5: Stage finished 20/08/08 15:33:54.135 Thread-4 INFO DAGScheduler: Job 3 finished: top at CountVectorizer.scala:236, took 0.090087 s 20/08/08 15:33:54.137 Executor task launch worker for task 12 DEBUG ExecutorMetricsPoller: stageTCMP: (3, 0) -> 4 20/08/08 15:33:54.137 block-manager-storage-async-thread-pool-8 DEBUG BlockManagerStorageEndpoint: Sent response: 0 to 192.168.1.225:44433 20/08/08 15:33:54.137 Thread-4 INFO MapPartitionsRDD: Removing RDD 21 from persistence list 20/08/08 15:33:54.139 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_9 stored as values in memory (estimated size 37.9 KiB, free 365.9 MiB) 20/08/08 15:33:54.139 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_9 locally took 0 ms 20/08/08 15:33:54.139 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_9 without replication took 0 ms 20/08/08 15:33:54.140 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_9_piece0 stored as bytes in memory (estimated size 17.1 KiB, free 365.9 MiB) 20/08/08 15:33:54.141 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master broadcast_9_piece0 for BlockManagerId(driver, 192.168.1.225, 38159, None) 20/08/08 15:33:54.141 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added broadcast_9_piece0 in memory on 192.168.1.225:38159 (size: 17.1 KiB, free: 366.2 MiB) 20/08/08 15:33:54.141 dag-scheduler-event-loop DEBUG BlockManagerMaster: Updated info of block broadcast_9_piece0 20/08/08 15:33:54.141 dag-scheduler-event-loop DEBUG BlockManager: Told master about block broadcast_9_piece0 20/08/08 15:33:54.141 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_9_piece0 locally took 1 ms 20/08/08 15:33:54.141 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_9_piece0 without replication took 1 ms 20/08/08 15:33:54.142 dag-scheduler-event-loop INFO SparkContext: Created broadcast 9 from broadcast at DAGScheduler.scala:1348 20/08/08 15:33:54.142 dag-scheduler-event-loop INFO DAGScheduler: Submitting 10 missing tasks from ResultStage 17 (itemFactors MapPartitionsRDD[55] at mapPartitions at ALS.scala:1054) (first 15 tasks are for partitions Vector(0, 1, 2, 3, 4, 5, 6, 7, 8, 9)) 20/08/08 15:33:54.142 dag-scheduler-event-loop INFO TaskSchedulerImpl: Adding task set 17.0 with 10 tasks resource profile 0 20/08/08 15:33:54.143 dag-scheduler-event-loop DEBUG TaskSetManager: Epoch for TaskSet 17.0: 5 20/08/08 15:33:54.143 dag-scheduler-event-loop DEBUG TaskSetManager: Adding pending tasks took 0 ms 20/08/08 15:33:54.143 dag-scheduler-event-loop DEBUG TaskSetManager: Valid locality levels for TaskSet 17.0: PROCESS_LOCAL, NODE_LOCAL, ANY 20/08/08 15:33:54.143 dispatcher-event-loop-3 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_17.0, runningTasks: 0 20/08/08 15:33:54.144 dispatcher-event-loop-3 INFO TaskSetManager: Starting task 0.0 in stage 17.0 (TID 63, 192.168.1.225, executor driver, partition 0, PROCESS_LOCAL, 7433 bytes) taskResourceAssignments Map() 20/08/08 15:33:54.144 dispatcher-event-loop-3 INFO TaskSetManager: Starting task 1.0 in stage 17.0 (TID 64, 192.168.1.225, executor driver, partition 1, PROCESS_LOCAL, 7433 bytes) taskResourceAssignments Map() 20/08/08 15:33:54.145 dispatcher-event-loop-3 INFO TaskSetManager: Starting task 2.0 in stage 17.0 (TID 65, 192.168.1.225, executor driver, partition 2, PROCESS_LOCAL, 7433 bytes) taskResourceAssignments Map() 20/08/08 15:33:54.145 dispatcher-event-loop-3 INFO TaskSetManager: Starting task 3.0 in stage 17.0 (TID 66, 192.168.1.225, executor driver, partition 3, PROCESS_LOCAL, 7433 bytes) taskResourceAssignments Map() 20/08/08 15:33:54.145 Executor task launch worker for task 64 INFO Executor: Running task 1.0 in stage 17.0 (TID 64) 20/08/08 15:33:54.145 Executor task launch worker for task 63 INFO Executor: Running task 0.0 in stage 17.0 (TID 63) 20/08/08 15:33:54.146 Executor task launch worker for task 63 DEBUG ExecutorMetricsPoller: stageTCMP: (17, 0) -> 1 20/08/08 15:33:54.146 Executor task launch worker for task 66 INFO Executor: Running task 3.0 in stage 17.0 (TID 66) 20/08/08 15:33:54.146 Executor task launch worker for task 65 INFO Executor: Running task 2.0 in stage 17.0 (TID 65) 20/08/08 15:33:54.146 Executor task launch worker for task 66 DEBUG ExecutorMetricsPoller: stageTCMP: (17, 0) -> 2 20/08/08 15:33:54.146 Executor task launch worker for task 65 DEBUG ExecutorMetricsPoller: stageTCMP: (17, 0) -> 3 20/08/08 15:33:54.147 Executor task launch worker for task 63 DEBUG BlockManager: Getting local block broadcast_9 20/08/08 15:33:54.147 Executor task launch worker for task 63 DEBUG BlockManager: Level for block broadcast_9 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:54.153 Executor task launch worker for task 64 DEBUG ExecutorMetricsPoller: stageTCMP: (17, 0) -> 4 20/08/08 15:33:54.156 Executor task launch worker for task 63 DEBUG BlockManager: Getting local block rdd_55_0 20/08/08 15:33:54.156 Executor task launch worker for task 63 DEBUG BlockManager: Block rdd_55_0 was not found 20/08/08 15:33:54.156 Executor task launch worker for task 63 DEBUG BlockManager: Getting remote block rdd_55_0 20/08/08 15:33:54.156 Executor task launch worker for task 63 DEBUG BlockManager: Block rdd_55_0 is unknown by block manager master 20/08/08 15:33:54.157 Executor task launch worker for task 63 DEBUG BlockManager: Getting local block rdd_24_0 20/08/08 15:33:54.157 Executor task launch worker for task 63 DEBUG BlockManager: Level for block rdd_24_0 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:54.157 Executor task launch worker for task 63 INFO BlockManager: Found block rdd_24_0 locally 20/08/08 15:33:54.157 Executor task launch worker for task 63 DEBUG BlockManager: Getting local block rdd_24_0 20/08/08 15:33:54.157 Executor task launch worker for task 63 DEBUG BlockManager: Level for block rdd_24_0 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:54.157 Executor task launch worker for task 63 INFO BlockManager: Found block rdd_24_0 locally 20/08/08 15:33:54.157 Executor task launch worker for task 63 DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 4 20/08/08 15:33:54.157 Executor task launch worker for task 63 DEBUG MapOutputTrackerMaster: Convert map statuses for shuffle 4, mappers 0-10, partitions 0-1 20/08/08 15:33:54.157 Executor task launch worker for task 63 DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 20/08/08 15:33:54.158 Executor task launch worker for task 63 INFO ShuffleBlockFetcherIterator: Getting 3 (684.0 B) non-empty blocks including 3 (684.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 20/08/08 15:33:54.158 Executor task launch worker for task 63 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 20/08/08 15:33:54.158 Executor task launch worker for task 63 DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_4_33_0,0), (shuffle_4_34_0,1), (shuffle_4_35_0,2) 20/08/08 15:33:54.158 Executor task launch worker for task 63 INFO BlockManager: Getting local shuffle block shuffle_4_33_0 20/08/08 15:33:54.158 Executor task launch worker for task 63 INFO BlockManager: Getting local shuffle block shuffle_4_34_0 20/08/08 15:33:54.158 Executor task launch worker for task 63 INFO BlockManager: Getting local shuffle block shuffle_4_35_0 20/08/08 15:33:54.158 Executor task launch worker for task 63 DEBUG ShuffleBlockFetcherIterator: Got local blocks in 1 ms 20/08/08 15:33:54.159 Executor task launch worker for task 64 DEBUG BlockManager: Getting local block rdd_55_1 20/08/08 15:33:54.159 Executor task launch worker for task 64 DEBUG BlockManager: Block rdd_55_1 was not found 20/08/08 15:33:54.159 Executor task launch worker for task 64 DEBUG BlockManager: Getting remote block rdd_55_1 20/08/08 15:33:54.159 Executor task launch worker for task 64 DEBUG BlockManager: Block rdd_55_1 is unknown by block manager master 20/08/08 15:33:54.159 Executor task launch worker for task 64 DEBUG BlockManager: Getting local block rdd_24_1 20/08/08 15:33:54.159 Executor task launch worker for task 64 DEBUG BlockManager: Level for block rdd_24_1 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:54.159 Executor task launch worker for task 64 INFO BlockManager: Found block rdd_24_1 locally 20/08/08 15:33:54.160 Executor task launch worker for task 64 DEBUG BlockManager: Getting local block rdd_24_1 20/08/08 15:33:54.160 Executor task launch worker for task 64 DEBUG BlockManager: Level for block rdd_24_1 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:54.160 Executor task launch worker for task 64 INFO BlockManager: Found block rdd_24_1 locally 20/08/08 15:33:54.160 Executor task launch worker for task 64 DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 4 20/08/08 15:33:54.160 Executor task launch worker for task 64 DEBUG MapOutputTrackerMaster: Convert map statuses for shuffle 4, mappers 0-10, partitions 1-2 20/08/08 15:33:54.160 Executor task launch worker for task 64 DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 20/08/08 15:33:54.160 Executor task launch worker for task 63 DEBUG TaskMemoryManager: Task 63 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@27688332 20/08/08 15:33:54.161 Executor task launch worker for task 64 INFO ShuffleBlockFetcherIterator: Getting 3 (730.0 B) non-empty blocks including 3 (730.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 20/08/08 15:33:54.161 Executor task launch worker for task 64 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 20/08/08 15:33:54.161 Executor task launch worker for task 63 DEBUG TaskMemoryManager: Task 63 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@58bd331f 20/08/08 15:33:54.162 Executor task launch worker for task 63 DEBUG TaskMemoryManager: Task 63 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@2f24ba48 20/08/08 15:33:54.162 Executor task launch worker for task 63 INFO MemoryStore: Block rdd_55_0 stored as values in memory (estimated size 88.0 B, free 365.9 MiB) 20/08/08 15:33:54.163 Executor task launch worker for task 65 DEBUG BlockManager: Getting local block rdd_55_2 20/08/08 15:33:54.163 Executor task launch worker for task 65 DEBUG BlockManager: Block rdd_55_2 was not found 20/08/08 15:33:54.163 Executor task launch worker for task 65 DEBUG BlockManager: Getting remote block rdd_55_2 20/08/08 15:33:54.167 Executor task launch worker for task 66 DEBUG BlockManager: Getting local block rdd_55_3 20/08/08 15:33:54.167 Executor task launch worker for task 66 DEBUG BlockManager: Block rdd_55_3 was not found 20/08/08 15:33:54.167 Executor task launch worker for task 66 DEBUG BlockManager: Getting remote block rdd_55_3 20/08/08 15:33:54.167 Executor task launch worker for task 64 DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_4_33_1,0), (shuffle_4_34_1,1), (shuffle_4_35_1,2) 20/08/08 15:33:54.167 Executor task launch worker for task 64 INFO BlockManager: Getting local shuffle block shuffle_4_33_1 20/08/08 15:33:54.168 Executor task launch worker for task 64 INFO BlockManager: Getting local shuffle block shuffle_4_34_1 20/08/08 15:33:54.168 Executor task launch worker for task 64 INFO BlockManager: Getting local shuffle block shuffle_4_35_1 20/08/08 15:33:54.168 Executor task launch worker for task 64 DEBUG ShuffleBlockFetcherIterator: Got local blocks in 7 ms 20/08/08 15:33:54.169 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_55_0 for BlockManagerId(driver, 192.168.1.225, 38159, None) 20/08/08 15:33:54.169 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_55_0 in memory on 192.168.1.225:38159 (size: 88.0 B, free: 366.2 MiB) 20/08/08 15:33:54.169 Executor task launch worker for task 63 DEBUG BlockManagerMaster: Updated info of block rdd_55_0 20/08/08 15:33:54.169 Executor task launch worker for task 63 DEBUG BlockManager: Told master about block rdd_55_0 20/08/08 15:33:54.169 Executor task launch worker for task 63 DEBUG BlockManager: Put block rdd_55_0 locally took 12 ms 20/08/08 15:33:54.170 Executor task launch worker for task 63 DEBUG BlockManager: Putting block rdd_55_0 without replication took 12 ms 20/08/08 15:33:54.170 Executor task launch worker for task 63 DEBUG BlockManager: Getting local block rdd_55_0 20/08/08 15:33:54.170 Executor task launch worker for task 63 DEBUG BlockManager: Level for block rdd_55_0 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:54.171 Executor task launch worker for task 63 INFO Executor: Finished task 0.0 in stage 17.0 (TID 63). 2256 bytes result sent to driver 20/08/08 15:33:54.171 Executor task launch worker for task 63 DEBUG ExecutorMetricsPoller: stageTCMP: (17, 0) -> 3 20/08/08 15:33:54.171 Executor task launch worker for task 66 DEBUG BlockManager: Block rdd_55_3 is unknown by block manager master 20/08/08 15:33:54.171 dispatcher-event-loop-1 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_17.0, runningTasks: 3 20/08/08 15:33:54.171 Executor task launch worker for task 65 DEBUG BlockManager: Block rdd_55_2 is unknown by block manager master 20/08/08 15:33:54.171 Executor task launch worker for task 66 DEBUG BlockManager: Getting local block rdd_24_3 20/08/08 15:33:54.171 Executor task launch worker for task 66 DEBUG BlockManager: Level for block rdd_24_3 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:54.171 Executor task launch worker for task 65 DEBUG BlockManager: Getting local block rdd_24_2 20/08/08 15:33:54.171 Executor task launch worker for task 65 DEBUG BlockManager: Level for block rdd_24_2 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:54.172 Executor task launch worker for task 65 INFO BlockManager: Found block rdd_24_2 locally 20/08/08 15:33:54.172 Executor task launch worker for task 65 DEBUG BlockManager: Getting local block rdd_24_2 20/08/08 15:33:54.172 Executor task launch worker for task 65 DEBUG BlockManager: Level for block rdd_24_2 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:54.172 Executor task launch worker for task 65 INFO BlockManager: Found block rdd_24_2 locally 20/08/08 15:33:54.172 Executor task launch worker for task 65 DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 4 20/08/08 15:33:54.172 Executor task launch worker for task 65 DEBUG MapOutputTrackerMaster: Convert map statuses for shuffle 4, mappers 0-10, partitions 2-3 20/08/08 15:33:54.172 Executor task launch worker for task 65 DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 20/08/08 15:33:54.173 Executor task launch worker for task 64 DEBUG TaskMemoryManager: Task 64 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@7b80cfdd 20/08/08 15:33:54.173 Executor task launch worker for task 65 INFO ShuffleBlockFetcherIterator: Getting 3 (707.0 B) non-empty blocks including 3 (707.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 20/08/08 15:33:54.173 Executor task launch worker for task 65 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 20/08/08 15:33:54.173 Executor task launch worker for task 65 DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_4_33_2,0), (shuffle_4_34_2,1), (shuffle_4_35_2,2) 20/08/08 15:33:54.173 Executor task launch worker for task 65 INFO BlockManager: Getting local shuffle block shuffle_4_33_2 20/08/08 15:33:54.173 Executor task launch worker for task 65 INFO BlockManager: Getting local shuffle block shuffle_4_34_2 20/08/08 15:33:54.173 Executor task launch worker for task 65 INFO BlockManager: Getting local shuffle block shuffle_4_35_2 20/08/08 15:33:54.173 Executor task launch worker for task 64 DEBUG TaskMemoryManager: Task 64 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@2d1efbc7 20/08/08 15:33:54.173 Executor task launch worker for task 65 DEBUG ShuffleBlockFetcherIterator: Got local blocks in 1 ms 20/08/08 15:33:54.174 Executor task launch worker for task 64 DEBUG TaskMemoryManager: Task 64 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@2bf18b8f 20/08/08 15:33:54.174 Executor task launch worker for task 64 INFO MemoryStore: Block rdd_55_1 stored as values in memory (estimated size 88.0 B, free 365.9 MiB) 20/08/08 15:33:54.171 Executor task launch worker for task 66 INFO BlockManager: Found block rdd_24_3 locally 20/08/08 15:33:54.172 dispatcher-event-loop-1 INFO TaskSetManager: Starting task 4.0 in stage 17.0 (TID 67, 192.168.1.225, executor driver, partition 4, PROCESS_LOCAL, 7433 bytes) taskResourceAssignments Map() 20/08/08 15:33:54.174 Executor task launch worker for task 66 DEBUG BlockManager: Getting local block rdd_24_3 20/08/08 15:33:54.174 Executor task launch worker for task 66 DEBUG BlockManager: Level for block rdd_24_3 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:54.175 Executor task launch worker for task 66 INFO BlockManager: Found block rdd_24_3 locally 20/08/08 15:33:54.175 Executor task launch worker for task 66 DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 4 20/08/08 15:33:54.175 Executor task launch worker for task 66 DEBUG MapOutputTrackerMaster: Convert map statuses for shuffle 4, mappers 0-10, partitions 3-4 20/08/08 15:33:54.175 Executor task launch worker for task 67 INFO Executor: Running task 4.0 in stage 17.0 (TID 67) 20/08/08 15:33:54.175 task-result-getter-1 INFO TaskSetManager: Finished task 0.0 in stage 17.0 (TID 63) in 31 ms on 192.168.1.225 (executor driver) (1/10) 20/08/08 15:33:54.175 Executor task launch worker for task 66 DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 20/08/08 15:33:54.175 Executor task launch worker for task 65 DEBUG TaskMemoryManager: Task 65 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@31d05ba2 20/08/08 15:33:54.176 Executor task launch worker for task 67 DEBUG ExecutorMetricsPoller: stageTCMP: (17, 0) -> 4 20/08/08 15:33:54.176 Executor task launch worker for task 65 DEBUG TaskMemoryManager: Task 65 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@34648e07 20/08/08 15:33:54.176 Executor task launch worker for task 65 DEBUG TaskMemoryManager: Task 65 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@14afe3c6 20/08/08 15:33:54.177 Executor task launch worker for task 65 INFO MemoryStore: Block rdd_55_2 stored as values in memory (estimated size 88.0 B, free 365.9 MiB) 20/08/08 15:33:54.177 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_55_2 for BlockManagerId(driver, 192.168.1.225, 38159, None) 20/08/08 15:33:54.177 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_55_2 in memory on 192.168.1.225:38159 (size: 88.0 B, free: 366.2 MiB) 20/08/08 15:33:54.177 Executor task launch worker for task 66 INFO ShuffleBlockFetcherIterator: Getting 3 (684.0 B) non-empty blocks including 3 (684.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 20/08/08 15:33:54.177 Executor task launch worker for task 66 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 2 ms 20/08/08 15:33:54.178 Executor task launch worker for task 66 DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_4_33_3,0), (shuffle_4_34_3,1), (shuffle_4_35_3,2) 20/08/08 15:33:54.178 Executor task launch worker for task 66 INFO BlockManager: Getting local shuffle block shuffle_4_33_3 20/08/08 15:33:54.177 block-manager-storage-async-thread-pool-0 DEBUG BlockManagerStorageEndpoint: removing RDD 21 20/08/08 15:33:54.178 Executor task launch worker for task 66 INFO BlockManager: Getting local shuffle block shuffle_4_34_3 20/08/08 15:33:54.178 Executor task launch worker for task 66 INFO BlockManager: Getting local shuffle block shuffle_4_35_3 20/08/08 15:33:54.178 Executor task launch worker for task 66 DEBUG ShuffleBlockFetcherIterator: Got local blocks in 3 ms 20/08/08 15:33:54.178 block-manager-storage-async-thread-pool-0 INFO BlockManager: Removing RDD 21 20/08/08 15:33:54.180 Executor task launch worker for task 66 DEBUG TaskMemoryManager: Task 66 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@1f9c74f4 20/08/08 15:33:54.180 Executor task launch worker for task 66 DEBUG TaskMemoryManager: Task 66 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@37f9a017 20/08/08 15:33:54.180 block-manager-storage-async-thread-pool-0 DEBUG BlockManager: Removing block rdd_21_0 20/08/08 15:33:54.180 Executor task launch worker for task 66 DEBUG TaskMemoryManager: Task 66 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@64acaf48 20/08/08 15:33:54.181 Executor task launch worker for task 66 INFO MemoryStore: Block rdd_55_3 stored as values in memory (estimated size 16.0 B, free 365.9 MiB) 20/08/08 15:33:54.181 Executor task launch worker for task 65 DEBUG BlockManagerMaster: Updated info of block rdd_55_2 20/08/08 15:33:54.181 Executor task launch worker for task 65 DEBUG BlockManager: Told master about block rdd_55_2 20/08/08 15:33:54.181 Executor task launch worker for task 65 DEBUG BlockManager: Put block rdd_55_2 locally took 9 ms 20/08/08 15:33:54.181 Executor task launch worker for task 65 DEBUG BlockManager: Putting block rdd_55_2 without replication took 9 ms 20/08/08 15:33:54.181 Executor task launch worker for task 65 DEBUG BlockManager: Getting local block rdd_55_2 20/08/08 15:33:54.181 block-manager-storage-async-thread-pool-0 DEBUG MemoryStore: Block rdd_21_0 of size 16 dropped from memory (free 383955399) 20/08/08 15:33:54.181 Executor task launch worker for task 65 DEBUG BlockManager: Level for block rdd_55_2 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:54.182 Executor task launch worker for task 65 INFO Executor: Finished task 2.0 in stage 17.0 (TID 65). 2256 bytes result sent to driver 20/08/08 15:33:54.182 Executor task launch worker for task 65 DEBUG ExecutorMetricsPoller: stageTCMP: (17, 0) -> 3 20/08/08 15:33:54.182 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_55_3 for BlockManagerId(driver, 192.168.1.225, 38159, None) 20/08/08 15:33:54.182 block-manager-storage-async-thread-pool-0 DEBUG BlockManager: Removing block rdd_21_1 20/08/08 15:33:54.182 dispatcher-event-loop-2 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_17.0, runningTasks: 3 20/08/08 15:33:54.182 block-manager-storage-async-thread-pool-0 DEBUG MemoryStore: Block rdd_21_1 of size 120 dropped from memory (free 383955519) 20/08/08 15:33:54.183 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_55_3 in memory on 192.168.1.225:38159 (size: 16.0 B, free: 366.2 MiB) 20/08/08 15:33:54.183 block-manager-storage-async-thread-pool-0 DEBUG BlockManager: Removing block rdd_21_3 20/08/08 15:33:54.183 block-manager-storage-async-thread-pool-0 DEBUG MemoryStore: Block rdd_21_3 of size 120 dropped from memory (free 383955639) 20/08/08 15:33:54.183 dispatcher-event-loop-2 INFO TaskSetManager: Starting task 5.0 in stage 17.0 (TID 68, 192.168.1.225, executor driver, partition 5, PROCESS_LOCAL, 7433 bytes) taskResourceAssignments Map() 20/08/08 15:33:54.183 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_55_1 for BlockManagerId(driver, 192.168.1.225, 38159, None) 20/08/08 15:33:54.183 block-manager-storage-async-thread-pool-0 DEBUG BlockManager: Removing block rdd_21_2 20/08/08 15:33:54.183 block-manager-storage-async-thread-pool-0 DEBUG MemoryStore: Block rdd_21_2 of size 120 dropped from memory (free 383955759) 20/08/08 15:33:54.183 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_55_1 in memory on 192.168.1.225:38159 (size: 88.0 B, free: 366.2 MiB) 20/08/08 15:33:54.183 Executor task launch worker for task 68 INFO Executor: Running task 5.0 in stage 17.0 (TID 68) 20/08/08 15:33:54.184 task-result-getter-3 INFO TaskSetManager: Finished task 2.0 in stage 17.0 (TID 65) in 39 ms on 192.168.1.225 (executor driver) (2/10) 20/08/08 15:33:54.182 Executor task launch worker for task 67 DEBUG BlockManager: Getting local block rdd_55_4 20/08/08 15:33:54.184 Executor task launch worker for task 67 DEBUG BlockManager: Block rdd_55_4 was not found 20/08/08 15:33:54.184 Executor task launch worker for task 67 DEBUG BlockManager: Getting remote block rdd_55_4 20/08/08 15:33:54.183 Executor task launch worker for task 66 DEBUG BlockManagerMaster: Updated info of block rdd_55_3 20/08/08 15:33:54.184 Executor task launch worker for task 66 DEBUG BlockManager: Told master about block rdd_55_3 20/08/08 15:33:54.184 Executor task launch worker for task 66 DEBUG BlockManager: Put block rdd_55_3 locally took 13 ms 20/08/08 15:33:54.184 Executor task launch worker for task 66 DEBUG BlockManager: Putting block rdd_55_3 without replication took 13 ms 20/08/08 15:33:54.184 Executor task launch worker for task 66 DEBUG BlockManager: Getting local block rdd_55_3 20/08/08 15:33:54.184 Executor task launch worker for task 66 DEBUG BlockManager: Level for block rdd_55_3 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:54.185 Executor task launch worker for task 68 DEBUG ExecutorMetricsPoller: stageTCMP: (17, 0) -> 4 20/08/08 15:33:54.185 Executor task launch worker for task 67 DEBUG BlockManager: Block rdd_55_4 is unknown by block manager master 20/08/08 15:33:54.185 Executor task launch worker for task 67 DEBUG BlockManager: Getting local block rdd_24_4 20/08/08 15:33:54.185 Executor task launch worker for task 67 DEBUG BlockManager: Level for block rdd_24_4 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:54.185 Executor task launch worker for task 67 INFO BlockManager: Found block rdd_24_4 locally 20/08/08 15:33:54.185 Executor task launch worker for task 67 DEBUG BlockManager: Getting local block rdd_24_4 20/08/08 15:33:54.185 Executor task launch worker for task 67 DEBUG BlockManager: Level for block rdd_24_4 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:54.185 Executor task launch worker for task 67 INFO BlockManager: Found block rdd_24_4 locally 20/08/08 15:33:54.185 Executor task launch worker for task 67 DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 4 20/08/08 15:33:54.185 Executor task launch worker for task 67 DEBUG MapOutputTrackerMaster: Convert map statuses for shuffle 4, mappers 0-10, partitions 4-5 20/08/08 15:33:54.186 Executor task launch worker for task 67 DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 20/08/08 15:33:54.186 Executor task launch worker for task 67 INFO ShuffleBlockFetcherIterator: Getting 3 (684.0 B) non-empty blocks including 3 (684.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 20/08/08 15:33:54.186 Executor task launch worker for task 67 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 20/08/08 15:33:54.186 Executor task launch worker for task 67 DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_4_33_4,0), (shuffle_4_34_4,1), (shuffle_4_35_4,2) 20/08/08 15:33:54.186 Executor task launch worker for task 67 INFO BlockManager: Getting local shuffle block shuffle_4_33_4 20/08/08 15:33:54.187 Executor task launch worker for task 67 INFO BlockManager: Getting local shuffle block shuffle_4_34_4 20/08/08 15:33:54.187 Executor task launch worker for task 67 INFO BlockManager: Getting local shuffle block shuffle_4_35_4 20/08/08 15:33:54.187 Executor task launch worker for task 67 DEBUG ShuffleBlockFetcherIterator: Got local blocks in 1 ms 20/08/08 15:33:54.188 Executor task launch worker for task 67 DEBUG TaskMemoryManager: Task 67 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@2cf7944b 20/08/08 15:33:54.189 Executor task launch worker for task 67 DEBUG TaskMemoryManager: Task 67 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@74ddde2c 20/08/08 15:33:54.189 Executor task launch worker for task 67 DEBUG TaskMemoryManager: Task 67 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@4b362bff 20/08/08 15:33:54.189 Executor task launch worker for task 67 INFO MemoryStore: Block rdd_55_4 stored as values in memory (estimated size 16.0 B, free 365.9 MiB) 20/08/08 15:33:54.190 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_55_4 for BlockManagerId(driver, 192.168.1.225, 38159, None) 20/08/08 15:33:54.190 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_55_4 in memory on 192.168.1.225:38159 (size: 16.0 B, free: 366.2 MiB) 20/08/08 15:33:54.191 Executor task launch worker for task 68 DEBUG BlockManager: Getting local block rdd_55_5 20/08/08 15:33:54.191 Executor task launch worker for task 68 DEBUG BlockManager: Block rdd_55_5 was not found 20/08/08 15:33:54.191 Executor task launch worker for task 68 DEBUG BlockManager: Getting remote block rdd_55_5 20/08/08 15:33:54.192 Executor task launch worker for task 64 DEBUG BlockManagerMaster: Updated info of block rdd_55_1 20/08/08 15:33:54.192 Executor task launch worker for task 64 DEBUG BlockManager: Told master about block rdd_55_1 20/08/08 15:33:54.192 Executor task launch worker for task 64 DEBUG BlockManager: Put block rdd_55_1 locally took 32 ms 20/08/08 15:33:54.192 Executor task launch worker for task 64 DEBUG BlockManager: Putting block rdd_55_1 without replication took 32 ms 20/08/08 15:33:54.192 Executor task launch worker for task 64 DEBUG BlockManager: Getting local block rdd_55_1 20/08/08 15:33:54.192 Executor task launch worker for task 64 DEBUG BlockManager: Level for block rdd_55_1 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:54.193 Executor task launch worker for task 64 INFO Executor: Finished task 1.0 in stage 17.0 (TID 64). 2256 bytes result sent to driver 20/08/08 15:33:54.193 Executor task launch worker for task 68 DEBUG BlockManager: Block rdd_55_5 is unknown by block manager master 20/08/08 15:33:54.193 Executor task launch worker for task 66 INFO Executor: Finished task 3.0 in stage 17.0 (TID 66). 2213 bytes result sent to driver 20/08/08 15:33:54.193 Executor task launch worker for task 64 DEBUG ExecutorMetricsPoller: stageTCMP: (17, 0) -> 3 20/08/08 15:33:54.193 Executor task launch worker for task 68 DEBUG BlockManager: Getting local block rdd_24_5 20/08/08 15:33:54.194 Executor task launch worker for task 68 DEBUG BlockManager: Level for block rdd_24_5 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:54.194 Executor task launch worker for task 68 INFO BlockManager: Found block rdd_24_5 locally 20/08/08 15:33:54.194 Executor task launch worker for task 68 DEBUG BlockManager: Getting local block rdd_24_5 20/08/08 15:33:54.194 dispatcher-event-loop-1 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_17.0, runningTasks: 3 20/08/08 15:33:54.194 Executor task launch worker for task 68 DEBUG BlockManager: Level for block rdd_24_5 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:54.194 Executor task launch worker for task 68 INFO BlockManager: Found block rdd_24_5 locally 20/08/08 15:33:54.194 Executor task launch worker for task 68 DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 4 20/08/08 15:33:54.194 Executor task launch worker for task 68 DEBUG MapOutputTrackerMaster: Convert map statuses for shuffle 4, mappers 0-10, partitions 5-6 20/08/08 15:33:54.194 dispatcher-event-loop-1 INFO TaskSetManager: Starting task 6.0 in stage 17.0 (TID 69, 192.168.1.225, executor driver, partition 6, PROCESS_LOCAL, 7433 bytes) taskResourceAssignments Map() 20/08/08 15:33:54.194 Executor task launch worker for task 68 DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 20/08/08 15:33:54.195 dispatcher-event-loop-1 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_17.0, runningTasks: 3 20/08/08 15:33:54.195 Executor task launch worker for task 68 INFO ShuffleBlockFetcherIterator: Getting 3 (684.0 B) non-empty blocks including 3 (684.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 20/08/08 15:33:54.195 Executor task launch worker for task 68 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 20/08/08 15:33:54.195 dispatcher-event-loop-1 INFO TaskSetManager: Starting task 7.0 in stage 17.0 (TID 70, 192.168.1.225, executor driver, partition 7, PROCESS_LOCAL, 7433 bytes) taskResourceAssignments Map() 20/08/08 15:33:54.195 Executor task launch worker for task 68 DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_4_33_5,0), (shuffle_4_34_5,1), (shuffle_4_35_5,2) 20/08/08 15:33:54.195 Executor task launch worker for task 66 DEBUG ExecutorMetricsPoller: stageTCMP: (17, 0) -> 2 20/08/08 15:33:54.195 Executor task launch worker for task 68 INFO BlockManager: Getting local shuffle block shuffle_4_33_5 20/08/08 15:33:54.195 Executor task launch worker for task 70 INFO Executor: Running task 7.0 in stage 17.0 (TID 70) 20/08/08 15:33:54.196 task-result-getter-2 INFO TaskSetManager: Finished task 1.0 in stage 17.0 (TID 64) in 52 ms on 192.168.1.225 (executor driver) (3/10) 20/08/08 15:33:54.196 Executor task launch worker for task 70 DEBUG ExecutorMetricsPoller: stageTCMP: (17, 0) -> 3 20/08/08 15:33:54.196 task-result-getter-2 INFO TaskSetManager: Finished task 3.0 in stage 17.0 (TID 66) in 51 ms on 192.168.1.225 (executor driver) (4/10) 20/08/08 15:33:54.202 Thread-4 DEBUG CountVectorizerModel: Input schema: {"type":"struct","fields":[{"name":"id","type":"long","nullable":true,"metadata":{}},{"name":"words","type":{"type":"array","elementType":"string","containsNull":true},"nullable":true,"metadata":{}},{"name":"expected","type":{"type":"udt","class":"org.apache.spark.ml.linalg.VectorUDT","pyClass":"pyspark.ml.linalg.VectorUDT","sqlType":{"type":"struct","fields":[{"name":"type","type":"byte","nullable":false,"metadata":{}},{"name":"size","type":"integer","nullable":true,"metadata":{}},{"name":"indices","type":{"type":"array","elementType":"integer","containsNull":false},"nullable":true,"metadata":{}},{"name":"values","type":{"type":"array","elementType":"double","containsNull":false},"nullable":true,"metadata":{}}]}},"nullable":true,"metadata":{}}]} 20/08/08 15:33:54.203 Thread-4 DEBUG CountVectorizerModel: Expected output schema: {"type":"struct","fields":[{"name":"id","type":"long","nullable":true,"metadata":{}},{"name":"words","type":{"type":"array","elementType":"string","containsNull":true},"nullable":true,"metadata":{}},{"name":"expected","type":{"type":"udt","class":"org.apache.spark.ml.linalg.VectorUDT","pyClass":"pyspark.ml.linalg.VectorUDT","sqlType":{"type":"struct","fields":[{"name":"type","type":"byte","nullable":false,"metadata":{}},{"name":"size","type":"integer","nullable":true,"metadata":{}},{"name":"indices","type":{"type":"array","elementType":"integer","containsNull":false},"nullable":true,"metadata":{}},{"name":"values","type":{"type":"array","elementType":"double","containsNull":false},"nullable":true,"metadata":{}}]}},"nullable":true,"metadata":{}},{"name":"features","type":{"type":"udt","class":"org.apache.spark.ml.linalg.VectorUDT","pyClass":"pyspark.ml.linalg.VectorUDT","sqlType":{"type":"struct","fields":[{"name":"type","type":"byte","nullable":false,"metadata":{}},{"name":"size","type":"integer","nullable":true,"metadata":{}},{"name":"indices","type":{"type":"array","elementType":"integer","containsNull":false},"nullable":true,"metadata":{}},{"name":"values","type":{"type":"array","elementType":"double","containsNull":false},"nullable":true,"metadata":{}}]}},"nullable":false,"metadata":{"ml_attr":{"attrs":{},"num_attrs":3}}}]} 20/08/08 15:33:54.203 Executor task launch worker for task 70 DEBUG BlockManager: Getting local block rdd_55_7 20/08/08 15:33:54.203 Executor task launch worker for task 70 DEBUG BlockManager: Block rdd_55_7 was not found 20/08/08 15:33:54.203 Executor task launch worker for task 70 DEBUG BlockManager: Getting remote block rdd_55_7 20/08/08 15:33:54.204 Executor task launch worker for task 70 DEBUG BlockManager: Block rdd_55_7 is unknown by block manager master 20/08/08 15:33:54.204 Executor task launch worker for task 70 DEBUG BlockManager: Getting local block rdd_24_7 20/08/08 15:33:54.204 Executor task launch worker for task 70 DEBUG BlockManager: Level for block rdd_24_7 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:54.204 Executor task launch worker for task 70 INFO BlockManager: Found block rdd_24_7 locally 20/08/08 15:33:54.204 Executor task launch worker for task 70 DEBUG BlockManager: Getting local block rdd_24_7 20/08/08 15:33:54.204 Executor task launch worker for task 70 DEBUG BlockManager: Level for block rdd_24_7 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:54.204 Executor task launch worker for task 70 INFO BlockManager: Found block rdd_24_7 locally 20/08/08 15:33:54.204 Executor task launch worker for task 70 DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 4 20/08/08 15:33:54.204 Executor task launch worker for task 70 DEBUG MapOutputTrackerMaster: Convert map statuses for shuffle 4, mappers 0-10, partitions 7-8 20/08/08 15:33:54.205 Executor task launch worker for task 70 DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 20/08/08 15:33:54.206 Executor task launch worker for task 70 INFO ShuffleBlockFetcherIterator: Getting 3 (684.0 B) non-empty blocks including 3 (684.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 20/08/08 15:33:54.206 Executor task launch worker for task 70 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 20/08/08 15:33:54.206 Executor task launch worker for task 70 DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_4_33_7,0), (shuffle_4_34_7,1), (shuffle_4_35_7,2) 20/08/08 15:33:54.206 Executor task launch worker for task 70 INFO BlockManager: Getting local shuffle block shuffle_4_33_7 20/08/08 15:33:54.206 Executor task launch worker for task 70 INFO BlockManager: Getting local shuffle block shuffle_4_34_7 20/08/08 15:33:54.206 Executor task launch worker for task 70 INFO BlockManager: Getting local shuffle block shuffle_4_35_7 20/08/08 15:33:54.206 Executor task launch worker for task 70 DEBUG ShuffleBlockFetcherIterator: Got local blocks in 1 ms 20/08/08 15:33:54.208 Executor task launch worker for task 70 DEBUG TaskMemoryManager: Task 70 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@3972315f 20/08/08 15:33:54.208 Executor task launch worker for task 70 DEBUG TaskMemoryManager: Task 70 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@3427b4df 20/08/08 15:33:54.209 Executor task launch worker for task 70 DEBUG TaskMemoryManager: Task 70 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@17269041 20/08/08 15:33:54.209 Executor task launch worker for task 70 INFO MemoryStore: Block rdd_55_7 stored as values in memory (estimated size 16.0 B, free 365.9 MiB) 20/08/08 15:33:54.195 Executor task launch worker for task 69 INFO Executor: Running task 6.0 in stage 17.0 (TID 69) 20/08/08 15:33:54.195 Executor task launch worker for task 67 DEBUG BlockManagerMaster: Updated info of block rdd_55_4 20/08/08 15:33:54.209 Executor task launch worker for task 67 DEBUG BlockManager: Told master about block rdd_55_4 20/08/08 15:33:54.209 Executor task launch worker for task 67 DEBUG BlockManager: Put block rdd_55_4 locally took 24 ms 20/08/08 15:33:54.209 Executor task launch worker for task 68 INFO BlockManager: Getting local shuffle block shuffle_4_34_5 20/08/08 15:33:54.210 Executor task launch worker for task 67 DEBUG BlockManager: Putting block rdd_55_4 without replication took 24 ms 20/08/08 15:33:54.210 Executor task launch worker for task 67 DEBUG BlockManager: Getting local block rdd_55_4 20/08/08 15:33:54.210 Executor task launch worker for task 68 INFO BlockManager: Getting local shuffle block shuffle_4_35_5 20/08/08 15:33:54.210 Executor task launch worker for task 67 DEBUG BlockManager: Level for block rdd_55_4 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:54.210 Executor task launch worker for task 68 DEBUG ShuffleBlockFetcherIterator: Got local blocks in 15 ms 20/08/08 15:33:54.211 Executor task launch worker for task 67 INFO Executor: Finished task 4.0 in stage 17.0 (TID 67). 2213 bytes result sent to driver 20/08/08 15:33:54.211 Executor task launch worker for task 68 DEBUG TaskMemoryManager: Task 68 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@523a7e0d 20/08/08 15:33:54.212 Executor task launch worker for task 68 DEBUG TaskMemoryManager: Task 68 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@1fb9ecdc 20/08/08 15:33:54.212 Executor task launch worker for task 68 DEBUG TaskMemoryManager: Task 68 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@3f4b2e58 20/08/08 15:33:54.212 Executor task launch worker for task 68 INFO MemoryStore: Block rdd_55_5 stored as values in memory (estimated size 16.0 B, free 365.9 MiB) 20/08/08 15:33:54.213 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_55_7 for BlockManagerId(driver, 192.168.1.225, 38159, None) 20/08/08 15:33:54.213 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_55_7 in memory on 192.168.1.225:38159 (size: 16.0 B, free: 366.2 MiB) 20/08/08 15:33:54.213 Executor task launch worker for task 70 DEBUG BlockManagerMaster: Updated info of block rdd_55_7 20/08/08 15:33:54.213 Executor task launch worker for task 70 DEBUG BlockManager: Told master about block rdd_55_7 20/08/08 15:33:54.213 Executor task launch worker for task 70 DEBUG BlockManager: Put block rdd_55_7 locally took 9 ms 20/08/08 15:33:54.213 Executor task launch worker for task 70 DEBUG BlockManager: Putting block rdd_55_7 without replication took 9 ms 20/08/08 15:33:54.213 Executor task launch worker for task 70 DEBUG BlockManager: Getting local block rdd_55_7 20/08/08 15:33:54.213 Executor task launch worker for task 70 DEBUG BlockManager: Level for block rdd_55_7 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:54.214 Executor task launch worker for task 70 INFO Executor: Finished task 7.0 in stage 17.0 (TID 70). 2213 bytes result sent to driver 20/08/08 15:33:54.215 dispatcher-event-loop-3 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_17.0, runningTasks: 3 20/08/08 15:33:54.215 dispatcher-event-loop-3 INFO TaskSetManager: Starting task 8.0 in stage 17.0 (TID 71, 192.168.1.225, executor driver, partition 8, PROCESS_LOCAL, 7433 bytes) taskResourceAssignments Map() 20/08/08 15:33:54.216 Executor task launch worker for task 71 INFO Executor: Running task 8.0 in stage 17.0 (TID 71) 20/08/08 15:33:54.216 Executor task launch worker for task 70 DEBUG ExecutorMetricsPoller: stageTCMP: (17, 0) -> 2 20/08/08 15:33:54.216 task-result-getter-1 INFO TaskSetManager: Finished task 7.0 in stage 17.0 (TID 70) in 21 ms on 192.168.1.225 (executor driver) (5/10) 20/08/08 15:33:54.216 Thread-4 INFO MemoryStore: Block broadcast_7 stored as values in memory (estimated size 256.0 B, free 366.2 MiB) 20/08/08 15:33:54.216 Thread-4 DEBUG BlockManager: Put block broadcast_7 locally took 13 ms 20/08/08 15:33:54.217 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_55_5 for BlockManagerId(driver, 192.168.1.225, 38159, None) 20/08/08 15:33:54.217 Thread-4 DEBUG BlockManager: Putting block broadcast_7 without replication took 13 ms 20/08/08 15:33:54.217 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_55_5 in memory on 192.168.1.225:38159 (size: 16.0 B, free: 366.2 MiB) 20/08/08 15:33:54.217 Executor task launch worker for task 67 DEBUG ExecutorMetricsPoller: stageTCMP: (17, 0) -> 1 20/08/08 15:33:54.218 dispatcher-event-loop-1 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_17.0, runningTasks: 3 20/08/08 15:33:54.218 Executor task launch worker for task 69 DEBUG ExecutorMetricsPoller: stageTCMP: (17, 0) -> 2 20/08/08 15:33:54.218 Executor task launch worker for task 71 DEBUG ExecutorMetricsPoller: stageTCMP: (17, 0) -> 3 20/08/08 15:33:54.218 dispatcher-event-loop-1 INFO TaskSetManager: Starting task 9.0 in stage 17.0 (TID 72, 192.168.1.225, executor driver, partition 9, PROCESS_LOCAL, 7433 bytes) taskResourceAssignments Map() 20/08/08 15:33:54.219 task-result-getter-3 INFO TaskSetManager: Finished task 4.0 in stage 17.0 (TID 67) in 48 ms on 192.168.1.225 (executor driver) (6/10) 20/08/08 15:33:54.219 Thread-4 INFO MemoryStore: Block broadcast_7_piece0 stored as bytes in memory (estimated size 264.0 B, free 366.2 MiB) 20/08/08 15:33:54.221 Executor task launch worker for task 72 INFO Executor: Running task 9.0 in stage 17.0 (TID 72) 20/08/08 15:33:54.221 Executor task launch worker for task 72 DEBUG ExecutorMetricsPoller: stageTCMP: (17, 0) -> 4 20/08/08 15:33:54.223 Executor task launch worker for task 71 DEBUG BlockManager: Getting local block rdd_55_8 20/08/08 15:33:54.224 Executor task launch worker for task 71 DEBUG BlockManager: Block rdd_55_8 was not found 20/08/08 15:33:54.224 Executor task launch worker for task 71 DEBUG BlockManager: Getting remote block rdd_55_8 20/08/08 15:33:54.224 Executor task launch worker for task 71 DEBUG BlockManager: Block rdd_55_8 is unknown by block manager master 20/08/08 15:33:54.224 Executor task launch worker for task 71 DEBUG BlockManager: Getting local block rdd_24_8 20/08/08 15:33:54.224 Executor task launch worker for task 71 DEBUG BlockManager: Level for block rdd_24_8 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:54.224 Executor task launch worker for task 71 INFO BlockManager: Found block rdd_24_8 locally 20/08/08 15:33:54.224 Executor task launch worker for task 71 DEBUG BlockManager: Getting local block rdd_24_8 20/08/08 15:33:54.224 Executor task launch worker for task 71 DEBUG BlockManager: Level for block rdd_24_8 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:54.224 Executor task launch worker for task 71 INFO BlockManager: Found block rdd_24_8 locally 20/08/08 15:33:54.224 Executor task launch worker for task 71 DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 4 20/08/08 15:33:54.224 Executor task launch worker for task 71 DEBUG MapOutputTrackerMaster: Convert map statuses for shuffle 4, mappers 0-10, partitions 8-9 20/08/08 15:33:54.225 Executor task launch worker for task 71 DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 20/08/08 15:33:54.225 Executor task launch worker for task 71 INFO ShuffleBlockFetcherIterator: Getting 3 (684.0 B) non-empty blocks including 3 (684.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 20/08/08 15:33:54.225 Executor task launch worker for task 71 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 20/08/08 15:33:54.225 Executor task launch worker for task 71 DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_4_33_8,0), (shuffle_4_34_8,1), (shuffle_4_35_8,2) 20/08/08 15:33:54.225 Executor task launch worker for task 71 INFO BlockManager: Getting local shuffle block shuffle_4_33_8 20/08/08 15:33:54.225 Executor task launch worker for task 71 INFO BlockManager: Getting local shuffle block shuffle_4_34_8 20/08/08 15:33:54.226 Executor task launch worker for task 71 INFO BlockManager: Getting local shuffle block shuffle_4_35_8 20/08/08 15:33:54.226 Executor task launch worker for task 71 DEBUG ShuffleBlockFetcherIterator: Got local blocks in 1 ms 20/08/08 15:33:54.226 Executor task launch worker for task 72 DEBUG BlockManager: Getting local block rdd_55_9 20/08/08 15:33:54.226 Executor task launch worker for task 72 DEBUG BlockManager: Block rdd_55_9 was not found 20/08/08 15:33:54.226 Executor task launch worker for task 72 DEBUG BlockManager: Getting remote block rdd_55_9 20/08/08 15:33:54.227 Executor task launch worker for task 72 DEBUG BlockManager: Block rdd_55_9 is unknown by block manager master 20/08/08 15:33:54.227 Executor task launch worker for task 72 DEBUG BlockManager: Getting local block rdd_24_9 20/08/08 15:33:54.227 Executor task launch worker for task 72 DEBUG BlockManager: Level for block rdd_24_9 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:54.227 Executor task launch worker for task 72 INFO BlockManager: Found block rdd_24_9 locally 20/08/08 15:33:54.227 Executor task launch worker for task 72 DEBUG BlockManager: Getting local block rdd_24_9 20/08/08 15:33:54.227 Executor task launch worker for task 72 DEBUG BlockManager: Level for block rdd_24_9 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:54.227 Executor task launch worker for task 72 INFO BlockManager: Found block rdd_24_9 locally 20/08/08 15:33:54.227 Executor task launch worker for task 72 DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 4 20/08/08 15:33:54.227 Executor task launch worker for task 72 DEBUG MapOutputTrackerMaster: Convert map statuses for shuffle 4, mappers 0-10, partitions 9-10 20/08/08 15:33:54.227 Executor task launch worker for task 72 DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 20/08/08 15:33:54.227 Executor task launch worker for task 71 DEBUG TaskMemoryManager: Task 71 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@5d820aa6 20/08/08 15:33:54.228 Executor task launch worker for task 72 INFO ShuffleBlockFetcherIterator: Getting 3 (684.0 B) non-empty blocks including 3 (684.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 20/08/08 15:33:54.228 Executor task launch worker for task 72 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 20/08/08 15:33:54.228 Executor task launch worker for task 71 DEBUG TaskMemoryManager: Task 71 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@731418c5 20/08/08 15:33:54.228 Executor task launch worker for task 71 DEBUG TaskMemoryManager: Task 71 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@5ca5b9f7 20/08/08 15:33:54.228 Executor task launch worker for task 72 DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_4_33_9,0), (shuffle_4_34_9,1), (shuffle_4_35_9,2) 20/08/08 15:33:54.228 Executor task launch worker for task 72 INFO BlockManager: Getting local shuffle block shuffle_4_33_9 20/08/08 15:33:54.228 Executor task launch worker for task 72 INFO BlockManager: Getting local shuffle block shuffle_4_34_9 20/08/08 15:33:54.220 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master broadcast_7_piece0 for BlockManagerId(driver, 192.168.1.225, 36433, None) 20/08/08 15:33:54.228 Executor task launch worker for task 71 INFO MemoryStore: Block rdd_55_8 stored as values in memory (estimated size 16.0 B, free 365.9 MiB) 20/08/08 15:33:54.228 Executor task launch worker for task 72 INFO BlockManager: Getting local shuffle block shuffle_4_35_9 20/08/08 15:33:54.228 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_55_8 for BlockManagerId(driver, 192.168.1.225, 38159, None) 20/08/08 15:33:54.229 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added broadcast_7_piece0 in memory on 192.168.1.225:36433 (size: 264.0 B, free: 366.3 MiB) 20/08/08 15:33:54.229 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_55_8 in memory on 192.168.1.225:38159 (size: 16.0 B, free: 366.2 MiB) 20/08/08 15:33:54.229 Executor task launch worker for task 71 DEBUG BlockManagerMaster: Updated info of block rdd_55_8 20/08/08 15:33:54.229 Executor task launch worker for task 71 DEBUG BlockManager: Told master about block rdd_55_8 20/08/08 15:33:54.229 Executor task launch worker for task 71 DEBUG BlockManager: Put block rdd_55_8 locally took 5 ms 20/08/08 15:33:54.229 Executor task launch worker for task 71 DEBUG BlockManager: Putting block rdd_55_8 without replication took 5 ms 20/08/08 15:33:54.229 Executor task launch worker for task 71 DEBUG BlockManager: Getting local block rdd_55_8 20/08/08 15:33:54.229 Executor task launch worker for task 71 DEBUG BlockManager: Level for block rdd_55_8 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:54.230 Thread-4 DEBUG BlockManagerMaster: Updated info of block broadcast_7_piece0 20/08/08 15:33:54.230 Thread-4 DEBUG BlockManager: Told master about block broadcast_7_piece0 20/08/08 15:33:54.230 Thread-4 DEBUG BlockManager: Put block broadcast_7_piece0 locally took 10 ms 20/08/08 15:33:54.230 Thread-4 DEBUG BlockManager: Putting block broadcast_7_piece0 without replication took 11 ms 20/08/08 15:33:54.230 Executor task launch worker for task 71 INFO Executor: Finished task 8.0 in stage 17.0 (TID 71). 2213 bytes result sent to driver 20/08/08 15:33:54.231 dispatcher-event-loop-0 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_17.0, runningTasks: 3 20/08/08 15:33:54.230 Thread-4 INFO SparkContext: Created broadcast 7 from broadcast at CountVectorizer.scala:306 20/08/08 15:33:54.231 dispatcher-event-loop-0 DEBUG TaskSetManager: No tasks for locality level PROCESS_LOCAL, so moving to locality level NODE_LOCAL 20/08/08 15:33:54.231 dispatcher-event-loop-0 DEBUG TaskSetManager: No tasks for locality level NODE_LOCAL, so moving to locality level ANY 20/08/08 15:33:54.231 Executor task launch worker for task 10 DEBUG GenerateSafeProjection: code for newInstance(class org.apache.spark.ml.linalg.VectorUDT).deserialize: /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificSafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificSafeProjection extends org.apache.spark.sql.catalyst.expressions.codegen.BaseProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private InternalRow mutableRow; /* 009 */ private boolean resultIsNull_0; /* 010 */ private InternalRow[] mutableStateArray_0 = new InternalRow[1]; /* 011 */ /* 012 */ public SpecificSafeProjection(Object[] references) { /* 013 */ this.references = references; /* 014 */ mutableRow = (InternalRow) references[references.length - 1]; /* 015 */ /* 016 */ /* 017 */ } /* 018 */ /* 019 */ public void initialize(int partitionIndex) { /* 020 */ /* 021 */ } /* 022 */ /* 023 */ public java.lang.Object apply(java.lang.Object _i) { /* 024 */ InternalRow i = (InternalRow) _i; /* 025 */ final org.apache.spark.ml.linalg.VectorUDT value_1 = false ? /* 026 */ null : new org.apache.spark.ml.linalg.VectorUDT(); /* 027 */ boolean isNull_0 = true; /* 028 */ org.apache.spark.ml.linalg.Vector value_0 = null; /* 029 */ if (!false) { /* 030 */ resultIsNull_0 = false; /* 031 */ if (!resultIsNull_0) { /* 032 */ boolean isNull_2 = i.isNullAt(0); /* 033 */ InternalRow value_2 = isNull_2 ? /* 034 */ null : (i.getStruct(0, 4)); /* 035 */ resultIsNull_0 = isNull_2; /* 036 */ mutableStateArray_0[0] = value_2; /* 037 */ } /* 038 */ /* 039 */ isNull_0 = resultIsNull_0; /* 040 */ if (!isNull_0) { /* 041 */ /* 042 */ Object funcResult_0 = null; /* 043 */ funcResult_0 = value_1.deserialize(mutableStateArray_0[0]); /* 044 */ /* 045 */ if (funcResult_0 != null) { /* 046 */ value_0 = (org.apache.spark.ml.linalg.Vector) funcResult_0; /* 047 */ } else { /* 048 */ isNull_0 = true; /* 049 */ } /* 050 */ /* 051 */ /* 052 */ } /* 053 */ } /* 054 */ if (isNull_0) { /* 055 */ mutableRow.setNullAt(0); /* 056 */ } else { /* 057 */ /* 058 */ mutableRow.update(0, value_0); /* 059 */ } /* 060 */ /* 061 */ return mutableRow; /* 062 */ } /* 063 */ /* 064 */ /* 065 */ } 20/08/08 15:33:54.231 task-result-getter-2 INFO TaskSetManager: Finished task 8.0 in stage 17.0 (TID 71) in 16 ms on 192.168.1.225 (executor driver) (7/10) 20/08/08 15:33:54.232 Executor task launch worker for task 71 DEBUG ExecutorMetricsPoller: stageTCMP: (17, 0) -> 3 20/08/08 15:33:54.228 Executor task launch worker for task 72 DEBUG ShuffleBlockFetcherIterator: Got local blocks in 1 ms 20/08/08 15:33:54.229 block-manager-storage-async-thread-pool-2 DEBUG BlockManagerStorageEndpoint: Done removing RDD 21, response is 0 20/08/08 15:33:54.234 Executor task launch worker for task 72 DEBUG TaskMemoryManager: Task 72 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@24e78766 20/08/08 15:33:54.235 Executor task launch worker for task 72 DEBUG TaskMemoryManager: Task 72 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@77198253 20/08/08 15:33:54.235 Executor task launch worker for task 72 DEBUG TaskMemoryManager: Task 72 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@7f9ea82f 20/08/08 15:33:54.235 Executor task launch worker for task 72 INFO MemoryStore: Block rdd_55_9 stored as values in memory (estimated size 16.0 B, free 365.9 MiB) 20/08/08 15:33:54.236 Executor task launch worker for task 12 DEBUG GenerateSafeProjection: code for newInstance(class org.apache.spark.ml.linalg.VectorUDT).deserialize: /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificSafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificSafeProjection extends org.apache.spark.sql.catalyst.expressions.codegen.BaseProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private InternalRow mutableRow; /* 009 */ private boolean resultIsNull_0; /* 010 */ private InternalRow[] mutableStateArray_0 = new InternalRow[1]; /* 011 */ /* 012 */ public SpecificSafeProjection(Object[] references) { /* 013 */ this.references = references; /* 014 */ mutableRow = (InternalRow) references[references.length - 1]; /* 015 */ /* 016 */ /* 017 */ } /* 018 */ /* 019 */ public void initialize(int partitionIndex) { /* 020 */ /* 021 */ } /* 022 */ /* 023 */ public java.lang.Object apply(java.lang.Object _i) { /* 024 */ InternalRow i = (InternalRow) _i; /* 025 */ final org.apache.spark.ml.linalg.VectorUDT value_1 = false ? /* 026 */ null : new org.apache.spark.ml.linalg.VectorUDT(); /* 027 */ boolean isNull_0 = true; /* 028 */ org.apache.spark.ml.linalg.Vector value_0 = null; /* 029 */ if (!false) { /* 030 */ resultIsNull_0 = false; /* 031 */ if (!resultIsNull_0) { /* 032 */ boolean isNull_2 = i.isNullAt(0); /* 033 */ InternalRow value_2 = isNull_2 ? /* 034 */ null : (i.getStruct(0, 4)); /* 035 */ resultIsNull_0 = isNull_2; /* 036 */ mutableStateArray_0[0] = value_2; /* 037 */ } /* 038 */ /* 039 */ isNull_0 = resultIsNull_0; /* 040 */ if (!isNull_0) { /* 041 */ /* 042 */ Object funcResult_0 = null; /* 043 */ funcResult_0 = value_1.deserialize(mutableStateArray_0[0]); /* 044 */ /* 045 */ if (funcResult_0 != null) { /* 046 */ value_0 = (org.apache.spark.ml.linalg.Vector) funcResult_0; /* 047 */ } else { /* 048 */ isNull_0 = true; /* 049 */ } /* 050 */ /* 051 */ /* 052 */ } /* 053 */ } /* 054 */ if (isNull_0) { /* 055 */ mutableRow.setNullAt(0); /* 056 */ } else { /* 057 */ /* 058 */ mutableRow.update(0, value_0); /* 059 */ } /* 060 */ /* 061 */ return mutableRow; /* 062 */ } /* 063 */ /* 064 */ /* 065 */ } 20/08/08 15:33:54.236 Executor task launch worker for task 69 DEBUG BlockManager: Getting local block rdd_55_6 20/08/08 15:33:54.237 Executor task launch worker for task 69 DEBUG BlockManager: Block rdd_55_6 was not found 20/08/08 15:33:54.237 Executor task launch worker for task 69 DEBUG BlockManager: Getting remote block rdd_55_6 20/08/08 15:33:54.239 Executor task launch worker for task 68 DEBUG BlockManagerMaster: Updated info of block rdd_55_5 20/08/08 15:33:54.239 Executor task launch worker for task 68 DEBUG BlockManager: Told master about block rdd_55_5 20/08/08 15:33:54.239 Executor task launch worker for task 68 DEBUG BlockManager: Put block rdd_55_5 locally took 46 ms 20/08/08 15:33:54.239 Executor task launch worker for task 68 DEBUG BlockManager: Putting block rdd_55_5 without replication took 46 ms 20/08/08 15:33:54.239 Executor task launch worker for task 68 DEBUG BlockManager: Getting local block rdd_55_5 20/08/08 15:33:54.239 Executor task launch worker for task 68 DEBUG BlockManager: Level for block rdd_55_5 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:54.240 Executor task launch worker for task 68 INFO Executor: Finished task 5.0 in stage 17.0 (TID 68). 2213 bytes result sent to driver 20/08/08 15:33:54.242 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_55_9 for BlockManagerId(driver, 192.168.1.225, 38159, None) 20/08/08 15:33:54.242 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_55_9 in memory on 192.168.1.225:38159 (size: 16.0 B, free: 366.2 MiB) 20/08/08 15:33:54.242 Executor task launch worker for task 72 DEBUG BlockManagerMaster: Updated info of block rdd_55_9 20/08/08 15:33:54.242 Executor task launch worker for task 72 DEBUG BlockManager: Told master about block rdd_55_9 20/08/08 15:33:54.255 block-manager-storage-async-thread-pool-2 DEBUG BlockManagerStorageEndpoint: Sent response: 0 to 192.168.1.225:35519 20/08/08 15:33:54.261 block-manager-storage-async-thread-pool-7 DEBUG BlockManagerStorageEndpoint: removing RDD 16 20/08/08 15:33:54.261 block-manager-storage-async-thread-pool-7 INFO BlockManager: Removing RDD 16 20/08/08 15:33:54.261 Executor task launch worker for task 72 DEBUG BlockManager: Put block rdd_55_9 locally took 34 ms 20/08/08 15:33:54.261 block-manager-storage-async-thread-pool-7 DEBUG BlockManager: Removing block rdd_16_0 20/08/08 15:33:54.261 Executor task launch worker for task 72 DEBUG BlockManager: Putting block rdd_55_9 without replication took 34 ms 20/08/08 15:33:54.261 Executor task launch worker for task 72 DEBUG BlockManager: Getting local block rdd_55_9 20/08/08 15:33:54.261 block-manager-storage-async-thread-pool-7 DEBUG MemoryStore: Block rdd_16_0 of size 176 dropped from memory (free 383623302) 20/08/08 15:33:54.261 Executor task launch worker for task 72 DEBUG BlockManager: Level for block rdd_55_9 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:54.261 block-manager-storage-async-thread-pool-7 DEBUG BlockManager: Removing block rdd_16_2 20/08/08 15:33:54.261 block-manager-storage-async-thread-pool-7 DEBUG MemoryStore: Block rdd_16_2 of size 328 dropped from memory (free 383623630) 20/08/08 15:33:54.262 block-manager-storage-async-thread-pool-7 DEBUG BlockManager: Removing block rdd_16_3 20/08/08 15:33:54.262 block-manager-storage-async-thread-pool-7 DEBUG MemoryStore: Block rdd_16_3 of size 328 dropped from memory (free 383623958) 20/08/08 15:33:54.262 block-manager-storage-async-thread-pool-7 DEBUG BlockManager: Removing block rdd_16_1 20/08/08 15:33:54.262 block-manager-storage-async-thread-pool-7 DEBUG MemoryStore: Block rdd_16_1 of size 176 dropped from memory (free 383624134) 20/08/08 15:33:54.262 Executor task launch worker for task 72 INFO Executor: Finished task 9.0 in stage 17.0 (TID 72). 2213 bytes result sent to driver 20/08/08 15:33:54.237 Executor task launch worker for task 10 DEBUG GenerateUnsafeProjection: code for newInstance(class org.apache.spark.ml.linalg.VectorUDT).serialize: /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificUnsafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificUnsafeProjection extends org.apache.spark.sql.catalyst.expressions.UnsafeProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private boolean resultIsNull_0; /* 009 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[] mutableStateArray_1 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[2]; /* 010 */ private org.apache.spark.ml.linalg.Vector[] mutableStateArray_0 = new org.apache.spark.ml.linalg.Vector[1]; /* 011 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[] mutableStateArray_2 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[2]; /* 012 */ /* 013 */ public SpecificUnsafeProjection(Object[] references) { /* 014 */ this.references = references; /* 015 */ /* 016 */ mutableStateArray_1[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(1, 32); /* 017 */ mutableStateArray_1[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(mutableStateArray_1[0], 4); /* 018 */ mutableStateArray_2[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(mutableStateArray_1[1], 4); /* 019 */ mutableStateArray_2[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(mutableStateArray_1[1], 8); /* 020 */ /* 021 */ } /* 022 */ /* 023 */ public void initialize(int partitionIndex) { /* 024 */ /* 025 */ } /* 026 */ /* 027 */ // Scala.Function1 need this /* 028 */ public java.lang.Object apply(java.lang.Object row) { /* 029 */ return apply((InternalRow) row); /* 030 */ } /* 031 */ /* 032 */ public UnsafeRow apply(InternalRow i) { /* 033 */ mutableStateArray_1[0].reset(); /* 034 */ /* 035 */ /* 036 */ mutableStateArray_1[0].zeroOutNullBytes(); /* 037 */ /* 038 */ final org.apache.spark.ml.linalg.VectorUDT value_1 = false ? /* 039 */ null : new org.apache.spark.ml.linalg.VectorUDT(); /* 040 */ boolean isNull_0 = true; /* 041 */ InternalRow value_0 = null; /* 042 */ if (!false) { /* 043 */ resultIsNull_0 = false; /* 044 */ if (!resultIsNull_0) { /* 045 */ boolean isNull_2 = i.isNullAt(0); /* 046 */ org.apache.spark.ml.linalg.Vector value_2 = isNull_2 ? /* 047 */ null : ((org.apache.spark.ml.linalg.Vector)i.get(0, null)); /* 048 */ resultIsNull_0 = isNull_2; /* 049 */ mutableStateArray_0[0] = value_2; /* 050 */ } /* 051 */ /* 052 */ isNull_0 = resultIsNull_0; /* 053 */ if (!isNull_0) { /* 054 */ /* 055 */ Object funcResult_0 = null; /* 056 */ funcResult_0 = value_1.serialize(mutableStateArray_0[0]); /* 057 */ /* 058 */ if (funcResult_0 != null) { /* 059 */ value_0 = (InternalRow) funcResult_0; /* 060 */ } else { /* 061 */ isNull_0 = true; /* 062 */ } /* 063 */ /* 064 */ /* 065 */ } /* 066 */ } /* 067 */ if (isNull_0) { /* 068 */ mutableStateArray_1[0].setNullAt(0); /* 069 */ } else { /* 070 */ final InternalRow tmpInput_0 = value_0; /* 071 */ if (tmpInput_0 instanceof UnsafeRow) { /* 072 */ mutableStateArray_1[0].write(0, (UnsafeRow) tmpInput_0); /* 073 */ } else { /* 074 */ // Remember the current cursor so that we can calculate how many bytes are /* 075 */ // written later. /* 076 */ final int previousCursor_0 = mutableStateArray_1[0].cursor(); /* 077 */ /* 078 */ mutableStateArray_1[1].resetRowWriter(); /* 079 */ /* 080 */ /* 081 */ mutableStateArray_1[1].write(0, (tmpInput_0.getByte(0))); /* 082 */ /* 083 */ /* 084 */ if ((tmpInput_0.isNullAt(1))) { /* 085 */ mutableStateArray_1[1].setNullAt(1); /* 086 */ } else { /* 087 */ mutableStateArray_1[1].write(1, (tmpInput_0.getInt(1))); /* 088 */ } /* 089 */ /* 090 */ /* 091 */ if ((tmpInput_0.isNullAt(2))) { /* 092 */ mutableStateArray_1[1].setNullAt(2); /* 093 */ } else { /* 094 */ // Remember the current cursor so that we can calculate how many bytes are /* 095 */ // written later. /* 096 */ final int previousCursor_1 = mutableStateArray_1[1].cursor(); /* 097 */ /* 098 */ final ArrayData tmpInput_1 = (tmpInput_0.getArray(2)); /* 099 */ if (tmpInput_1 instanceof UnsafeArrayData) { /* 100 */ mutableStateArray_1[1].write((UnsafeArrayData) tmpInput_1); /* 101 */ } else { /* 102 */ final int numElements_0 = tmpInput_1.numElements(); /* 103 */ mutableStateArray_2[0].initialize(numElements_0); /* 104 */ /* 105 */ for (int index_0 = 0; index_0 < numElements_0; index_0++) { /* 106 */ mutableStateArray_2[0].write(index_0, tmpInput_1.getInt(index_0)); /* 107 */ } /* 108 */ } /* 109 */ /* 110 */ mutableStateArray_1[1].setOffsetAndSizeFromPreviousCursor(2, previousCursor_1); /* 111 */ } /* 112 */ /* 113 */ /* 114 */ if ((tmpInput_0.isNullAt(3))) { /* 115 */ mutableStateArray_1[1].setNullAt(3); /* 116 */ } else { /* 117 */ // Remember the current cursor so that we can calculate how many bytes are /* 118 */ // written later. /* 119 */ final int previousCursor_2 = mutableStateArray_1[1].cursor(); /* 120 */ /* 121 */ final ArrayData tmpInput_2 = (tmpInput_0.getArray(3)); /* 122 */ if (tmpInput_2 instanceof UnsafeArrayData) { /* 123 */ mutableStateArray_1[1].write((UnsafeArrayData) tmpInput_2); /* 124 */ } else { /* 125 */ final int numElements_1 = tmpInput_2.numElements(); /* 126 */ mutableStateArray_2[1].initialize(numElements_1); /* 127 */ /* 128 */ for (int index_1 = 0; index_1 < numElements_1; index_1++) { /* 129 */ mutableStateArray_2[1].write(index_1, tmpInput_2.getDouble(index_1)); /* 130 */ } /* 131 */ } /* 132 */ /* 133 */ mutableStateArray_1[1].setOffsetAndSizeFromPreviousCursor(3, previousCursor_2); /* 134 */ } /* 135 */ /* 136 */ /* 137 */ mutableStateArray_1[0].setOffsetAndSizeFromPreviousCursor(0, previousCursor_0); /* 138 */ } /* 139 */ } /* 140 */ return (mutableStateArray_1[0].getRow()); /* 141 */ } /* 142 */ /* 143 */ /* 144 */ } 20/08/08 15:33:54.266 block-manager-storage-async-thread-pool-12 DEBUG BlockManagerStorageEndpoint: Done removing RDD 20, response is 0 20/08/08 15:33:54.271 block-manager-storage-async-thread-pool-14 DEBUG BlockManagerStorageEndpoint: Done removing RDD 16, response is 0 20/08/08 15:33:54.271 Executor task launch worker for task 68 DEBUG ExecutorMetricsPoller: stageTCMP: (17, 0) -> 2 20/08/08 15:33:54.271 block-manager-storage-async-thread-pool-11 DEBUG BlockManagerStorageEndpoint: Done removing RDD 25, response is 0 20/08/08 15:33:54.272 dispatcher-event-loop-2 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_17.0, runningTasks: 2 20/08/08 15:33:54.272 task-result-getter-0 INFO TaskSetManager: Finished task 5.0 in stage 17.0 (TID 68) in 89 ms on 192.168.1.225 (executor driver) (8/10) 20/08/08 15:33:54.272 Executor task launch worker for task 72 DEBUG ExecutorMetricsPoller: stageTCMP: (17, 0) -> 1 20/08/08 15:33:54.273 Executor task launch worker for task 69 DEBUG BlockManager: Block rdd_55_6 is unknown by block manager master 20/08/08 15:33:54.273 Executor task launch worker for task 69 DEBUG BlockManager: Getting local block rdd_24_6 20/08/08 15:33:54.273 dispatcher-event-loop-1 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_17.0, runningTasks: 1 20/08/08 15:33:54.273 Executor task launch worker for task 69 DEBUG BlockManager: Level for block rdd_24_6 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:54.274 Executor task launch worker for task 69 INFO BlockManager: Found block rdd_24_6 locally 20/08/08 15:33:54.274 task-result-getter-1 INFO TaskSetManager: Finished task 9.0 in stage 17.0 (TID 72) in 56 ms on 192.168.1.225 (executor driver) (9/10) 20/08/08 15:33:54.274 Executor task launch worker for task 69 DEBUG BlockManager: Getting local block rdd_24_6 20/08/08 15:33:54.274 Executor task launch worker for task 69 DEBUG BlockManager: Level for block rdd_24_6 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:54.274 Executor task launch worker for task 69 INFO BlockManager: Found block rdd_24_6 locally 20/08/08 15:33:54.274 Executor task launch worker for task 69 DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 4 20/08/08 15:33:54.274 Executor task launch worker for task 69 DEBUG MapOutputTrackerMaster: Convert map statuses for shuffle 4, mappers 0-10, partitions 6-7 20/08/08 15:33:54.274 Executor task launch worker for task 69 DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 20/08/08 15:33:54.275 Executor task launch worker for task 69 INFO ShuffleBlockFetcherIterator: Getting 3 (684.0 B) non-empty blocks including 3 (684.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 20/08/08 15:33:54.275 Executor task launch worker for task 69 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 20/08/08 15:33:54.275 Executor task launch worker for task 69 DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_4_33_6,0), (shuffle_4_34_6,1), (shuffle_4_35_6,2) 20/08/08 15:33:54.275 Executor task launch worker for task 69 INFO BlockManager: Getting local shuffle block shuffle_4_33_6 20/08/08 15:33:54.275 Executor task launch worker for task 69 INFO BlockManager: Getting local shuffle block shuffle_4_34_6 20/08/08 15:33:54.275 Executor task launch worker for task 69 INFO BlockManager: Getting local shuffle block shuffle_4_35_6 20/08/08 15:33:54.276 Executor task launch worker for task 69 DEBUG ShuffleBlockFetcherIterator: Got local blocks in 1 ms 20/08/08 15:33:54.277 Executor task launch worker for task 69 DEBUG TaskMemoryManager: Task 69 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@38b1f5b5 20/08/08 15:33:54.278 Executor task launch worker for task 69 DEBUG TaskMemoryManager: Task 69 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@39d62aeb 20/08/08 15:33:54.278 Executor task launch worker for task 69 DEBUG TaskMemoryManager: Task 69 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@61661ffd 20/08/08 15:33:54.278 Executor task launch worker for task 69 INFO MemoryStore: Block rdd_55_6 stored as values in memory (estimated size 16.0 B, free 365.9 MiB) 20/08/08 15:33:54.279 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_55_6 for BlockManagerId(driver, 192.168.1.225, 38159, None) 20/08/08 15:33:54.279 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_55_6 in memory on 192.168.1.225:38159 (size: 16.0 B, free: 366.2 MiB) 20/08/08 15:33:54.280 Executor task launch worker for task 69 DEBUG BlockManagerMaster: Updated info of block rdd_55_6 20/08/08 15:33:54.280 Executor task launch worker for task 69 DEBUG BlockManager: Told master about block rdd_55_6 20/08/08 15:33:54.280 Executor task launch worker for task 69 DEBUG BlockManager: Put block rdd_55_6 locally took 6 ms 20/08/08 15:33:54.280 Executor task launch worker for task 69 DEBUG BlockManager: Putting block rdd_55_6 without replication took 6 ms 20/08/08 15:33:54.280 Executor task launch worker for task 69 DEBUG BlockManager: Getting local block rdd_55_6 20/08/08 15:33:54.280 Executor task launch worker for task 69 DEBUG BlockManager: Level for block rdd_55_6 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:54.281 Executor task launch worker for task 69 INFO Executor: Finished task 6.0 in stage 17.0 (TID 69). 2213 bytes result sent to driver 20/08/08 15:33:54.281 Thread-4 DEBUG Analyzer$ResolveReferences: Resolving 'words to words#38 20/08/08 15:33:54.281 dispatcher-event-loop-3 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_17.0, runningTasks: 0 20/08/08 15:33:54.282 Executor task launch worker for task 69 DEBUG ExecutorMetricsPoller: removing (17, 0) from stageTCMP 20/08/08 15:33:54.282 task-result-getter-3 INFO TaskSetManager: Finished task 6.0 in stage 17.0 (TID 69) in 88 ms on 192.168.1.225 (executor driver) (10/10) 20/08/08 15:33:54.282 task-result-getter-3 INFO TaskSchedulerImpl: Removed TaskSet 17.0, whose tasks have all completed, from pool 20/08/08 15:33:54.283 dag-scheduler-event-loop INFO DAGScheduler: ResultStage 17 (count at ALS.scala:1067) finished in 0.223 s 20/08/08 15:33:54.283 dag-scheduler-event-loop DEBUG DAGScheduler: After removal of stage 17, remaining stages = 4 20/08/08 15:33:54.283 dag-scheduler-event-loop DEBUG DAGScheduler: After removal of stage 14, remaining stages = 3 20/08/08 15:33:54.283 dag-scheduler-event-loop DEBUG DAGScheduler: After removal of stage 13, remaining stages = 2 20/08/08 15:33:54.283 dag-scheduler-event-loop DEBUG DAGScheduler: After removal of stage 16, remaining stages = 1 20/08/08 15:33:54.283 dag-scheduler-event-loop DEBUG DAGScheduler: After removal of stage 15, remaining stages = 0 20/08/08 15:33:54.283 dag-scheduler-event-loop INFO DAGScheduler: Job 4 is finished. Cancelling potential speculative or zombie tasks for this job 20/08/08 15:33:54.283 dag-scheduler-event-loop INFO TaskSchedulerImpl: Killing all running tasks in stage 17: Stage finished 20/08/08 15:33:54.283 Thread-4 INFO DAGScheduler: Job 4 finished: count at ALS.scala:1067, took 0.236309 s 20/08/08 15:33:54.284 Thread-4 INFO MapPartitionsRDD: Removing RDD 36 from persistence list 20/08/08 15:33:54.285 Thread-4 INFO MapPartitionsRDD: Removing RDD 24 from persistence list 20/08/08 15:33:54.242 Executor task launch worker for task 12 DEBUG GenerateUnsafeProjection: code for newInstance(class org.apache.spark.ml.linalg.VectorUDT).serialize: /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificUnsafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificUnsafeProjection extends org.apache.spark.sql.catalyst.expressions.UnsafeProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private boolean resultIsNull_0; /* 009 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[] mutableStateArray_1 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[2]; /* 010 */ private org.apache.spark.ml.linalg.Vector[] mutableStateArray_0 = new org.apache.spark.ml.linalg.Vector[1]; /* 011 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[] mutableStateArray_2 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[2]; /* 012 */ /* 013 */ public SpecificUnsafeProjection(Object[] references) { /* 014 */ this.references = references; /* 015 */ /* 016 */ mutableStateArray_1[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(1, 32); /* 017 */ mutableStateArray_1[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(mutableStateArray_1[0], 4); /* 018 */ mutableStateArray_2[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(mutableStateArray_1[1], 4); /* 019 */ mutableStateArray_2[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(mutableStateArray_1[1], 8); /* 020 */ /* 021 */ } /* 022 */ /* 023 */ public void initialize(int partitionIndex) { /* 024 */ /* 025 */ } /* 026 */ /* 027 */ // Scala.Function1 need this /* 028 */ public java.lang.Object apply(java.lang.Object row) { /* 029 */ return apply((InternalRow) row); /* 030 */ } /* 031 */ /* 032 */ public UnsafeRow apply(InternalRow i) { /* 033 */ mutableStateArray_1[0].reset(); /* 034 */ /* 035 */ /* 036 */ mutableStateArray_1[0].zeroOutNullBytes(); /* 037 */ /* 038 */ final org.apache.spark.ml.linalg.VectorUDT value_1 = false ? /* 039 */ null : new org.apache.spark.ml.linalg.VectorUDT(); /* 040 */ boolean isNull_0 = true; /* 041 */ InternalRow value_0 = null; /* 042 */ if (!false) { /* 043 */ resultIsNull_0 = false; /* 044 */ if (!resultIsNull_0) { /* 045 */ boolean isNull_2 = i.isNullAt(0); /* 046 */ org.apache.spark.ml.linalg.Vector value_2 = isNull_2 ? /* 047 */ null : ((org.apache.spark.ml.linalg.Vector)i.get(0, null)); /* 048 */ resultIsNull_0 = isNull_2; /* 049 */ mutableStateArray_0[0] = value_2; /* 050 */ } /* 051 */ /* 052 */ isNull_0 = resultIsNull_0; /* 053 */ if (!isNull_0) { /* 054 */ /* 055 */ Object funcResult_0 = null; /* 056 */ funcResult_0 = value_1.serialize(mutableStateArray_0[0]); /* 057 */ /* 058 */ if (funcResult_0 != null) { /* 059 */ value_0 = (InternalRow) funcResult_0; /* 060 */ } else { /* 061 */ isNull_0 = true; /* 062 */ } /* 063 */ /* 064 */ /* 065 */ } /* 066 */ } /* 067 */ if (isNull_0) { /* 068 */ mutableStateArray_1[0].setNullAt(0); /* 069 */ } else { /* 070 */ final InternalRow tmpInput_0 = value_0; /* 071 */ if (tmpInput_0 instanceof UnsafeRow) { /* 072 */ mutableStateArray_1[0].write(0, (UnsafeRow) tmpInput_0); /* 073 */ } else { /* 074 */ // Remember the current cursor so that we can calculate how many bytes are /* 075 */ // written later. /* 076 */ final int previousCursor_0 = mutableStateArray_1[0].cursor(); /* 077 */ /* 078 */ mutableStateArray_1[1].resetRowWriter(); /* 079 */ /* 080 */ /* 081 */ mutableStateArray_1[1].write(0, (tmpInput_0.getByte(0))); /* 082 */ /* 083 */ /* 084 */ if ((tmpInput_0.isNullAt(1))) { /* 085 */ mutableStateArray_1[1].setNullAt(1); /* 086 */ } else { /* 087 */ mutableStateArray_1[1].write(1, (tmpInput_0.getInt(1))); /* 088 */ } /* 089 */ /* 090 */ /* 091 */ if ((tmpInput_0.isNullAt(2))) { /* 092 */ mutableStateArray_1[1].setNullAt(2); /* 093 */ } else { /* 094 */ // Remember the current cursor so that we can calculate how many bytes are /* 095 */ // written later. /* 096 */ final int previousCursor_1 = mutableStateArray_1[1].cursor(); /* 097 */ /* 098 */ final ArrayData tmpInput_1 = (tmpInput_0.getArray(2)); /* 099 */ if (tmpInput_1 instanceof UnsafeArrayData) { /* 100 */ mutableStateArray_1[1].write((UnsafeArrayData) tmpInput_1); /* 101 */ } else { /* 102 */ final int numElements_0 = tmpInput_1.numElements(); /* 103 */ mutableStateArray_2[0].initialize(numElements_0); /* 104 */ /* 105 */ for (int index_0 = 0; index_0 < numElements_0; index_0++) { /* 106 */ mutableStateArray_2[0].write(index_0, tmpInput_1.getInt(index_0)); /* 107 */ } /* 108 */ } /* 109 */ /* 110 */ mutableStateArray_1[1].setOffsetAndSizeFromPreviousCursor(2, previousCursor_1); /* 111 */ } /* 112 */ /* 113 */ /* 114 */ if ((tmpInput_0.isNullAt(3))) { /* 115 */ mutableStateArray_1[1].setNullAt(3); /* 116 */ } else { /* 117 */ // Remember the current cursor so that we can calculate how many bytes are /* 118 */ // written later. /* 119 */ final int previousCursor_2 = mutableStateArray_1[1].cursor(); /* 120 */ /* 121 */ final ArrayData tmpInput_2 = (tmpInput_0.getArray(3)); /* 122 */ if (tmpInput_2 instanceof UnsafeArrayData) { /* 123 */ mutableStateArray_1[1].write((UnsafeArrayData) tmpInput_2); /* 124 */ } else { /* 125 */ final int numElements_1 = tmpInput_2.numElements(); /* 126 */ mutableStateArray_2[1].initialize(numElements_1); /* 127 */ /* 128 */ for (int index_1 = 0; index_1 < numElements_1; index_1++) { /* 129 */ mutableStateArray_2[1].write(index_1, tmpInput_2.getDouble(index_1)); /* 130 */ } /* 131 */ } /* 132 */ /* 133 */ mutableStateArray_1[1].setOffsetAndSizeFromPreviousCursor(3, previousCursor_2); /* 134 */ } /* 135 */ /* 136 */ /* 137 */ mutableStateArray_1[0].setOffsetAndSizeFromPreviousCursor(0, previousCursor_0); /* 138 */ } /* 139 */ } /* 140 */ return (mutableStateArray_1[0].getRow()); /* 141 */ } /* 142 */ /* 143 */ /* 144 */ } 20/08/08 15:33:54.301 Executor task launch worker for task 9 DEBUG GenerateSafeProjection: code for newInstance(class org.apache.spark.ml.linalg.VectorUDT).deserialize: /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificSafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificSafeProjection extends org.apache.spark.sql.catalyst.expressions.codegen.BaseProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private InternalRow mutableRow; /* 009 */ private boolean resultIsNull_0; /* 010 */ private InternalRow[] mutableStateArray_0 = new InternalRow[1]; /* 011 */ /* 012 */ public SpecificSafeProjection(Object[] references) { /* 013 */ this.references = references; /* 014 */ mutableRow = (InternalRow) references[references.length - 1]; /* 015 */ /* 016 */ /* 017 */ } /* 018 */ /* 019 */ public void initialize(int partitionIndex) { /* 020 */ /* 021 */ } /* 022 */ /* 023 */ public java.lang.Object apply(java.lang.Object _i) { /* 024 */ InternalRow i = (InternalRow) _i; /* 025 */ final org.apache.spark.ml.linalg.VectorUDT value_1 = false ? /* 026 */ null : new org.apache.spark.ml.linalg.VectorUDT(); /* 027 */ boolean isNull_0 = true; /* 028 */ org.apache.spark.ml.linalg.Vector value_0 = null; /* 029 */ if (!false) { /* 030 */ resultIsNull_0 = false; /* 031 */ if (!resultIsNull_0) { /* 032 */ boolean isNull_2 = i.isNullAt(0); /* 033 */ InternalRow value_2 = isNull_2 ? /* 034 */ null : (i.getStruct(0, 4)); /* 035 */ resultIsNull_0 = isNull_2; /* 036 */ mutableStateArray_0[0] = value_2; /* 037 */ } /* 038 */ /* 039 */ isNull_0 = resultIsNull_0; /* 040 */ if (!isNull_0) { /* 041 */ /* 042 */ Object funcResult_0 = null; /* 043 */ funcResult_0 = value_1.deserialize(mutableStateArray_0[0]); /* 044 */ /* 045 */ if (funcResult_0 != null) { /* 046 */ value_0 = (org.apache.spark.ml.linalg.Vector) funcResult_0; /* 047 */ } else { /* 048 */ isNull_0 = true; /* 049 */ } /* 050 */ /* 051 */ /* 052 */ } /* 053 */ } /* 054 */ if (isNull_0) { /* 055 */ mutableRow.setNullAt(0); /* 056 */ } else { /* 057 */ /* 058 */ mutableRow.update(0, value_0); /* 059 */ } /* 060 */ /* 061 */ return mutableRow; /* 062 */ } /* 063 */ /* 064 */ /* 065 */ } 20/08/08 15:33:54.308 Executor task launch worker for task 9 DEBUG GenerateUnsafeProjection: code for newInstance(class org.apache.spark.ml.linalg.VectorUDT).serialize: /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificUnsafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificUnsafeProjection extends org.apache.spark.sql.catalyst.expressions.UnsafeProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private boolean resultIsNull_0; /* 009 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[] mutableStateArray_1 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[2]; /* 010 */ private org.apache.spark.ml.linalg.Vector[] mutableStateArray_0 = new org.apache.spark.ml.linalg.Vector[1]; /* 011 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[] mutableStateArray_2 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[2]; /* 012 */ /* 013 */ public SpecificUnsafeProjection(Object[] references) { /* 014 */ this.references = references; /* 015 */ /* 016 */ mutableStateArray_1[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(1, 32); /* 017 */ mutableStateArray_1[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(mutableStateArray_1[0], 4); /* 018 */ mutableStateArray_2[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(mutableStateArray_1[1], 4); /* 019 */ mutableStateArray_2[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(mutableStateArray_1[1], 8); /* 020 */ /* 021 */ } /* 022 */ /* 023 */ public void initialize(int partitionIndex) { /* 024 */ /* 025 */ } /* 026 */ /* 027 */ // Scala.Function1 need this /* 028 */ public java.lang.Object apply(java.lang.Object row) { /* 029 */ return apply((InternalRow) row); /* 030 */ } /* 031 */ /* 032 */ public UnsafeRow apply(InternalRow i) { /* 033 */ mutableStateArray_1[0].reset(); /* 034 */ /* 035 */ /* 036 */ mutableStateArray_1[0].zeroOutNullBytes(); /* 037 */ /* 038 */ final org.apache.spark.ml.linalg.VectorUDT value_1 = false ? /* 039 */ null : new org.apache.spark.ml.linalg.VectorUDT(); /* 040 */ boolean isNull_0 = true; /* 041 */ InternalRow value_0 = null; /* 042 */ if (!false) { /* 043 */ resultIsNull_0 = false; /* 044 */ if (!resultIsNull_0) { /* 045 */ boolean isNull_2 = i.isNullAt(0); /* 046 */ org.apache.spark.ml.linalg.Vector value_2 = isNull_2 ? /* 047 */ null : ((org.apache.spark.ml.linalg.Vector)i.get(0, null)); /* 048 */ resultIsNull_0 = isNull_2; /* 049 */ mutableStateArray_0[0] = value_2; /* 050 */ } /* 051 */ /* 052 */ isNull_0 = resultIsNull_0; /* 053 */ if (!isNull_0) { /* 054 */ /* 055 */ Object funcResult_0 = null; /* 056 */ funcResult_0 = value_1.serialize(mutableStateArray_0[0]); /* 057 */ /* 058 */ if (funcResult_0 != null) { /* 059 */ value_0 = (InternalRow) funcResult_0; /* 060 */ } else { /* 061 */ isNull_0 = true; /* 062 */ } /* 063 */ /* 064 */ /* 065 */ } /* 066 */ } /* 067 */ if (isNull_0) { /* 068 */ mutableStateArray_1[0].setNullAt(0); /* 069 */ } else { /* 070 */ final InternalRow tmpInput_0 = value_0; /* 071 */ if (tmpInput_0 instanceof UnsafeRow) { /* 072 */ mutableStateArray_1[0].write(0, (UnsafeRow) tmpInput_0); /* 073 */ } else { /* 074 */ // Remember the current cursor so that we can calculate how many bytes are /* 075 */ // written later. /* 076 */ final int previousCursor_0 = mutableStateArray_1[0].cursor(); /* 077 */ /* 078 */ mutableStateArray_1[1].resetRowWriter(); /* 079 */ /* 080 */ /* 081 */ mutableStateArray_1[1].write(0, (tmpInput_0.getByte(0))); /* 082 */ /* 083 */ /* 084 */ if ((tmpInput_0.isNullAt(1))) { /* 085 */ mutableStateArray_1[1].setNullAt(1); /* 086 */ } else { /* 087 */ mutableStateArray_1[1].write(1, (tmpInput_0.getInt(1))); /* 088 */ } /* 089 */ /* 090 */ /* 091 */ if ((tmpInput_0.isNullAt(2))) { /* 092 */ mutableStateArray_1[1].setNullAt(2); /* 093 */ } else { /* 094 */ // Remember the current cursor so that we can calculate how many bytes are /* 095 */ // written later. /* 096 */ final int previousCursor_1 = mutableStateArray_1[1].cursor(); /* 097 */ /* 098 */ final ArrayData tmpInput_1 = (tmpInput_0.getArray(2)); /* 099 */ if (tmpInput_1 instanceof UnsafeArrayData) { /* 100 */ mutableStateArray_1[1].write((UnsafeArrayData) tmpInput_1); /* 101 */ } else { /* 102 */ final int numElements_0 = tmpInput_1.numElements(); /* 103 */ mutableStateArray_2[0].initialize(numElements_0); /* 104 */ /* 105 */ for (int index_0 = 0; index_0 < numElements_0; index_0++) { /* 106 */ mutableStateArray_2[0].write(index_0, tmpInput_1.getInt(index_0)); /* 107 */ } /* 108 */ } /* 109 */ /* 110 */ mutableStateArray_1[1].setOffsetAndSizeFromPreviousCursor(2, previousCursor_1); /* 111 */ } /* 112 */ /* 113 */ /* 114 */ if ((tmpInput_0.isNullAt(3))) { /* 115 */ mutableStateArray_1[1].setNullAt(3); /* 116 */ } else { /* 117 */ // Remember the current cursor so that we can calculate how many bytes are /* 118 */ // written later. /* 119 */ final int previousCursor_2 = mutableStateArray_1[1].cursor(); /* 120 */ /* 121 */ final ArrayData tmpInput_2 = (tmpInput_0.getArray(3)); /* 122 */ if (tmpInput_2 instanceof UnsafeArrayData) { /* 123 */ mutableStateArray_1[1].write((UnsafeArrayData) tmpInput_2); /* 124 */ } else { /* 125 */ final int numElements_1 = tmpInput_2.numElements(); /* 126 */ mutableStateArray_2[1].initialize(numElements_1); /* 127 */ /* 128 */ for (int index_1 = 0; index_1 < numElements_1; index_1++) { /* 129 */ mutableStateArray_2[1].write(index_1, tmpInput_2.getDouble(index_1)); /* 130 */ } /* 131 */ } /* 132 */ /* 133 */ mutableStateArray_1[1].setOffsetAndSizeFromPreviousCursor(3, previousCursor_2); /* 134 */ } /* 135 */ /* 136 */ /* 137 */ mutableStateArray_1[0].setOffsetAndSizeFromPreviousCursor(0, previousCursor_0); /* 138 */ } /* 139 */ } /* 140 */ return (mutableStateArray_1[0].getRow()); /* 141 */ } /* 142 */ /* 143 */ /* 144 */ } 20/08/08 15:33:54.311 Executor task launch worker for task 9 DEBUG GenerateSafeProjection: code for newInstance(class org.apache.spark.ml.linalg.VectorUDT).deserialize: /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificSafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificSafeProjection extends org.apache.spark.sql.catalyst.expressions.codegen.BaseProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private InternalRow mutableRow; /* 009 */ private boolean resultIsNull_0; /* 010 */ private InternalRow[] mutableStateArray_0 = new InternalRow[1]; /* 011 */ /* 012 */ public SpecificSafeProjection(Object[] references) { /* 013 */ this.references = references; /* 014 */ mutableRow = (InternalRow) references[references.length - 1]; /* 015 */ /* 016 */ /* 017 */ } /* 018 */ /* 019 */ public void initialize(int partitionIndex) { /* 020 */ /* 021 */ } /* 022 */ /* 023 */ public java.lang.Object apply(java.lang.Object _i) { /* 024 */ InternalRow i = (InternalRow) _i; /* 025 */ final org.apache.spark.ml.linalg.VectorUDT value_1 = false ? /* 026 */ null : new org.apache.spark.ml.linalg.VectorUDT(); /* 027 */ boolean isNull_0 = true; /* 028 */ org.apache.spark.ml.linalg.Vector value_0 = null; /* 029 */ if (!false) { /* 030 */ resultIsNull_0 = false; /* 031 */ if (!resultIsNull_0) { /* 032 */ boolean isNull_2 = i.isNullAt(0); /* 033 */ InternalRow value_2 = isNull_2 ? /* 034 */ null : (i.getStruct(0, 4)); /* 035 */ resultIsNull_0 = isNull_2; /* 036 */ mutableStateArray_0[0] = value_2; /* 037 */ } /* 038 */ /* 039 */ isNull_0 = resultIsNull_0; /* 040 */ if (!isNull_0) { /* 041 */ /* 042 */ Object funcResult_0 = null; /* 043 */ funcResult_0 = value_1.deserialize(mutableStateArray_0[0]); /* 044 */ /* 045 */ if (funcResult_0 != null) { /* 046 */ value_0 = (org.apache.spark.ml.linalg.Vector) funcResult_0; /* 047 */ } else { /* 048 */ isNull_0 = true; /* 049 */ } /* 050 */ /* 051 */ /* 052 */ } /* 053 */ } /* 054 */ if (isNull_0) { /* 055 */ mutableRow.setNullAt(0); /* 056 */ } else { /* 057 */ /* 058 */ mutableRow.update(0, value_0); /* 059 */ } /* 060 */ /* 061 */ return mutableRow; /* 062 */ } /* 063 */ /* 064 */ /* 065 */ } 20/08/08 15:33:54.312 Executor task launch worker for task 9 DEBUG BlockManager: Getting local block broadcast_3 20/08/08 15:33:54.312 Executor task launch worker for task 9 DEBUG BlockManager: Level for block broadcast_3 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:54.313 block-manager-storage-async-thread-pool-11 DEBUG BlockManagerStorageEndpoint: Sent response: 0 to 192.168.1.225:44433 20/08/08 15:33:54.313 block-manager-storage-async-thread-pool-12 DEBUG BlockManagerStorageEndpoint: Sent response: 0 to 192.168.1.225:44433 20/08/08 15:33:54.316 Executor task launch worker for task 12 DEBUG GenerateSafeProjection: code for newInstance(class org.apache.spark.ml.linalg.VectorUDT).deserialize: /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificSafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificSafeProjection extends org.apache.spark.sql.catalyst.expressions.codegen.BaseProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private InternalRow mutableRow; /* 009 */ private boolean resultIsNull_0; /* 010 */ private InternalRow[] mutableStateArray_0 = new InternalRow[1]; /* 011 */ /* 012 */ public SpecificSafeProjection(Object[] references) { /* 013 */ this.references = references; /* 014 */ mutableRow = (InternalRow) references[references.length - 1]; /* 015 */ /* 016 */ /* 017 */ } /* 018 */ /* 019 */ public void initialize(int partitionIndex) { /* 020 */ /* 021 */ } /* 022 */ /* 023 */ public java.lang.Object apply(java.lang.Object _i) { /* 024 */ InternalRow i = (InternalRow) _i; /* 025 */ final org.apache.spark.ml.linalg.VectorUDT value_1 = false ? /* 026 */ null : new org.apache.spark.ml.linalg.VectorUDT(); /* 027 */ boolean isNull_0 = true; /* 028 */ org.apache.spark.ml.linalg.Vector value_0 = null; /* 029 */ if (!false) { /* 030 */ resultIsNull_0 = false; /* 031 */ if (!resultIsNull_0) { /* 032 */ boolean isNull_2 = i.isNullAt(0); /* 033 */ InternalRow value_2 = isNull_2 ? /* 034 */ null : (i.getStruct(0, 4)); /* 035 */ resultIsNull_0 = isNull_2; /* 036 */ mutableStateArray_0[0] = value_2; /* 037 */ } /* 038 */ /* 039 */ isNull_0 = resultIsNull_0; /* 040 */ if (!isNull_0) { /* 041 */ /* 042 */ Object funcResult_0 = null; /* 043 */ funcResult_0 = value_1.deserialize(mutableStateArray_0[0]); /* 044 */ /* 045 */ if (funcResult_0 != null) { /* 046 */ value_0 = (org.apache.spark.ml.linalg.Vector) funcResult_0; /* 047 */ } else { /* 048 */ isNull_0 = true; /* 049 */ } /* 050 */ /* 051 */ /* 052 */ } /* 053 */ } /* 054 */ if (isNull_0) { /* 055 */ mutableRow.setNullAt(0); /* 056 */ } else { /* 057 */ /* 058 */ mutableRow.update(0, value_0); /* 059 */ } /* 060 */ /* 061 */ return mutableRow; /* 062 */ } /* 063 */ /* 064 */ /* 065 */ } 20/08/08 15:33:54.319 Executor task launch worker for task 10 DEBUG GenerateSafeProjection: code for newInstance(class org.apache.spark.ml.linalg.VectorUDT).deserialize: /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificSafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificSafeProjection extends org.apache.spark.sql.catalyst.expressions.codegen.BaseProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private InternalRow mutableRow; /* 009 */ private boolean resultIsNull_0; /* 010 */ private InternalRow[] mutableStateArray_0 = new InternalRow[1]; /* 011 */ /* 012 */ public SpecificSafeProjection(Object[] references) { /* 013 */ this.references = references; /* 014 */ mutableRow = (InternalRow) references[references.length - 1]; /* 015 */ /* 016 */ /* 017 */ } /* 018 */ /* 019 */ public void initialize(int partitionIndex) { /* 020 */ /* 021 */ } /* 022 */ /* 023 */ public java.lang.Object apply(java.lang.Object _i) { /* 024 */ InternalRow i = (InternalRow) _i; /* 025 */ final org.apache.spark.ml.linalg.VectorUDT value_1 = false ? /* 026 */ null : new org.apache.spark.ml.linalg.VectorUDT(); /* 027 */ boolean isNull_0 = true; /* 028 */ org.apache.spark.ml.linalg.Vector value_0 = null; /* 029 */ if (!false) { /* 030 */ resultIsNull_0 = false; /* 031 */ if (!resultIsNull_0) { /* 032 */ boolean isNull_2 = i.isNullAt(0); /* 033 */ InternalRow value_2 = isNull_2 ? /* 034 */ null : (i.getStruct(0, 4)); /* 035 */ resultIsNull_0 = isNull_2; /* 036 */ mutableStateArray_0[0] = value_2; /* 037 */ } /* 038 */ /* 039 */ isNull_0 = resultIsNull_0; /* 040 */ if (!isNull_0) { /* 041 */ /* 042 */ Object funcResult_0 = null; /* 043 */ funcResult_0 = value_1.deserialize(mutableStateArray_0[0]); /* 044 */ /* 045 */ if (funcResult_0 != null) { /* 046 */ value_0 = (org.apache.spark.ml.linalg.Vector) funcResult_0; /* 047 */ } else { /* 048 */ isNull_0 = true; /* 049 */ } /* 050 */ /* 051 */ /* 052 */ } /* 053 */ } /* 054 */ if (isNull_0) { /* 055 */ mutableRow.setNullAt(0); /* 056 */ } else { /* 057 */ /* 058 */ mutableRow.update(0, value_0); /* 059 */ } /* 060 */ /* 061 */ return mutableRow; /* 062 */ } /* 063 */ /* 064 */ /* 065 */ } 20/08/08 15:33:54.329 Thread-4 DEBUG Analyzer$ResolveReferences: Resolving 'features to features#48 20/08/08 15:33:54.330 Thread-4 DEBUG Analyzer$ResolveReferences: Resolving 'expected to expected#39 20/08/08 15:33:54.320 Executor task launch worker for task 11 DEBUG GenerateSafeProjection: code for newInstance(class org.apache.spark.ml.linalg.VectorUDT).deserialize: /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificSafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificSafeProjection extends org.apache.spark.sql.catalyst.expressions.codegen.BaseProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private InternalRow mutableRow; /* 009 */ private boolean resultIsNull_0; /* 010 */ private InternalRow[] mutableStateArray_0 = new InternalRow[1]; /* 011 */ /* 012 */ public SpecificSafeProjection(Object[] references) { /* 013 */ this.references = references; /* 014 */ mutableRow = (InternalRow) references[references.length - 1]; /* 015 */ /* 016 */ /* 017 */ } /* 018 */ /* 019 */ public void initialize(int partitionIndex) { /* 020 */ /* 021 */ } /* 022 */ /* 023 */ public java.lang.Object apply(java.lang.Object _i) { /* 024 */ InternalRow i = (InternalRow) _i; /* 025 */ final org.apache.spark.ml.linalg.VectorUDT value_1 = false ? /* 026 */ null : new org.apache.spark.ml.linalg.VectorUDT(); /* 027 */ boolean isNull_0 = true; /* 028 */ org.apache.spark.ml.linalg.Vector value_0 = null; /* 029 */ if (!false) { /* 030 */ resultIsNull_0 = false; /* 031 */ if (!resultIsNull_0) { /* 032 */ boolean isNull_2 = i.isNullAt(0); /* 033 */ InternalRow value_2 = isNull_2 ? /* 034 */ null : (i.getStruct(0, 4)); /* 035 */ resultIsNull_0 = isNull_2; /* 036 */ mutableStateArray_0[0] = value_2; /* 037 */ } /* 038 */ /* 039 */ isNull_0 = resultIsNull_0; /* 040 */ if (!isNull_0) { /* 041 */ /* 042 */ Object funcResult_0 = null; /* 043 */ funcResult_0 = value_1.deserialize(mutableStateArray_0[0]); /* 044 */ /* 045 */ if (funcResult_0 != null) { /* 046 */ value_0 = (org.apache.spark.ml.linalg.Vector) funcResult_0; /* 047 */ } else { /* 048 */ isNull_0 = true; /* 049 */ } /* 050 */ /* 051 */ /* 052 */ } /* 053 */ } /* 054 */ if (isNull_0) { /* 055 */ mutableRow.setNullAt(0); /* 056 */ } else { /* 057 */ /* 058 */ mutableRow.update(0, value_0); /* 059 */ } /* 060 */ /* 061 */ return mutableRow; /* 062 */ } /* 063 */ /* 064 */ /* 065 */ } 20/08/08 15:33:54.338 Executor task launch worker for task 10 INFO PythonRunner: Times: total = 22, boot = -1497, init = 1519, finish = 0 20/08/08 15:33:54.339 block-manager-storage-async-thread-pool-15 DEBUG BlockManagerStorageEndpoint: removing RDD 36 20/08/08 15:33:54.340 block-manager-storage-async-thread-pool-15 INFO BlockManager: Removing RDD 36 20/08/08 15:33:54.338 Executor task launch worker for task 9 INFO PythonRunner: Times: total = 13, boot = -1525, init = 1538, finish = 0 20/08/08 15:33:54.339 Executor task launch worker for task 12 INFO PythonRunner: Times: total = 17, boot = -1468, init = 1484, finish = 1 20/08/08 15:33:54.340 Executor task launch worker for task 11 DEBUG GenerateUnsafeProjection: code for newInstance(class org.apache.spark.ml.linalg.VectorUDT).serialize: /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificUnsafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificUnsafeProjection extends org.apache.spark.sql.catalyst.expressions.UnsafeProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private boolean resultIsNull_0; /* 009 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[] mutableStateArray_1 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[2]; /* 010 */ private org.apache.spark.ml.linalg.Vector[] mutableStateArray_0 = new org.apache.spark.ml.linalg.Vector[1]; /* 011 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[] mutableStateArray_2 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[2]; /* 012 */ /* 013 */ public SpecificUnsafeProjection(Object[] references) { /* 014 */ this.references = references; /* 015 */ /* 016 */ mutableStateArray_1[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(1, 32); /* 017 */ mutableStateArray_1[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(mutableStateArray_1[0], 4); /* 018 */ mutableStateArray_2[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(mutableStateArray_1[1], 4); /* 019 */ mutableStateArray_2[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(mutableStateArray_1[1], 8); /* 020 */ /* 021 */ } /* 022 */ /* 023 */ public void initialize(int partitionIndex) { /* 024 */ /* 025 */ } /* 026 */ /* 027 */ // Scala.Function1 need this /* 028 */ public java.lang.Object apply(java.lang.Object row) { /* 029 */ return apply((InternalRow) row); /* 030 */ } /* 031 */ /* 032 */ public UnsafeRow apply(InternalRow i) { /* 033 */ mutableStateArray_1[0].reset(); /* 034 */ /* 035 */ /* 036 */ mutableStateArray_1[0].zeroOutNullBytes(); /* 037 */ /* 038 */ final org.apache.spark.ml.linalg.VectorUDT value_1 = false ? /* 039 */ null : new org.apache.spark.ml.linalg.VectorUDT(); /* 040 */ boolean isNull_0 = true; /* 041 */ InternalRow value_0 = null; /* 042 */ if (!false) { /* 043 */ resultIsNull_0 = false; /* 044 */ if (!resultIsNull_0) { /* 045 */ boolean isNull_2 = i.isNullAt(0); /* 046 */ org.apache.spark.ml.linalg.Vector value_2 = isNull_2 ? /* 047 */ null : ((org.apache.spark.ml.linalg.Vector)i.get(0, null)); /* 048 */ resultIsNull_0 = isNull_2; /* 049 */ mutableStateArray_0[0] = value_2; /* 050 */ } /* 051 */ /* 052 */ isNull_0 = resultIsNull_0; /* 053 */ if (!isNull_0) { /* 054 */ /* 055 */ Object funcResult_0 = null; /* 056 */ funcResult_0 = value_1.serialize(mutableStateArray_0[0]); /* 057 */ /* 058 */ if (funcResult_0 != null) { /* 059 */ value_0 = (InternalRow) funcResult_0; /* 060 */ } else { /* 061 */ isNull_0 = true; /* 062 */ } /* 063 */ /* 064 */ /* 065 */ } /* 066 */ } /* 067 */ if (isNull_0) { /* 068 */ mutableStateArray_1[0].setNullAt(0); /* 069 */ } else { /* 070 */ final InternalRow tmpInput_0 = value_0; /* 071 */ if (tmpInput_0 instanceof UnsafeRow) { /* 072 */ mutableStateArray_1[0].write(0, (UnsafeRow) tmpInput_0); /* 073 */ } else { /* 074 */ // Remember the current cursor so that we can calculate how many bytes are /* 075 */ // written later. /* 076 */ final int previousCursor_0 = mutableStateArray_1[0].cursor(); /* 077 */ /* 078 */ mutableStateArray_1[1].resetRowWriter(); /* 079 */ /* 080 */ /* 081 */ mutableStateArray_1[1].write(0, (tmpInput_0.getByte(0))); /* 082 */ /* 083 */ /* 084 */ if ((tmpInput_0.isNullAt(1))) { /* 085 */ mutableStateArray_1[1].setNullAt(1); /* 086 */ } else { /* 087 */ mutableStateArray_1[1].write(1, (tmpInput_0.getInt(1))); /* 088 */ } /* 089 */ /* 090 */ /* 091 */ if ((tmpInput_0.isNullAt(2))) { /* 092 */ mutableStateArray_1[1].setNullAt(2); /* 093 */ } else { /* 094 */ // Remember the current cursor so that we can calculate how many bytes are /* 095 */ // written later. /* 096 */ final int previousCursor_1 = mutableStateArray_1[1].cursor(); /* 097 */ /* 098 */ final ArrayData tmpInput_1 = (tmpInput_0.getArray(2)); /* 099 */ if (tmpInput_1 instanceof UnsafeArrayData) { /* 100 */ mutableStateArray_1[1].write((UnsafeArrayData) tmpInput_1); /* 101 */ } else { /* 102 */ final int numElements_0 = tmpInput_1.numElements(); /* 103 */ mutableStateArray_2[0].initialize(numElements_0); /* 104 */ /* 105 */ for (int index_0 = 0; index_0 < numElements_0; index_0++) { /* 106 */ mutableStateArray_2[0].write(index_0, tmpInput_1.getInt(index_0)); /* 107 */ } /* 108 */ } /* 109 */ /* 110 */ mutableStateArray_1[1].setOffsetAndSizeFromPreviousCursor(2, previousCursor_1); /* 111 */ } /* 112 */ /* 113 */ /* 114 */ if ((tmpInput_0.isNullAt(3))) { /* 115 */ mutableStateArray_1[1].setNullAt(3); /* 116 */ } else { /* 117 */ // Remember the current cursor so that we can calculate how many bytes are /* 118 */ // written later. /* 119 */ final int previousCursor_2 = mutableStateArray_1[1].cursor(); /* 120 */ /* 121 */ final ArrayData tmpInput_2 = (tmpInput_0.getArray(3)); /* 122 */ if (tmpInput_2 instanceof UnsafeArrayData) { /* 123 */ mutableStateArray_1[1].write((UnsafeArrayData) tmpInput_2); /* 124 */ } else { /* 125 */ final int numElements_1 = tmpInput_2.numElements(); /* 126 */ mutableStateArray_2[1].initialize(numElements_1); /* 127 */ /* 128 */ for (int index_1 = 0; index_1 < numElements_1; index_1++) { /* 129 */ mutableStateArray_2[1].write(index_1, tmpInput_2.getDouble(index_1)); /* 130 */ } /* 131 */ } /* 132 */ /* 133 */ mutableStateArray_1[1].setOffsetAndSizeFromPreviousCursor(3, previousCursor_2); /* 134 */ } /* 135 */ /* 136 */ /* 137 */ mutableStateArray_1[0].setOffsetAndSizeFromPreviousCursor(0, previousCursor_0); /* 138 */ } /* 139 */ } /* 140 */ return (mutableStateArray_1[0].getRow()); /* 141 */ } /* 142 */ /* 143 */ /* 144 */ } 20/08/08 15:33:54.344 Executor task launch worker for task 11 DEBUG GenerateSafeProjection: code for newInstance(class org.apache.spark.ml.linalg.VectorUDT).deserialize: /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificSafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificSafeProjection extends org.apache.spark.sql.catalyst.expressions.codegen.BaseProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private InternalRow mutableRow; /* 009 */ private boolean resultIsNull_0; /* 010 */ private InternalRow[] mutableStateArray_0 = new InternalRow[1]; /* 011 */ /* 012 */ public SpecificSafeProjection(Object[] references) { /* 013 */ this.references = references; /* 014 */ mutableRow = (InternalRow) references[references.length - 1]; /* 015 */ /* 016 */ /* 017 */ } /* 018 */ /* 019 */ public void initialize(int partitionIndex) { /* 020 */ /* 021 */ } /* 022 */ /* 023 */ public java.lang.Object apply(java.lang.Object _i) { /* 024 */ InternalRow i = (InternalRow) _i; /* 025 */ final org.apache.spark.ml.linalg.VectorUDT value_1 = false ? /* 026 */ null : new org.apache.spark.ml.linalg.VectorUDT(); /* 027 */ boolean isNull_0 = true; /* 028 */ org.apache.spark.ml.linalg.Vector value_0 = null; /* 029 */ if (!false) { /* 030 */ resultIsNull_0 = false; /* 031 */ if (!resultIsNull_0) { /* 032 */ boolean isNull_2 = i.isNullAt(0); /* 033 */ InternalRow value_2 = isNull_2 ? /* 034 */ null : (i.getStruct(0, 4)); /* 035 */ resultIsNull_0 = isNull_2; /* 036 */ mutableStateArray_0[0] = value_2; /* 037 */ } /* 038 */ /* 039 */ isNull_0 = resultIsNull_0; /* 040 */ if (!isNull_0) { /* 041 */ /* 042 */ Object funcResult_0 = null; /* 043 */ funcResult_0 = value_1.deserialize(mutableStateArray_0[0]); /* 044 */ /* 045 */ if (funcResult_0 != null) { /* 046 */ value_0 = (org.apache.spark.ml.linalg.Vector) funcResult_0; /* 047 */ } else { /* 048 */ isNull_0 = true; /* 049 */ } /* 050 */ /* 051 */ /* 052 */ } /* 053 */ } /* 054 */ if (isNull_0) { /* 055 */ mutableRow.setNullAt(0); /* 056 */ } else { /* 057 */ /* 058 */ mutableRow.update(0, value_0); /* 059 */ } /* 060 */ /* 061 */ return mutableRow; /* 062 */ } /* 063 */ /* 064 */ /* 065 */ } 20/08/08 15:33:54.344 Executor task launch worker for task 11 INFO PythonRunner: Times: total = 73, boot = -1544, init = 1617, finish = 0 20/08/08 15:33:54.352 block-manager-storage-async-thread-pool-14 DEBUG BlockManagerStorageEndpoint: Sent response: 0 to 192.168.1.225:44433 20/08/08 15:33:54.379 block-manager-storage-async-thread-pool-17 DEBUG BlockManagerStorageEndpoint: Done removing RDD 36, response is 0 20/08/08 15:33:54.370 Executor task launch worker for task 10 INFO Executor: Finished task 1.0 in stage 3.0 (TID 10). 2450 bytes result sent to driver 20/08/08 15:33:54.389 Executor task launch worker for task 10 DEBUG ExecutorMetricsPoller: stageTCMP: (3, 0) -> 3 20/08/08 15:33:54.390 dispatcher-event-loop-3 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_3.0, runningTasks: 3 20/08/08 15:33:54.390 dispatcher-event-loop-3 DEBUG TaskSetManager: No tasks for locality level NO_PREF, so moving to locality level ANY 20/08/08 15:33:54.391 task-result-getter-1 INFO TaskSetManager: Finished task 1.0 in stage 3.0 (TID 10) in 274 ms on 192.168.1.225 (executor driver) (1/4) 20/08/08 15:33:54.395 Executor task launch worker for task 11 INFO Executor: Finished task 2.0 in stage 3.0 (TID 11). 2450 bytes result sent to driver 20/08/08 15:33:54.395 Executor task launch worker for task 12 INFO Executor: Finished task 3.0 in stage 3.0 (TID 12). 2450 bytes result sent to driver 20/08/08 15:33:54.395 Executor task launch worker for task 11 DEBUG ExecutorMetricsPoller: stageTCMP: (3, 0) -> 2 20/08/08 15:33:54.395 Executor task launch worker for task 12 DEBUG ExecutorMetricsPoller: stageTCMP: (3, 0) -> 1 20/08/08 15:33:54.395 dispatcher-event-loop-2 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_3.0, runningTasks: 2 20/08/08 15:33:54.396 dispatcher-event-loop-2 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_3.0, runningTasks: 1 20/08/08 15:33:54.396 Executor task launch worker for task 9 INFO Executor: Finished task 0.0 in stage 3.0 (TID 9). 2450 bytes result sent to driver 20/08/08 15:33:54.397 dispatcher-event-loop-0 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_3.0, runningTasks: 0 20/08/08 15:33:54.397 Executor task launch worker for task 9 DEBUG ExecutorMetricsPoller: removing (3, 0) from stageTCMP 20/08/08 15:33:54.397 task-result-getter-2 INFO TaskSetManager: Finished task 3.0 in stage 3.0 (TID 12) in 279 ms on 192.168.1.225 (executor driver) (2/4) 20/08/08 15:33:54.398 task-result-getter-3 INFO TaskSetManager: Finished task 0.0 in stage 3.0 (TID 9) in 282 ms on 192.168.1.225 (executor driver) (3/4) 20/08/08 15:33:54.398 task-result-getter-0 INFO TaskSetManager: Finished task 2.0 in stage 3.0 (TID 11) in 281 ms on 192.168.1.225 (executor driver) (4/4) 20/08/08 15:33:54.398 task-result-getter-0 INFO TaskSchedulerImpl: Removed TaskSet 3.0, whose tasks have all completed, from pool 20/08/08 15:33:54.402 dag-scheduler-event-loop DEBUG DAGScheduler: ShuffleMapTask finished on driver 20/08/08 15:33:54.402 block-manager-storage-async-thread-pool-17 DEBUG BlockManagerStorageEndpoint: Sent response: 0 to 192.168.1.225:44433 20/08/08 15:33:54.404 dag-scheduler-event-loop DEBUG DAGScheduler: ShuffleMapTask finished on driver 20/08/08 15:33:54.405 dag-scheduler-event-loop DEBUG DAGScheduler: ShuffleMapTask finished on driver 20/08/08 15:33:54.409 dag-scheduler-event-loop DEBUG DAGScheduler: ShuffleMapTask finished on driver 20/08/08 15:33:54.410 dag-scheduler-event-loop INFO DAGScheduler: ShuffleMapStage 3 (collect at ClusteringMetrics.scala:97) finished in 0.314 s 20/08/08 15:33:54.410 dag-scheduler-event-loop INFO DAGScheduler: looking for newly runnable stages 20/08/08 15:33:54.432 dag-scheduler-event-loop INFO DAGScheduler: running: Set() 20/08/08 15:33:54.432 dag-scheduler-event-loop INFO DAGScheduler: waiting: Set(ResultStage 4) 20/08/08 15:33:54.432 dag-scheduler-event-loop INFO DAGScheduler: failed: Set() 20/08/08 15:33:54.432 dag-scheduler-event-loop DEBUG MapOutputTrackerMaster: Increasing epoch to 2 20/08/08 15:33:54.432 dag-scheduler-event-loop DEBUG DAGScheduler: submitStage(ResultStage 4 (name=collect at ClusteringMetrics.scala:97;jobs=2)) 20/08/08 15:33:54.432 dag-scheduler-event-loop DEBUG DAGScheduler: missing: List() 20/08/08 15:33:54.432 dag-scheduler-event-loop INFO DAGScheduler: Submitting ResultStage 4 (MapPartitionsRDD[17] at collect at ClusteringMetrics.scala:97), which has no missing parents 20/08/08 15:33:54.432 dag-scheduler-event-loop DEBUG DAGScheduler: submitMissingTasks(ResultStage 4) 20/08/08 15:33:54.441 block-manager-storage-async-thread-pool-18 DEBUG BlockManagerStorageEndpoint: removing RDD 24 20/08/08 15:33:54.441 block-manager-storage-async-thread-pool-18 INFO BlockManager: Removing RDD 24 20/08/08 15:33:54.441 block-manager-storage-async-thread-pool-18 DEBUG BlockManager: Removing block rdd_24_0 20/08/08 15:33:54.441 block-manager-storage-async-thread-pool-18 DEBUG MemoryStore: Block rdd_24_0 of size 192 dropped from memory (free 383624310) 20/08/08 15:33:54.442 block-manager-storage-async-thread-pool-18 DEBUG BlockManager: Removing block rdd_24_4 20/08/08 15:33:54.442 block-manager-storage-async-thread-pool-18 DEBUG MemoryStore: Block rdd_24_4 of size 16 dropped from memory (free 383624326) 20/08/08 15:33:54.442 block-manager-storage-async-thread-pool-18 DEBUG BlockManager: Removing block rdd_24_5 20/08/08 15:33:54.442 block-manager-storage-async-thread-pool-18 DEBUG MemoryStore: Block rdd_24_5 of size 16 dropped from memory (free 383624342) 20/08/08 15:33:54.442 block-manager-storage-async-thread-pool-18 DEBUG BlockManager: Removing block rdd_24_6 20/08/08 15:33:54.442 block-manager-storage-async-thread-pool-18 DEBUG MemoryStore: Block rdd_24_6 of size 16 dropped from memory (free 383624358) 20/08/08 15:33:54.442 block-manager-storage-async-thread-pool-18 DEBUG BlockManager: Removing block rdd_24_7 20/08/08 15:33:54.442 block-manager-storage-async-thread-pool-18 DEBUG MemoryStore: Block rdd_24_7 of size 16 dropped from memory (free 383624374) 20/08/08 15:33:54.442 block-manager-storage-async-thread-pool-18 DEBUG BlockManager: Removing block rdd_24_8 20/08/08 15:33:54.442 block-manager-storage-async-thread-pool-18 DEBUG MemoryStore: Block rdd_24_8 of size 16 dropped from memory (free 383624390) 20/08/08 15:33:54.442 block-manager-storage-async-thread-pool-18 DEBUG BlockManager: Removing block rdd_24_3 20/08/08 15:33:54.442 block-manager-storage-async-thread-pool-18 DEBUG MemoryStore: Block rdd_24_3 of size 16 dropped from memory (free 383624406) 20/08/08 15:33:54.442 block-manager-storage-async-thread-pool-18 DEBUG BlockManager: Removing block rdd_24_2 20/08/08 15:33:54.442 block-manager-storage-async-thread-pool-18 DEBUG MemoryStore: Block rdd_24_2 of size 192 dropped from memory (free 383624598) 20/08/08 15:33:54.443 block-manager-storage-async-thread-pool-18 DEBUG BlockManager: Removing block rdd_24_9 20/08/08 15:33:54.443 block-manager-storage-async-thread-pool-18 DEBUG MemoryStore: Block rdd_24_9 of size 16 dropped from memory (free 383624614) 20/08/08 15:33:54.443 block-manager-storage-async-thread-pool-18 DEBUG BlockManager: Removing block rdd_24_1 20/08/08 15:33:54.443 block-manager-storage-async-thread-pool-18 DEBUG MemoryStore: Block rdd_24_1 of size 208 dropped from memory (free 383624822) 20/08/08 15:33:54.446 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_5 stored as values in memory (estimated size 12.8 KiB, free 366.2 MiB) 20/08/08 15:33:54.446 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_5 locally took 1 ms 20/08/08 15:33:54.446 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_5 without replication took 1 ms 20/08/08 15:33:54.447 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_5_piece0 stored as bytes in memory (estimated size 5.9 KiB, free 366.2 MiB) 20/08/08 15:33:54.448 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master broadcast_5_piece0 for BlockManagerId(driver, 192.168.1.225, 36391, None) 20/08/08 15:33:54.452 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added broadcast_5_piece0 in memory on 192.168.1.225:36391 (size: 5.9 KiB, free: 366.3 MiB) 20/08/08 15:33:54.452 dag-scheduler-event-loop DEBUG BlockManagerMaster: Updated info of block broadcast_5_piece0 20/08/08 15:33:54.452 dag-scheduler-event-loop DEBUG BlockManager: Told master about block broadcast_5_piece0 20/08/08 15:33:54.452 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_5_piece0 locally took 5 ms 20/08/08 15:33:54.452 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_5_piece0 without replication took 5 ms 20/08/08 15:33:54.452 dag-scheduler-event-loop INFO SparkContext: Created broadcast 5 from broadcast at DAGScheduler.scala:1348 20/08/08 15:33:54.453 dag-scheduler-event-loop INFO DAGScheduler: Submitting 1 missing tasks from ResultStage 4 (MapPartitionsRDD[17] at collect at ClusteringMetrics.scala:97) (first 15 tasks are for partitions Vector(0)) 20/08/08 15:33:54.453 dag-scheduler-event-loop INFO TaskSchedulerImpl: Adding task set 4.0 with 1 tasks resource profile 0 20/08/08 15:33:54.454 dag-scheduler-event-loop DEBUG TaskSetManager: Epoch for TaskSet 4.0: 2 20/08/08 15:33:54.454 dag-scheduler-event-loop DEBUG TaskSetManager: Adding pending tasks took 0 ms 20/08/08 15:33:54.454 dag-scheduler-event-loop DEBUG TaskSetManager: Valid locality levels for TaskSet 4.0: NODE_LOCAL, ANY 20/08/08 15:33:54.454 dispatcher-event-loop-3 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_4.0, runningTasks: 0 20/08/08 15:33:54.456 dispatcher-event-loop-3 INFO TaskSetManager: Starting task 0.0 in stage 4.0 (TID 13, 192.168.1.225, executor driver, partition 0, NODE_LOCAL, 7325 bytes) taskResourceAssignments Map() 20/08/08 15:33:54.456 dispatcher-event-loop-3 DEBUG TaskSetManager: No tasks for locality level NODE_LOCAL, so moving to locality level ANY 20/08/08 15:33:54.457 Executor task launch worker for task 13 INFO Executor: Running task 0.0 in stage 4.0 (TID 13) 20/08/08 15:33:54.458 Executor task launch worker for task 13 DEBUG ExecutorMetricsPoller: stageTCMP: (4, 0) -> 1 20/08/08 15:33:54.458 Executor task launch worker for task 13 DEBUG BlockManager: Getting local block broadcast_5 20/08/08 15:33:54.458 Executor task launch worker for task 13 DEBUG BlockManager: Level for block broadcast_5 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/08/08 15:33:54.464 Executor task launch worker for task 13 DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 1 20/08/08 15:33:54.464 Executor task launch worker for task 13 DEBUG MapOutputTrackerMaster: Convert map statuses for shuffle 1, mappers 0-4, partitions 0-1 20/08/08 15:33:54.465 Executor task launch worker for task 13 DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 20/08/08 15:33:54.465 Executor task launch worker for task 13 INFO ShuffleBlockFetcherIterator: Getting 4 (282.0 B) non-empty blocks including 4 (282.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 20/08/08 15:33:54.465 Executor task launch worker for task 13 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 20/08/08 15:33:54.465 Executor task launch worker for task 13 DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_1_9_0,0), (shuffle_1_10_0,1), (shuffle_1_11_0,2), (shuffle_1_12_0,3) 20/08/08 15:33:54.465 Executor task launch worker for task 13 INFO BlockManager: Getting local shuffle block shuffle_1_9_0 20/08/08 15:33:54.466 Executor task launch worker for task 13 INFO BlockManager: Getting local shuffle block shuffle_1_10_0 20/08/08 15:33:54.466 Executor task launch worker for task 13 INFO BlockManager: Getting local shuffle block shuffle_1_11_0 20/08/08 15:33:54.466 Executor task launch worker for task 13 INFO BlockManager: Getting local shuffle block shuffle_1_12_0 20/08/08 15:33:54.466 Executor task launch worker for task 13 DEBUG ShuffleBlockFetcherIterator: Got local blocks in 1 ms 20/08/08 15:33:54.464 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(28) 20/08/08 15:33:54.477 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 28 20/08/08 15:33:54.479 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 28 20/08/08 15:33:54.479 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(67) 20/08/08 15:33:54.479 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 67 20/08/08 15:33:54.479 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 67 20/08/08 15:33:54.479 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(83) 20/08/08 15:33:54.479 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 83 20/08/08 15:33:54.479 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 83 20/08/08 15:33:54.479 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(107) 20/08/08 15:33:54.479 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 107 20/08/08 15:33:54.479 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 107 20/08/08 15:33:54.479 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(66) 20/08/08 15:33:54.479 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 66 20/08/08 15:33:54.479 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 66 20/08/08 15:33:54.479 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(122) 20/08/08 15:33:54.479 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 122 20/08/08 15:33:54.479 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 122 20/08/08 15:33:54.479 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(78) 20/08/08 15:33:54.479 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 78 20/08/08 15:33:54.479 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 78 20/08/08 15:33:54.479 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(82) 20/08/08 15:33:54.479 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 82 20/08/08 15:33:54.479 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 82 20/08/08 15:33:54.479 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(56) 20/08/08 15:33:54.479 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 56 20/08/08 15:33:54.480 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 56 20/08/08 15:33:54.480 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(121) 20/08/08 15:33:54.480 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 121 20/08/08 15:33:54.480 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 121 20/08/08 15:33:54.480 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(45) 20/08/08 15:33:54.480 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 45 20/08/08 15:33:54.480 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 45 20/08/08 15:33:54.480 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(32) 20/08/08 15:33:54.480 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 32 20/08/08 15:33:54.480 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 32 20/08/08 15:33:54.480 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(104) 20/08/08 15:33:54.480 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 104 20/08/08 15:33:54.480 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 104 20/08/08 15:33:54.480 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(52) 20/08/08 15:33:54.480 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 52 20/08/08 15:33:54.480 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 52 20/08/08 15:33:54.480 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(103) 20/08/08 15:33:54.480 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 103 20/08/08 15:33:54.480 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 103 20/08/08 15:33:54.480 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(116) 20/08/08 15:33:54.480 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 116 20/08/08 15:33:54.480 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 116 20/08/08 15:33:54.480 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(93) 20/08/08 15:33:54.480 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 93 20/08/08 15:33:54.480 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 93 20/08/08 15:33:54.480 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(75) 20/08/08 15:33:54.480 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 75 20/08/08 15:33:54.480 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 75 20/08/08 15:33:54.480 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(118) 20/08/08 15:33:54.480 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 118 20/08/08 15:33:54.480 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 118 20/08/08 15:33:54.480 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(86) 20/08/08 15:33:54.480 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 86 20/08/08 15:33:54.480 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 86 20/08/08 15:33:54.480 block-manager-storage-async-thread-pool-20 DEBUG BlockManagerStorageEndpoint: Done removing RDD 24, response is 0 20/08/08 15:33:54.480 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(129) 20/08/08 15:33:54.481 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 129 20/08/08 15:33:54.481 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 129 20/08/08 15:33:54.481 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(43) 20/08/08 15:33:54.481 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 43 20/08/08 15:33:54.481 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 43 20/08/08 15:33:54.481 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(100) 20/08/08 15:33:54.481 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 100 20/08/08 15:33:54.481 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 100 20/08/08 15:33:54.481 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(130) 20/08/08 15:33:54.481 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 130 20/08/08 15:33:54.481 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 130 20/08/08 15:33:54.481 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(49) 20/08/08 15:33:54.481 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 49 20/08/08 15:33:54.481 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 49 20/08/08 15:33:54.481 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(105) 20/08/08 15:33:54.481 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 105 20/08/08 15:33:54.481 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 105 20/08/08 15:33:54.481 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(88) 20/08/08 15:33:54.481 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 88 20/08/08 15:33:54.481 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 88 20/08/08 15:33:54.481 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(80) 20/08/08 15:33:54.481 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 80 20/08/08 15:33:54.481 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 80 20/08/08 15:33:54.481 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(47) 20/08/08 15:33:54.481 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 47 20/08/08 15:33:54.481 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 47 20/08/08 15:33:54.481 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(94) 20/08/08 15:33:54.481 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 94 20/08/08 15:33:54.481 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 94 20/08/08 15:33:54.481 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(89) 20/08/08 15:33:54.481 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 89 20/08/08 15:33:54.481 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 89 20/08/08 15:33:54.481 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(85) 20/08/08 15:33:54.482 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 85 20/08/08 15:33:54.482 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 85 20/08/08 15:33:54.482 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(54) 20/08/08 15:33:54.482 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 54 20/08/08 15:33:54.482 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 54 20/08/08 15:33:54.482 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(96) 20/08/08 15:33:54.482 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 96 20/08/08 15:33:54.482 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 96 20/08/08 15:33:54.482 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(90) 20/08/08 15:33:54.482 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 90 20/08/08 15:33:54.482 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 90 20/08/08 15:33:54.482 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(117) 20/08/08 15:33:54.482 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 117 20/08/08 15:33:54.482 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 117 20/08/08 15:33:54.482 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(70) 20/08/08 15:33:54.482 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 70 20/08/08 15:33:54.482 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 70 20/08/08 15:33:54.482 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(76) 20/08/08 15:33:54.482 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 76 20/08/08 15:33:54.482 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 76 20/08/08 15:33:54.482 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(109) 20/08/08 15:33:54.482 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 109 20/08/08 15:33:54.482 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 109 20/08/08 15:33:54.482 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(113) 20/08/08 15:33:54.482 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 113 20/08/08 15:33:54.482 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 113 20/08/08 15:33:54.482 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(44) 20/08/08 15:33:54.482 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 44 20/08/08 15:33:54.482 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 44 20/08/08 15:33:54.482 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(74) 20/08/08 15:33:54.482 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 74 20/08/08 15:33:54.482 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 74 20/08/08 15:33:54.482 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(97) 20/08/08 15:33:54.482 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 97 20/08/08 15:33:54.482 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 97 20/08/08 15:33:54.483 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(42) 20/08/08 15:33:54.483 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 42 20/08/08 15:33:54.483 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 42 20/08/08 15:33:54.483 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(124) 20/08/08 15:33:54.483 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 124 20/08/08 15:33:54.483 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 124 20/08/08 15:33:54.483 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(39) 20/08/08 15:33:54.483 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 39 20/08/08 15:33:54.483 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 39 20/08/08 15:33:54.483 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(128) 20/08/08 15:33:54.483 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 128 20/08/08 15:33:54.483 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 128 20/08/08 15:33:54.483 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(84) 20/08/08 15:33:54.483 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 84 20/08/08 15:33:54.483 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 84 20/08/08 15:33:54.483 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(112) 20/08/08 15:33:54.483 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 112 20/08/08 15:33:54.483 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 112 20/08/08 15:33:54.483 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(110) 20/08/08 15:33:54.483 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 110 20/08/08 15:33:54.483 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 110 20/08/08 15:33:54.483 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(63) 20/08/08 15:33:54.483 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 63 20/08/08 15:33:54.483 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 63 20/08/08 15:33:54.483 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(125) 20/08/08 15:33:54.483 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 125 20/08/08 15:33:54.483 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 125 20/08/08 15:33:54.483 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(40) 20/08/08 15:33:54.483 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 40 20/08/08 15:33:54.483 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 40 20/08/08 15:33:54.483 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(71) 20/08/08 15:33:54.483 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 71 20/08/08 15:33:54.483 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 71 20/08/08 15:33:54.483 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(98) 20/08/08 15:33:54.484 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 98 20/08/08 15:33:54.484 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 98 20/08/08 15:33:54.484 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(30) 20/08/08 15:33:54.484 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 30 20/08/08 15:33:54.484 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 30 20/08/08 15:33:54.484 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanShuffle(0) 20/08/08 15:33:54.484 Spark Context Cleaner DEBUG ContextCleaner: Cleaning shuffle 0 20/08/08 15:33:54.490 Executor task launch worker for task 13 INFO Executor: Finished task 0.0 in stage 4.0 (TID 13). 2648 bytes result sent to driver 20/08/08 15:33:54.491 Executor task launch worker for task 13 DEBUG ExecutorMetricsPoller: removing (4, 0) from stageTCMP 20/08/08 15:33:54.491 dispatcher-event-loop-1 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_4.0, runni