20/01/23 18:53:47.301 main WARN Utils: Your hostname, spark-jenkins-arm-worker resolves to a loopback address: 127.0.1.1; using 192.168.1.225 instead (on interface eth0) 20/01/23 18:53:47.301 main WARN Utils: Your hostname, spark-jenkins-arm-worker resolves to a loopback address: 127.0.1.1; using 192.168.1.225 instead (on interface eth0) 20/01/23 18:53:47.306 main WARN Utils: Your hostname, spark-jenkins-arm-worker resolves to a loopback address: 127.0.1.1; using 192.168.1.225 instead (on interface eth0) 20/01/23 18:53:47.307 main WARN Utils: Set SPARK_LOCAL_IP if you need to bind to another address 20/01/23 18:53:47.307 main WARN Utils: Set SPARK_LOCAL_IP if you need to bind to another address 20/01/23 18:53:47.308 main WARN Utils: Set SPARK_LOCAL_IP if you need to bind to another address 20/01/23 18:53:47.320 main WARN Utils: Your hostname, spark-jenkins-arm-worker resolves to a loopback address: 127.0.1.1; using 192.168.1.225 instead (on interface eth0) 20/01/23 18:53:47.322 main WARN Utils: Set SPARK_LOCAL_IP if you need to bind to another address 20/01/23 18:53:48.010 main DEBUG ShutdownHookManager: Adding shutdown hook 20/01/23 18:53:48.020 main DEBUG ShutdownHookManager: Adding shutdown hook 20/01/23 18:53:48.021 main DEBUG ShutdownHookManager: Adding shutdown hook 20/01/23 18:53:48.048 main DEBUG ShutdownHookManager: Adding shutdown hook 20/01/23 18:53:48.164 main DEBUG MutableMetricsFactory: field org.apache.hadoop.metrics2.lib.MutableRate org.apache.hadoop.security.UserGroupInformation$UgiMetrics.loginSuccess with annotation @org.apache.hadoop.metrics2.annotation.Metric(sampleName=Ops, about=, always=false, type=DEFAULT, valueName=Time, value=[Rate of successful kerberos logins and latency (milliseconds)]) 20/01/23 18:53:48.165 main DEBUG MutableMetricsFactory: field org.apache.hadoop.metrics2.lib.MutableRate org.apache.hadoop.security.UserGroupInformation$UgiMetrics.loginSuccess with annotation @org.apache.hadoop.metrics2.annotation.Metric(sampleName=Ops, always=false, about=, type=DEFAULT, valueName=Time, value=[Rate of successful kerberos logins and latency (milliseconds)]) 20/01/23 18:53:48.167 main DEBUG MutableMetricsFactory: field org.apache.hadoop.metrics2.lib.MutableRate org.apache.hadoop.security.UserGroupInformation$UgiMetrics.loginSuccess with annotation @org.apache.hadoop.metrics2.annotation.Metric(always=false, about=, sampleName=Ops, type=DEFAULT, valueName=Time, value=[Rate of successful kerberos logins and latency (milliseconds)]) 20/01/23 18:53:48.167 main DEBUG MutableMetricsFactory: field org.apache.hadoop.metrics2.lib.MutableRate org.apache.hadoop.security.UserGroupInformation$UgiMetrics.loginSuccess with annotation @org.apache.hadoop.metrics2.annotation.Metric(sampleName=Ops, always=false, about=, type=DEFAULT, valueName=Time, value=[Rate of successful kerberos logins and latency (milliseconds)]) 20/01/23 18:53:48.178 main DEBUG MutableMetricsFactory: field org.apache.hadoop.metrics2.lib.MutableRate org.apache.hadoop.security.UserGroupInformation$UgiMetrics.loginFailure with annotation @org.apache.hadoop.metrics2.annotation.Metric(sampleName=Ops, about=, always=false, type=DEFAULT, valueName=Time, value=[Rate of failed kerberos logins and latency (milliseconds)]) 20/01/23 18:53:48.178 main DEBUG MutableMetricsFactory: field org.apache.hadoop.metrics2.lib.MutableRate org.apache.hadoop.security.UserGroupInformation$UgiMetrics.loginFailure with annotation @org.apache.hadoop.metrics2.annotation.Metric(sampleName=Ops, always=false, about=, type=DEFAULT, valueName=Time, value=[Rate of failed kerberos logins and latency (milliseconds)]) 20/01/23 18:53:48.178 main DEBUG MutableMetricsFactory: field org.apache.hadoop.metrics2.lib.MutableRate org.apache.hadoop.security.UserGroupInformation$UgiMetrics.getGroups with annotation @org.apache.hadoop.metrics2.annotation.Metric(sampleName=Ops, about=, always=false, type=DEFAULT, valueName=Time, value=[GetGroups]) 20/01/23 18:53:48.178 main DEBUG MutableMetricsFactory: field org.apache.hadoop.metrics2.lib.MutableRate org.apache.hadoop.security.UserGroupInformation$UgiMetrics.getGroups with annotation @org.apache.hadoop.metrics2.annotation.Metric(sampleName=Ops, always=false, about=, type=DEFAULT, valueName=Time, value=[GetGroups]) 20/01/23 18:53:48.179 main DEBUG MutableMetricsFactory: field org.apache.hadoop.metrics2.lib.MutableRate org.apache.hadoop.security.UserGroupInformation$UgiMetrics.loginFailure with annotation @org.apache.hadoop.metrics2.annotation.Metric(always=false, about=, sampleName=Ops, type=DEFAULT, valueName=Time, value=[Rate of failed kerberos logins and latency (milliseconds)]) 20/01/23 18:53:48.179 main DEBUG MutableMetricsFactory: field org.apache.hadoop.metrics2.lib.MutableRate org.apache.hadoop.security.UserGroupInformation$UgiMetrics.loginFailure with annotation @org.apache.hadoop.metrics2.annotation.Metric(sampleName=Ops, always=false, about=, type=DEFAULT, valueName=Time, value=[Rate of failed kerberos logins and latency (milliseconds)]) 20/01/23 18:53:48.180 main DEBUG MutableMetricsFactory: field org.apache.hadoop.metrics2.lib.MutableRate org.apache.hadoop.security.UserGroupInformation$UgiMetrics.getGroups with annotation @org.apache.hadoop.metrics2.annotation.Metric(always=false, about=, sampleName=Ops, type=DEFAULT, valueName=Time, value=[GetGroups]) 20/01/23 18:53:48.180 main DEBUG MutableMetricsFactory: field org.apache.hadoop.metrics2.lib.MutableRate org.apache.hadoop.security.UserGroupInformation$UgiMetrics.getGroups with annotation @org.apache.hadoop.metrics2.annotation.Metric(sampleName=Ops, always=false, about=, type=DEFAULT, valueName=Time, value=[GetGroups]) 20/01/23 18:53:48.180 main DEBUG MetricsSystemImpl: UgiMetrics, User and group related metrics 20/01/23 18:53:48.180 main DEBUG MetricsSystemImpl: UgiMetrics, User and group related metrics 20/01/23 18:53:48.181 main DEBUG MetricsSystemImpl: UgiMetrics, User and group related metrics 20/01/23 18:53:48.181 main DEBUG MetricsSystemImpl: UgiMetrics, User and group related metrics 20/01/23 18:53:48.268 main DEBUG Shell: Failed to detect a valid hadoop home directory java.io.IOException: HADOOP_HOME or hadoop.home.dir are not set. at org.apache.hadoop.util.Shell.checkHadoopHome(Shell.java:329) at org.apache.hadoop.util.Shell.(Shell.java:354) at org.apache.hadoop.util.StringUtils.(StringUtils.java:80) at org.apache.hadoop.security.SecurityUtil.getAuthenticationMethod(SecurityUtil.java:611) at org.apache.hadoop.security.UserGroupInformation.initialize(UserGroupInformation.java:274) at org.apache.hadoop.security.UserGroupInformation.ensureInitialized(UserGroupInformation.java:262) at org.apache.hadoop.security.UserGroupInformation.loginUserFromSubject(UserGroupInformation.java:807) at org.apache.hadoop.security.UserGroupInformation.getLoginUser(UserGroupInformation.java:777) at org.apache.hadoop.security.UserGroupInformation.getCurrentUser(UserGroupInformation.java:650) at org.apache.spark.util.Utils$.$anonfun$getCurrentUserName$1(Utils.scala:2412) at scala.Option.getOrElse(Option.scala:189) at org.apache.spark.util.Utils$.getCurrentUserName(Utils.scala:2412) at org.apache.spark.SecurityManager.(SecurityManager.scala:79) at org.apache.spark.deploy.SparkSubmit.secMgr$lzycompute$1(SparkSubmit.scala:368) at org.apache.spark.deploy.SparkSubmit.secMgr$1(SparkSubmit.scala:368) at org.apache.spark.deploy.SparkSubmit.$anonfun$prepareSubmitEnvironment$8(SparkSubmit.scala:376) at scala.Option.map(Option.scala:230) at org.apache.spark.deploy.SparkSubmit.prepareSubmitEnvironment(SparkSubmit.scala:376) at org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:871) at org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180) at org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203) at org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90) at org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1007) at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1016) at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala) 20/01/23 18:53:48.261 main DEBUG Shell: Failed to detect a valid hadoop home directory java.io.IOException: HADOOP_HOME or hadoop.home.dir are not set. at org.apache.hadoop.util.Shell.checkHadoopHome(Shell.java:329) at org.apache.hadoop.util.Shell.(Shell.java:354) at org.apache.hadoop.util.StringUtils.(StringUtils.java:80) at org.apache.hadoop.security.SecurityUtil.getAuthenticationMethod(SecurityUtil.java:611) at org.apache.hadoop.security.UserGroupInformation.initialize(UserGroupInformation.java:274) at org.apache.hadoop.security.UserGroupInformation.ensureInitialized(UserGroupInformation.java:262) at org.apache.hadoop.security.UserGroupInformation.loginUserFromSubject(UserGroupInformation.java:807) at org.apache.hadoop.security.UserGroupInformation.getLoginUser(UserGroupInformation.java:777) at org.apache.hadoop.security.UserGroupInformation.getCurrentUser(UserGroupInformation.java:650) at org.apache.spark.util.Utils$.$anonfun$getCurrentUserName$1(Utils.scala:2412) at scala.Option.getOrElse(Option.scala:189) at org.apache.spark.util.Utils$.getCurrentUserName(Utils.scala:2412) at org.apache.spark.SecurityManager.(SecurityManager.scala:79) at org.apache.spark.deploy.SparkSubmit.secMgr$lzycompute$1(SparkSubmit.scala:368) at org.apache.spark.deploy.SparkSubmit.secMgr$1(SparkSubmit.scala:368) at org.apache.spark.deploy.SparkSubmit.$anonfun$prepareSubmitEnvironment$8(SparkSubmit.scala:376) at scala.Option.map(Option.scala:230) at org.apache.spark.deploy.SparkSubmit.prepareSubmitEnvironment(SparkSubmit.scala:376) at org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:871) at org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180) at org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203) at org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90) at org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1007) at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1016) at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala) 20/01/23 18:53:48.281 main DEBUG Shell: Failed to detect a valid hadoop home directory java.io.IOException: HADOOP_HOME or hadoop.home.dir are not set. at org.apache.hadoop.util.Shell.checkHadoopHome(Shell.java:329) at org.apache.hadoop.util.Shell.(Shell.java:354) at org.apache.hadoop.util.StringUtils.(StringUtils.java:80) at org.apache.hadoop.security.SecurityUtil.getAuthenticationMethod(SecurityUtil.java:611) at org.apache.hadoop.security.UserGroupInformation.initialize(UserGroupInformation.java:274) at org.apache.hadoop.security.UserGroupInformation.ensureInitialized(UserGroupInformation.java:262) at org.apache.hadoop.security.UserGroupInformation.loginUserFromSubject(UserGroupInformation.java:807) at org.apache.hadoop.security.UserGroupInformation.getLoginUser(UserGroupInformation.java:777) at org.apache.hadoop.security.UserGroupInformation.getCurrentUser(UserGroupInformation.java:650) at org.apache.spark.util.Utils$.$anonfun$getCurrentUserName$1(Utils.scala:2412) at scala.Option.getOrElse(Option.scala:189) at org.apache.spark.util.Utils$.getCurrentUserName(Utils.scala:2412) at org.apache.spark.SecurityManager.(SecurityManager.scala:79) at org.apache.spark.deploy.SparkSubmit.secMgr$lzycompute$1(SparkSubmit.scala:368) at org.apache.spark.deploy.SparkSubmit.secMgr$1(SparkSubmit.scala:368) at org.apache.spark.deploy.SparkSubmit.$anonfun$prepareSubmitEnvironment$8(SparkSubmit.scala:376) at scala.Option.map(Option.scala:230) at org.apache.spark.deploy.SparkSubmit.prepareSubmitEnvironment(SparkSubmit.scala:376) at org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:871) at org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180) at org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203) at org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90) at org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1007) at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1016) at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala) 20/01/23 18:53:48.288 main DEBUG Shell: Failed to detect a valid hadoop home directory java.io.IOException: HADOOP_HOME or hadoop.home.dir are not set. at org.apache.hadoop.util.Shell.checkHadoopHome(Shell.java:329) at org.apache.hadoop.util.Shell.(Shell.java:354) at org.apache.hadoop.util.StringUtils.(StringUtils.java:80) at org.apache.hadoop.security.SecurityUtil.getAuthenticationMethod(SecurityUtil.java:611) at org.apache.hadoop.security.UserGroupInformation.initialize(UserGroupInformation.java:274) at org.apache.hadoop.security.UserGroupInformation.ensureInitialized(UserGroupInformation.java:262) at org.apache.hadoop.security.UserGroupInformation.loginUserFromSubject(UserGroupInformation.java:807) at org.apache.hadoop.security.UserGroupInformation.getLoginUser(UserGroupInformation.java:777) at org.apache.hadoop.security.UserGroupInformation.getCurrentUser(UserGroupInformation.java:650) at org.apache.spark.util.Utils$.$anonfun$getCurrentUserName$1(Utils.scala:2412) at scala.Option.getOrElse(Option.scala:189) at org.apache.spark.util.Utils$.getCurrentUserName(Utils.scala:2412) at org.apache.spark.SecurityManager.(SecurityManager.scala:79) at org.apache.spark.deploy.SparkSubmit.secMgr$lzycompute$1(SparkSubmit.scala:368) at org.apache.spark.deploy.SparkSubmit.secMgr$1(SparkSubmit.scala:368) at org.apache.spark.deploy.SparkSubmit.$anonfun$prepareSubmitEnvironment$8(SparkSubmit.scala:376) at scala.Option.map(Option.scala:230) at org.apache.spark.deploy.SparkSubmit.prepareSubmitEnvironment(SparkSubmit.scala:376) at org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:871) at org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180) at org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203) at org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90) at org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1007) at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1016) at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala) 20/01/23 18:53:48.304 main DEBUG Shell: setsid exited with exit code 0 20/01/23 18:53:48.305 main DEBUG Shell: setsid exited with exit code 0 20/01/23 18:53:48.308 main DEBUG Shell: setsid exited with exit code 0 20/01/23 18:53:48.310 main DEBUG Shell: setsid exited with exit code 0 20/01/23 18:53:48.317 main DEBUG KerberosName: Kerberos krb5 configuration not found, setting default realm to empty 20/01/23 18:53:48.317 main DEBUG KerberosName: Kerberos krb5 configuration not found, setting default realm to empty 20/01/23 18:53:48.317 main DEBUG KerberosName: Kerberos krb5 configuration not found, setting default realm to empty 20/01/23 18:53:48.317 main DEBUG KerberosName: Kerberos krb5 configuration not found, setting default realm to empty 20/01/23 18:53:48.323 main DEBUG Groups: Creating new Groups object 20/01/23 18:53:48.323 main DEBUG Groups: Creating new Groups object 20/01/23 18:53:48.323 main DEBUG Groups: Creating new Groups object 20/01/23 18:53:48.323 main DEBUG Groups: Creating new Groups object 20/01/23 18:53:48.328 main DEBUG NativeCodeLoader: Trying to load the custom-built native-hadoop library... 20/01/23 18:53:48.328 main DEBUG NativeCodeLoader: Trying to load the custom-built native-hadoop library... 20/01/23 18:53:48.328 main DEBUG NativeCodeLoader: Trying to load the custom-built native-hadoop library... 20/01/23 18:53:48.328 main DEBUG NativeCodeLoader: Trying to load the custom-built native-hadoop library... 20/01/23 18:53:48.330 main DEBUG NativeCodeLoader: Failed to load native-hadoop with error: java.lang.UnsatisfiedLinkError: no hadoop in java.library.path 20/01/23 18:53:48.330 main DEBUG NativeCodeLoader: Failed to load native-hadoop with error: java.lang.UnsatisfiedLinkError: no hadoop in java.library.path 20/01/23 18:53:48.330 main DEBUG NativeCodeLoader: Failed to load native-hadoop with error: java.lang.UnsatisfiedLinkError: no hadoop in java.library.path 20/01/23 18:53:48.330 main DEBUG NativeCodeLoader: java.library.path=/home/jenkins/deps/arrow/arrow-cpp-bin/lib:/usr/java/packages/lib/aarch64:/usr/lib/aarch64-linux-gnu/jni:/lib/aarch64-linux-gnu:/usr/lib/aarch64-linux-gnu:/usr/lib/jni:/lib:/usr/lib 20/01/23 18:53:48.330 main DEBUG NativeCodeLoader: java.library.path=/home/jenkins/deps/arrow/arrow-cpp-bin/lib:/usr/java/packages/lib/aarch64:/usr/lib/aarch64-linux-gnu/jni:/lib/aarch64-linux-gnu:/usr/lib/aarch64-linux-gnu:/usr/lib/jni:/lib:/usr/lib 20/01/23 18:53:48.330 main DEBUG NativeCodeLoader: java.library.path=/home/jenkins/deps/arrow/arrow-cpp-bin/lib:/usr/java/packages/lib/aarch64:/usr/lib/aarch64-linux-gnu/jni:/lib/aarch64-linux-gnu:/usr/lib/aarch64-linux-gnu:/usr/lib/jni:/lib:/usr/lib 20/01/23 18:53:48.330 main WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 20/01/23 18:53:48.330 main WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 20/01/23 18:53:48.330 main WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 20/01/23 18:53:48.331 main DEBUG PerformanceAdvisory: Falling back to shell based 20/01/23 18:53:48.331 main DEBUG PerformanceAdvisory: Falling back to shell based 20/01/23 18:53:48.331 main DEBUG PerformanceAdvisory: Falling back to shell based 20/01/23 18:53:48.331 main DEBUG JniBasedUnixGroupsMappingWithFallback: Group mapping impl=org.apache.hadoop.security.ShellBasedUnixGroupsMapping 20/01/23 18:53:48.331 main DEBUG JniBasedUnixGroupsMappingWithFallback: Group mapping impl=org.apache.hadoop.security.ShellBasedUnixGroupsMapping 20/01/23 18:53:48.331 main DEBUG JniBasedUnixGroupsMappingWithFallback: Group mapping impl=org.apache.hadoop.security.ShellBasedUnixGroupsMapping 20/01/23 18:53:48.330 main DEBUG NativeCodeLoader: Failed to load native-hadoop with error: java.lang.UnsatisfiedLinkError: no hadoop in java.library.path 20/01/23 18:53:48.332 main DEBUG NativeCodeLoader: java.library.path=/home/jenkins/deps/arrow/arrow-cpp-bin/lib:/usr/java/packages/lib/aarch64:/usr/lib/aarch64-linux-gnu/jni:/lib/aarch64-linux-gnu:/usr/lib/aarch64-linux-gnu:/usr/lib/jni:/lib:/usr/lib 20/01/23 18:53:48.332 main WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 20/01/23 18:53:48.333 main DEBUG PerformanceAdvisory: Falling back to shell based 20/01/23 18:53:48.333 main DEBUG JniBasedUnixGroupsMappingWithFallback: Group mapping impl=org.apache.hadoop.security.ShellBasedUnixGroupsMapping 20/01/23 18:53:48.485 main DEBUG Groups: Group mapping impl=org.apache.hadoop.security.JniBasedUnixGroupsMappingWithFallback; cacheTimeout=300000; warningDeltaMs=5000 20/01/23 18:53:48.487 main DEBUG Groups: Group mapping impl=org.apache.hadoop.security.JniBasedUnixGroupsMappingWithFallback; cacheTimeout=300000; warningDeltaMs=5000 20/01/23 18:53:48.488 main DEBUG Groups: Group mapping impl=org.apache.hadoop.security.JniBasedUnixGroupsMappingWithFallback; cacheTimeout=300000; warningDeltaMs=5000 20/01/23 18:53:48.491 main DEBUG Groups: Group mapping impl=org.apache.hadoop.security.JniBasedUnixGroupsMappingWithFallback; cacheTimeout=300000; warningDeltaMs=5000 20/01/23 18:53:48.492 main DEBUG UserGroupInformation: hadoop login 20/01/23 18:53:48.492 main DEBUG UserGroupInformation: hadoop login 20/01/23 18:53:48.492 main DEBUG UserGroupInformation: hadoop login 20/01/23 18:53:48.493 main DEBUG UserGroupInformation: hadoop login commit 20/01/23 18:53:48.493 main DEBUG UserGroupInformation: hadoop login commit 20/01/23 18:53:48.493 main DEBUG UserGroupInformation: hadoop login commit 20/01/23 18:53:48.494 main DEBUG UserGroupInformation: hadoop login 20/01/23 18:53:48.494 main DEBUG UserGroupInformation: hadoop login commit 20/01/23 18:53:48.500 main DEBUG UserGroupInformation: using local user:UnixPrincipal: jenkins 20/01/23 18:53:48.500 main DEBUG UserGroupInformation: using local user:UnixPrincipal: jenkins 20/01/23 18:53:48.500 main DEBUG UserGroupInformation: using local user:UnixPrincipal: jenkins 20/01/23 18:53:48.500 main DEBUG UserGroupInformation: Using user: "UnixPrincipal: jenkins" with name jenkins 20/01/23 18:53:48.501 main DEBUG UserGroupInformation: Using user: "UnixPrincipal: jenkins" with name jenkins 20/01/23 18:53:48.501 main DEBUG UserGroupInformation: Using user: "UnixPrincipal: jenkins" with name jenkins 20/01/23 18:53:48.501 main DEBUG UserGroupInformation: User entry: "jenkins" 20/01/23 18:53:48.501 main DEBUG UserGroupInformation: User entry: "jenkins" 20/01/23 18:53:48.501 main DEBUG UserGroupInformation: User entry: "jenkins" 20/01/23 18:53:48.501 main DEBUG UserGroupInformation: using local user:UnixPrincipal: jenkins 20/01/23 18:53:48.501 main DEBUG UserGroupInformation: Assuming keytab is managed externally since logged in from subject. 20/01/23 18:53:48.501 main DEBUG UserGroupInformation: Assuming keytab is managed externally since logged in from subject. 20/01/23 18:53:48.501 main DEBUG UserGroupInformation: Assuming keytab is managed externally since logged in from subject. 20/01/23 18:53:48.501 main DEBUG UserGroupInformation: Using user: "UnixPrincipal: jenkins" with name jenkins 20/01/23 18:53:48.501 main DEBUG UserGroupInformation: User entry: "jenkins" 20/01/23 18:53:48.501 main DEBUG UserGroupInformation: Assuming keytab is managed externally since logged in from subject. 20/01/23 18:53:48.502 main DEBUG UserGroupInformation: UGI loginUser:jenkins (auth:SIMPLE) 20/01/23 18:53:48.502 main DEBUG UserGroupInformation: UGI loginUser:jenkins (auth:SIMPLE) 20/01/23 18:53:48.502 main DEBUG UserGroupInformation: UGI loginUser:jenkins (auth:SIMPLE) 20/01/23 18:53:48.502 main DEBUG UserGroupInformation: UGI loginUser:jenkins (auth:SIMPLE) 20/01/23 18:53:48.505 main INFO SecurityManager: Changing view acls to: jenkins 20/01/23 18:53:48.505 main INFO SecurityManager: Changing view acls to: jenkins 20/01/23 18:53:48.505 main INFO SecurityManager: Changing view acls to: jenkins 20/01/23 18:53:48.505 main INFO SecurityManager: Changing view acls to: jenkins 20/01/23 18:53:48.505 main INFO SecurityManager: Changing modify acls to: jenkins 20/01/23 18:53:48.505 main INFO SecurityManager: Changing modify acls to: jenkins 20/01/23 18:53:48.506 main INFO SecurityManager: Changing modify acls to: jenkins 20/01/23 18:53:48.506 main INFO SecurityManager: Changing modify acls to: jenkins 20/01/23 18:53:48.506 main INFO SecurityManager: Changing view acls groups to: 20/01/23 18:53:48.506 main INFO SecurityManager: Changing view acls groups to: 20/01/23 18:53:48.506 main INFO SecurityManager: Changing view acls groups to: 20/01/23 18:53:48.506 main INFO SecurityManager: Changing view acls groups to: 20/01/23 18:53:48.506 main INFO SecurityManager: Changing modify acls groups to: 20/01/23 18:53:48.506 main INFO SecurityManager: Changing modify acls groups to: 20/01/23 18:53:48.507 main INFO SecurityManager: Changing modify acls groups to: 20/01/23 18:53:48.507 main INFO SecurityManager: Changing modify acls groups to: 20/01/23 18:53:48.507 main INFO SecurityManager: SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(jenkins); groups with view permissions: Set(); users with modify permissions: Set(jenkins); groups with modify permissions: Set() 20/01/23 18:53:48.507 main INFO SecurityManager: SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(jenkins); groups with view permissions: Set(); users with modify permissions: Set(jenkins); groups with modify permissions: Set() 20/01/23 18:53:48.508 main INFO SecurityManager: SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(jenkins); groups with view permissions: Set(); users with modify permissions: Set(jenkins); groups with modify permissions: Set() 20/01/23 18:53:48.508 main INFO SecurityManager: SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(jenkins); groups with view permissions: Set(); users with modify permissions: Set(jenkins); groups with modify permissions: Set() 20/01/23 18:53:48.832 main DEBUG PythonGatewayServer: Started PythonGatewayServer on port 35897 20/01/23 18:53:48.832 main DEBUG PythonGatewayServer: Started PythonGatewayServer on port 39579 20/01/23 18:53:48.832 main DEBUG PythonGatewayServer: Started PythonGatewayServer on port 41211 20/01/23 18:53:48.840 main DEBUG PythonGatewayServer: Started PythonGatewayServer on port 33809 20/01/23 18:53:49.129 Thread-4 INFO SparkContext: Running Spark version 3.0.0-SNAPSHOT 20/01/23 18:53:49.129 Thread-4 INFO SparkContext: Running Spark version 3.0.0-SNAPSHOT 20/01/23 18:53:49.130 Thread-4 INFO SparkContext: Running Spark version 3.0.0-SNAPSHOT 20/01/23 18:53:49.136 Thread-4 INFO SparkContext: Running Spark version 3.0.0-SNAPSHOT 20/01/23 18:53:49.172 Thread-4 INFO ResourceUtils: ============================================================== 20/01/23 18:53:49.173 Thread-4 INFO ResourceUtils: ============================================================== 20/01/23 18:53:49.174 Thread-4 INFO ResourceUtils: Resources for spark.driver: 20/01/23 18:53:49.174 Thread-4 INFO ResourceUtils: ============================================================== 20/01/23 18:53:49.175 Thread-4 INFO SparkContext: Submitted application: ReusedPySparkTestCase 20/01/23 18:53:49.175 Thread-4 INFO ResourceUtils: ============================================================== 20/01/23 18:53:49.175 Thread-4 INFO ResourceUtils: ============================================================== 20/01/23 18:53:49.175 Thread-4 INFO ResourceUtils: Resources for spark.driver: 20/01/23 18:53:49.176 Thread-4 INFO ResourceUtils: ============================================================== 20/01/23 18:53:49.177 Thread-4 INFO SparkContext: Submitted application: ReusedPySparkTestCase 20/01/23 18:53:49.177 Thread-4 INFO ResourceUtils: Resources for spark.driver: 20/01/23 18:53:49.177 Thread-4 INFO ResourceUtils: Resources for spark.driver: 20/01/23 18:53:49.178 Thread-4 INFO ResourceUtils: ============================================================== 20/01/23 18:53:49.178 Thread-4 INFO ResourceUtils: ============================================================== 20/01/23 18:53:49.178 Thread-4 INFO SparkContext: Submitted application: ReusedPySparkTestCase 20/01/23 18:53:49.178 Thread-4 INFO SparkContext: Submitted application: ReusedPySparkTestCase 20/01/23 18:53:49.284 Thread-4 INFO SecurityManager: Changing view acls to: jenkins 20/01/23 18:53:49.284 Thread-4 INFO SecurityManager: Changing view acls to: jenkins 20/01/23 18:53:49.284 Thread-4 INFO SecurityManager: Changing view acls to: jenkins 20/01/23 18:53:49.284 Thread-4 INFO SecurityManager: Changing modify acls to: jenkins 20/01/23 18:53:49.284 Thread-4 INFO SecurityManager: Changing modify acls to: jenkins 20/01/23 18:53:49.284 Thread-4 INFO SecurityManager: Changing modify acls to: jenkins 20/01/23 18:53:49.284 Thread-4 INFO SecurityManager: Changing view acls groups to: 20/01/23 18:53:49.284 Thread-4 INFO SecurityManager: Changing view acls groups to: 20/01/23 18:53:49.284 Thread-4 INFO SecurityManager: Changing modify acls groups to: 20/01/23 18:53:49.284 Thread-4 INFO SecurityManager: Changing view acls groups to: 20/01/23 18:53:49.284 Thread-4 INFO SecurityManager: Changing modify acls groups to: 20/01/23 18:53:49.285 Thread-4 INFO SecurityManager: SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(jenkins); groups with view permissions: Set(); users with modify permissions: Set(jenkins); groups with modify permissions: Set() 20/01/23 18:53:49.285 Thread-4 INFO SecurityManager: Changing modify acls groups to: 20/01/23 18:53:49.285 Thread-4 INFO SecurityManager: SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(jenkins); groups with view permissions: Set(); users with modify permissions: Set(jenkins); groups with modify permissions: Set() 20/01/23 18:53:49.285 Thread-4 INFO SecurityManager: SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(jenkins); groups with view permissions: Set(); users with modify permissions: Set(jenkins); groups with modify permissions: Set() 20/01/23 18:53:49.287 Thread-4 INFO SecurityManager: Changing view acls to: jenkins 20/01/23 18:53:49.287 Thread-4 INFO SecurityManager: Changing modify acls to: jenkins 20/01/23 18:53:49.287 Thread-4 INFO SecurityManager: Changing view acls groups to: 20/01/23 18:53:49.288 Thread-4 INFO SecurityManager: Changing modify acls groups to: 20/01/23 18:53:49.288 Thread-4 INFO SecurityManager: SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(jenkins); groups with view permissions: Set(); users with modify permissions: Set(jenkins); groups with modify permissions: Set() 20/01/23 18:53:49.834 Thread-4 DEBUG TransportServer: Shuffle server started on port: 36373 20/01/23 18:53:49.834 Thread-4 DEBUG TransportServer: Shuffle server started on port: 42003 20/01/23 18:53:49.836 Thread-4 DEBUG TransportServer: Shuffle server started on port: 33699 20/01/23 18:53:49.841 Thread-4 DEBUG TransportServer: Shuffle server started on port: 45091 20/01/23 18:53:49.850 Thread-4 INFO Utils: Successfully started service 'sparkDriver' on port 42003. 20/01/23 18:53:49.851 Thread-4 INFO Utils: Successfully started service 'sparkDriver' on port 33699. 20/01/23 18:53:49.851 Thread-4 INFO Utils: Successfully started service 'sparkDriver' on port 36373. 20/01/23 18:53:49.852 Thread-4 DEBUG SparkEnv: Using serializer: class org.apache.spark.serializer.JavaSerializer 20/01/23 18:53:49.852 Thread-4 DEBUG SparkEnv: Using serializer: class org.apache.spark.serializer.JavaSerializer 20/01/23 18:53:49.853 Thread-4 DEBUG SparkEnv: Using serializer: class org.apache.spark.serializer.JavaSerializer 20/01/23 18:53:49.857 Thread-4 INFO Utils: Successfully started service 'sparkDriver' on port 45091. 20/01/23 18:53:49.858 Thread-4 DEBUG SparkEnv: Using serializer: class org.apache.spark.serializer.JavaSerializer 20/01/23 18:53:49.902 Thread-4 INFO SparkEnv: Registering MapOutputTracker 20/01/23 18:53:49.902 Thread-4 INFO SparkEnv: Registering MapOutputTracker 20/01/23 18:53:49.902 Thread-4 INFO SparkEnv: Registering MapOutputTracker 20/01/23 18:53:49.903 Thread-4 DEBUG MapOutputTrackerMasterEndpoint: init 20/01/23 18:53:49.903 Thread-4 DEBUG MapOutputTrackerMasterEndpoint: init 20/01/23 18:53:49.903 Thread-4 DEBUG MapOutputTrackerMasterEndpoint: init 20/01/23 18:53:49.903 Thread-4 INFO SparkEnv: Registering MapOutputTracker 20/01/23 18:53:49.904 Thread-4 DEBUG MapOutputTrackerMasterEndpoint: init 20/01/23 18:53:49.950 Thread-4 INFO SparkEnv: Registering BlockManagerMaster 20/01/23 18:53:49.951 Thread-4 INFO SparkEnv: Registering BlockManagerMaster 20/01/23 18:53:49.952 Thread-4 INFO SparkEnv: Registering BlockManagerMaster 20/01/23 18:53:49.954 Thread-4 INFO SparkEnv: Registering BlockManagerMaster 20/01/23 18:53:49.956 Thread-4 INFO BlockManagerMasterEndpoint: Using org.apache.spark.storage.DefaultTopologyMapper for getting topology information 20/01/23 18:53:49.956 Thread-4 INFO BlockManagerMasterEndpoint: Using org.apache.spark.storage.DefaultTopologyMapper for getting topology information 20/01/23 18:53:49.957 Thread-4 INFO BlockManagerMasterEndpoint: BlockManagerMasterEndpoint up 20/01/23 18:53:49.957 Thread-4 INFO BlockManagerMasterEndpoint: BlockManagerMasterEndpoint up 20/01/23 18:53:49.957 Thread-4 INFO BlockManagerMasterEndpoint: Using org.apache.spark.storage.DefaultTopologyMapper for getting topology information 20/01/23 18:53:49.958 Thread-4 INFO BlockManagerMasterEndpoint: BlockManagerMasterEndpoint up 20/01/23 18:53:49.959 Thread-4 INFO BlockManagerMasterEndpoint: Using org.apache.spark.storage.DefaultTopologyMapper for getting topology information 20/01/23 18:53:49.960 Thread-4 INFO BlockManagerMasterEndpoint: BlockManagerMasterEndpoint up 20/01/23 18:53:49.964 Thread-4 INFO SparkEnv: Registering BlockManagerMasterHeartbeat 20/01/23 18:53:49.966 Thread-4 INFO SparkEnv: Registering BlockManagerMasterHeartbeat 20/01/23 18:53:49.971 Thread-4 INFO SparkEnv: Registering BlockManagerMasterHeartbeat 20/01/23 18:53:49.979 Thread-4 INFO SparkEnv: Registering BlockManagerMasterHeartbeat 20/01/23 18:53:49.989 Thread-4 INFO DiskBlockManager: Created local directory at /home/jenkins/workspace/spark-master-test-python-arm/python/target/84b1e1ca-550e-4454-8658-acbf24c425b7/blockmgr-92559d8a-77d0-4a3c-9033-8adb5b379e4b 20/01/23 18:53:49.990 Thread-4 INFO DiskBlockManager: Created local directory at /home/jenkins/workspace/spark-master-test-python-arm/python/target/22a04de0-c639-4801-aead-76fbc7eb088f/blockmgr-c4592cc0-798e-4068-876d-d1c284190584 20/01/23 18:53:49.991 Thread-4 DEBUG DiskBlockManager: Adding shutdown hook 20/01/23 18:53:49.992 Thread-4 DEBUG DiskBlockManager: Adding shutdown hook 20/01/23 18:53:50.002 Thread-4 INFO DiskBlockManager: Created local directory at /home/jenkins/workspace/spark-master-test-python-arm/python/target/3882a34f-ef91-4fff-b528-45b3e0e821f9/blockmgr-3ef7e674-9d9b-4dc2-b16e-3afecc5a7a6f 20/01/23 18:53:50.004 Thread-4 DEBUG DiskBlockManager: Adding shutdown hook 20/01/23 18:53:50.013 Thread-4 INFO DiskBlockManager: Created local directory at /home/jenkins/workspace/spark-master-test-python-arm/python/target/bf3e7ead-50f2-4a0a-86d4-31fa6d898817/blockmgr-fff00d65-05b8-4856-9bc5-060ca5978820 20/01/23 18:53:50.016 Thread-4 DEBUG DiskBlockManager: Adding shutdown hook 20/01/23 18:53:50.047 Thread-4 INFO MemoryStore: MemoryStore started with capacity 366.3 MiB 20/01/23 18:53:50.051 Thread-4 INFO MemoryStore: MemoryStore started with capacity 366.3 MiB 20/01/23 18:53:50.062 Thread-4 INFO MemoryStore: MemoryStore started with capacity 366.3 MiB 20/01/23 18:53:50.069 Thread-4 INFO MemoryStore: MemoryStore started with capacity 366.3 MiB 20/01/23 18:53:50.078 Thread-4 INFO SparkEnv: Registering OutputCommitCoordinator 20/01/23 18:53:50.078 Thread-4 INFO SparkEnv: Registering OutputCommitCoordinator 20/01/23 18:53:50.079 Thread-4 DEBUG OutputCommitCoordinator$OutputCommitCoordinatorEndpoint: init 20/01/23 18:53:50.079 Thread-4 DEBUG OutputCommitCoordinator$OutputCommitCoordinatorEndpoint: init 20/01/23 18:53:50.088 Thread-4 INFO SparkEnv: Registering OutputCommitCoordinator 20/01/23 18:53:50.089 Thread-4 DEBUG OutputCommitCoordinator$OutputCommitCoordinatorEndpoint: init 20/01/23 18:53:50.094 Thread-4 INFO SparkEnv: Registering OutputCommitCoordinator 20/01/23 18:53:50.095 Thread-4 DEBUG OutputCommitCoordinator$OutputCommitCoordinatorEndpoint: init 20/01/23 18:53:50.421 Thread-4 INFO Executor: Starting executor ID driver on host 192.168.1.225 20/01/23 18:53:50.421 Thread-4 INFO Executor: Starting executor ID driver on host 192.168.1.225 20/01/23 18:53:50.422 Thread-4 INFO Executor: Starting executor ID driver on host 192.168.1.225 20/01/23 18:53:50.431 Thread-4 INFO Executor: Starting executor ID driver on host 192.168.1.225 20/01/23 18:53:50.462 Thread-4 DEBUG TransportServer: Shuffle server started on port: 45461 20/01/23 18:53:50.462 Thread-4 INFO Utils: Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 45461. 20/01/23 18:53:50.462 Thread-4 DEBUG TransportServer: Shuffle server started on port: 41233 20/01/23 18:53:50.462 Thread-4 INFO Utils: Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 41233. 20/01/23 18:53:50.463 Thread-4 INFO NettyBlockTransferService: Server created on 192.168.1.225:45461 20/01/23 18:53:50.463 Thread-4 INFO NettyBlockTransferService: Server created on 192.168.1.225:41233 20/01/23 18:53:50.465 Thread-4 INFO BlockManager: Using org.apache.spark.storage.RandomBlockReplicationPolicy for block replication policy 20/01/23 18:53:50.466 Thread-4 DEBUG TransportServer: Shuffle server started on port: 42431 20/01/23 18:53:50.466 Thread-4 INFO Utils: Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 42431. 20/01/23 18:53:50.466 Thread-4 INFO BlockManager: Using org.apache.spark.storage.RandomBlockReplicationPolicy for block replication policy 20/01/23 18:53:50.467 Thread-4 INFO NettyBlockTransferService: Server created on 192.168.1.225:42431 20/01/23 18:53:50.469 Thread-4 INFO BlockManager: Using org.apache.spark.storage.RandomBlockReplicationPolicy for block replication policy 20/01/23 18:53:50.470 Thread-4 DEBUG TransportServer: Shuffle server started on port: 35863 20/01/23 18:53:50.471 Thread-4 INFO Utils: Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 35863. 20/01/23 18:53:50.471 Thread-4 INFO NettyBlockTransferService: Server created on 192.168.1.225:35863 20/01/23 18:53:50.473 Thread-4 INFO BlockManager: Using org.apache.spark.storage.RandomBlockReplicationPolicy for block replication policy 20/01/23 18:53:50.478 Thread-4 INFO BlockManagerMaster: Registering BlockManager BlockManagerId(driver, 192.168.1.225, 41233, None) 20/01/23 18:53:50.479 Thread-4 INFO BlockManagerMaster: Registering BlockManager BlockManagerId(driver, 192.168.1.225, 45461, None) 20/01/23 18:53:50.480 Thread-4 INFO BlockManagerMaster: Registering BlockManager BlockManagerId(driver, 192.168.1.225, 42431, None) 20/01/23 18:53:50.482 dispatcher-BlockManagerMaster DEBUG DefaultTopologyMapper: Got a request for 192.168.1.225 20/01/23 18:53:50.483 dispatcher-BlockManagerMaster DEBUG DefaultTopologyMapper: Got a request for 192.168.1.225 20/01/23 18:53:50.483 dispatcher-BlockManagerMaster DEBUG DefaultTopologyMapper: Got a request for 192.168.1.225 20/01/23 18:53:50.484 dispatcher-BlockManagerMaster INFO BlockManagerMasterEndpoint: Registering block manager 192.168.1.225:41233 with 366.3 MiB RAM, BlockManagerId(driver, 192.168.1.225, 41233, None) 20/01/23 18:53:50.485 Thread-4 INFO BlockManagerMaster: Registering BlockManager BlockManagerId(driver, 192.168.1.225, 35863, None) 20/01/23 18:53:50.485 dispatcher-BlockManagerMaster INFO BlockManagerMasterEndpoint: Registering block manager 192.168.1.225:42431 with 366.3 MiB RAM, BlockManagerId(driver, 192.168.1.225, 42431, None) 20/01/23 18:53:50.484 dispatcher-BlockManagerMaster INFO BlockManagerMasterEndpoint: Registering block manager 192.168.1.225:45461 with 366.3 MiB RAM, BlockManagerId(driver, 192.168.1.225, 45461, None) 20/01/23 18:53:50.489 dispatcher-BlockManagerMaster DEBUG DefaultTopologyMapper: Got a request for 192.168.1.225 20/01/23 18:53:50.490 Thread-4 INFO BlockManagerMaster: Registered BlockManager BlockManagerId(driver, 192.168.1.225, 42431, None) 20/01/23 18:53:50.491 Thread-4 INFO BlockManagerMaster: Registered BlockManager BlockManagerId(driver, 192.168.1.225, 41233, None) 20/01/23 18:53:50.491 dispatcher-BlockManagerMaster INFO BlockManagerMasterEndpoint: Registering block manager 192.168.1.225:35863 with 366.3 MiB RAM, BlockManagerId(driver, 192.168.1.225, 35863, None) 20/01/23 18:53:50.491 Thread-4 INFO BlockManagerMaster: Registered BlockManager BlockManagerId(driver, 192.168.1.225, 45461, None) 20/01/23 18:53:50.492 Thread-4 INFO BlockManager: Initialized BlockManager: BlockManagerId(driver, 192.168.1.225, 42431, None) 20/01/23 18:53:50.492 Thread-4 INFO BlockManager: Initialized BlockManager: BlockManagerId(driver, 192.168.1.225, 41233, None) 20/01/23 18:53:50.493 Thread-4 INFO BlockManager: Initialized BlockManager: BlockManagerId(driver, 192.168.1.225, 45461, None) 20/01/23 18:53:50.495 Thread-4 INFO BlockManagerMaster: Registered BlockManager BlockManagerId(driver, 192.168.1.225, 35863, None) 20/01/23 18:53:50.496 Thread-4 INFO BlockManager: Initialized BlockManager: BlockManagerId(driver, 192.168.1.225, 35863, None) 20/01/23 18:53:50.998 Thread-4 DEBUG log: Logging to org.slf4j.impl.Log4jLoggerAdapter(org.eclipse.jetty.util.log) via org.eclipse.jetty.util.log.Slf4jLog 20/01/23 18:53:51.002 Thread-4 DEBUG log: Logging to org.slf4j.impl.Log4jLoggerAdapter(org.eclipse.jetty.util.log) via org.eclipse.jetty.util.log.Slf4jLog 20/01/23 18:53:51.003 Thread-4 INFO log: Logging initialized @6155ms to org.eclipse.jetty.util.log.Slf4jLog 20/01/23 18:53:51.005 Thread-4 INFO log: Logging initialized @6157ms to org.eclipse.jetty.util.log.Slf4jLog 20/01/23 18:53:51.010 Thread-4 DEBUG log: Logging to org.slf4j.impl.Log4jLoggerAdapter(org.eclipse.jetty.util.log) via org.eclipse.jetty.util.log.Slf4jLog 20/01/23 18:53:51.013 Thread-4 INFO log: Logging initialized @6163ms to org.eclipse.jetty.util.log.Slf4jLog 20/01/23 18:53:51.067 Thread-4 DEBUG DecoratedObjectFactory: Adding Decorator: org.eclipse.jetty.util.DeprecationWarning@457f19ef 20/01/23 18:53:51.069 Thread-4 DEBUG DecoratedObjectFactory: Adding Decorator: org.eclipse.jetty.util.DeprecationWarning@226f4935 20/01/23 18:53:51.073 Thread-4 DEBUG DecoratedObjectFactory: Adding Decorator: org.eclipse.jetty.util.DeprecationWarning@42ccea0a 20/01/23 18:53:51.077 Thread-4 DEBUG log: Logging to org.slf4j.impl.Log4jLoggerAdapter(org.eclipse.jetty.util.log) via org.eclipse.jetty.util.log.Slf4jLog 20/01/23 18:53:51.080 Thread-4 DEBUG ContainerLifeCycle: o.e.j.s.ServletContextHandler@1514ddea{/,null,UNAVAILABLE} added {ServletHandler@44519c7c{STOPPED},MANAGED} 20/01/23 18:53:51.080 Thread-4 INFO log: Logging initialized @6234ms to org.eclipse.jetty.util.log.Slf4jLog 20/01/23 18:53:51.084 Thread-4 DEBUG ContainerLifeCycle: o.e.j.s.ServletContextHandler@5c49b8d5{/,null,UNAVAILABLE} added {ServletHandler@6fd66eac{STOPPED},MANAGED} 20/01/23 18:53:51.093 Thread-4 DEBUG ContainerLifeCycle: o.e.j.s.ServletContextHandler@453f1d63{/,null,UNAVAILABLE} added {ServletHandler@562d985d{STOPPED},MANAGED} 20/01/23 18:53:51.120 Thread-4 DEBUG DecoratedObjectFactory: Adding Decorator: org.eclipse.jetty.util.DeprecationWarning@1f46ad93 20/01/23 18:53:51.129 Thread-4 DEBUG ContainerLifeCycle: o.e.j.s.ServletContextHandler@377c5ae6{/,null,UNAVAILABLE} added {ServletHandler@4a496693{STOPPED},MANAGED} 20/01/23 18:53:51.215 Thread-4 DEBUG SparkContext: Adding shutdown hook 20/01/23 18:53:51.244 Thread-4 DEBUG SparkContext: Adding shutdown hook 20/01/23 18:53:51.251 Thread-4 DEBUG SparkContext: Adding shutdown hook 20/01/23 18:53:51.290 Thread-4 DEBUG SparkContext: Adding shutdown hook 20/01/23 18:53:52.860 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$pythonToJava$1 20/01/23 18:53:52.860 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$pythonToJava$1 20/01/23 18:53:52.875 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$pythonToJava$1 20/01/23 18:53:52.908 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$pythonToJava$1) is now cleaned +++ 20/01/23 18:53:52.913 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$pythonToJava$1) is now cleaned +++ 20/01/23 18:53:52.929 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$toJavaArray$1 20/01/23 18:53:52.929 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$pythonToJava$1) is now cleaned +++ 20/01/23 18:53:52.931 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$toJavaArray$1 20/01/23 18:53:52.940 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$toJavaArray$1) is now cleaned +++ 20/01/23 18:53:52.933 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$toJavaArray$1) is now cleaned +++ 20/01/23 18:53:52.970 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$toJavaArray$1 20/01/23 18:53:52.972 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$toJavaArray$1) is now cleaned +++ 20/01/23 18:53:53.085 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$applySchemaToPythonRDD$1 20/01/23 18:53:53.087 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$applySchemaToPythonRDD$1 20/01/23 18:53:53.089 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$applySchemaToPythonRDD$1 20/01/23 18:53:53.090 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$applySchemaToPythonRDD$1) is now cleaned +++ 20/01/23 18:53:53.091 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$applySchemaToPythonRDD$1) is now cleaned +++ 20/01/23 18:53:53.092 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$applySchemaToPythonRDD$1) is now cleaned +++ 20/01/23 18:53:53.591 Thread-4 INFO SharedState: loading hive config file: file:/home/jenkins/workspace/spark-master-test-python-arm/sql/core/target/scala-2.12/test-classes/hive-site.xml 20/01/23 18:53:53.617 Thread-4 INFO SharedState: loading hive config file: file:/home/jenkins/workspace/spark-master-test-python-arm/sql/core/target/scala-2.12/test-classes/hive-site.xml 20/01/23 18:53:53.672 Thread-4 INFO SharedState: loading hive config file: file:/home/jenkins/workspace/spark-master-test-python-arm/sql/core/target/scala-2.12/test-classes/hive-site.xml 20/01/23 18:53:53.701 Thread-4 INFO SharedState: Setting hive.metastore.warehouse.dir ('null') to the value of spark.sql.warehouse.dir ('file:/home/jenkins/workspace/spark-master-test-python-arm/spark-warehouse'). 20/01/23 18:53:53.701 Thread-4 INFO SharedState: Warehouse path is 'file:/home/jenkins/workspace/spark-master-test-python-arm/spark-warehouse'. 20/01/23 18:53:53.783 Thread-4 INFO SharedState: Setting hive.metastore.warehouse.dir ('null') to the value of spark.sql.warehouse.dir ('file:/home/jenkins/workspace/spark-master-test-python-arm/spark-warehouse'). 20/01/23 18:53:53.785 Thread-4 INFO SharedState: Warehouse path is 'file:/home/jenkins/workspace/spark-master-test-python-arm/spark-warehouse'. 20/01/23 18:53:53.798 Thread-4 INFO SharedState: Setting hive.metastore.warehouse.dir ('null') to the value of spark.sql.warehouse.dir ('file:/home/jenkins/workspace/spark-master-test-python-arm/spark-warehouse'). 20/01/23 18:53:53.799 Thread-4 INFO SharedState: Warehouse path is 'file:/home/jenkins/workspace/spark-master-test-python-arm/spark-warehouse'. 20/01/23 18:53:54.520 Thread-4 INFO SharedState: loading hive config file: file:/home/jenkins/workspace/spark-master-test-python-arm/sql/core/target/scala-2.12/test-classes/hive-site.xml 20/01/23 18:53:54.598 Thread-4 INFO SharedState: Setting hive.metastore.warehouse.dir ('null') to the value of spark.sql.warehouse.dir ('file:/home/jenkins/workspace/spark-master-test-python-arm/spark-warehouse'). 20/01/23 18:53:54.599 Thread-4 INFO SharedState: Warehouse path is 'file:/home/jenkins/workspace/spark-master-test-python-arm/spark-warehouse'. 20/01/23 18:53:56.006 Thread-4 DEBUG CatalystSqlParser: Parsing command: double 20/01/23 18:53:56.425 Thread-4 DEBUG Analyzer$ResolveReferences: Resolving 'input to input#4 20/01/23 18:53:56.426 Thread-4 DEBUG Analyzer$ResolveReferences: Resolving 'output to output#7 20/01/23 18:53:57.110 Thread-4 DEBUG CountVectorizerModel: Input schema: {"type":"struct","fields":[{"name":"id","type":"long","nullable":true,"metadata":{}},{"name":"words","type":{"type":"array","elementType":"string","containsNull":true},"nullable":true,"metadata":{}},{"name":"expected","type":{"type":"udt","class":"org.apache.spark.ml.linalg.VectorUDT","pyClass":"pyspark.ml.linalg.VectorUDT","sqlType":{"type":"struct","fields":[{"name":"type","type":"byte","nullable":false,"metadata":{}},{"name":"size","type":"integer","nullable":true,"metadata":{}},{"name":"indices","type":{"type":"array","elementType":"integer","containsNull":false},"nullable":true,"metadata":{}},{"name":"values","type":{"type":"array","elementType":"double","containsNull":false},"nullable":true,"metadata":{}}]}},"nullable":true,"metadata":{}}]} 20/01/23 18:53:57.170 Thread-4 DEBUG Analyzer$ResolveReferences: Resolving 'prediction to prediction#1 20/01/23 18:53:57.171 Thread-4 DEBUG Analyzer$ResolveReferences: Resolving 'features to features#0 20/01/23 18:53:57.317 Thread-4 DEBUG CountVectorizerModel: Expected output schema: {"type":"struct","fields":[{"name":"id","type":"long","nullable":true,"metadata":{}},{"name":"words","type":{"type":"array","elementType":"string","containsNull":true},"nullable":true,"metadata":{}},{"name":"expected","type":{"type":"udt","class":"org.apache.spark.ml.linalg.VectorUDT","pyClass":"pyspark.ml.linalg.VectorUDT","sqlType":{"type":"struct","fields":[{"name":"type","type":"byte","nullable":false,"metadata":{}},{"name":"size","type":"integer","nullable":true,"metadata":{}},{"name":"indices","type":{"type":"array","elementType":"integer","containsNull":false},"nullable":true,"metadata":{}},{"name":"values","type":{"type":"array","elementType":"double","containsNull":false},"nullable":true,"metadata":{}}]}},"nullable":true,"metadata":{}},{"name":"features","type":{"type":"udt","class":"org.apache.spark.ml.linalg.VectorUDT","pyClass":"pyspark.ml.linalg.VectorUDT","sqlType":{"type":"struct","fields":[{"name":"type","type":"byte","nullable":false,"metadata":{}},{"name":"size","type":"integer","nullable":true,"metadata":{}},{"name":"indices","type":{"type":"array","elementType":"integer","containsNull":false},"nullable":true,"metadata":{}},{"name":"values","type":{"type":"array","elementType":"double","containsNull":false},"nullable":true,"metadata":{}}]}},"nullable":false,"metadata":{"ml_attr":{"attrs":{},"num_attrs":3}}}]} 20/01/23 18:53:57.372 Thread-4 DEBUG Analyzer$ResolveReferences: Resolving 'features to features#4 20/01/23 18:53:57.383 Thread-4 INFO MemoryStore: Block broadcast_0 stored as values in memory (estimated size 256.0 B, free 366.3 MiB) 20/01/23 18:53:57.385 Thread-4 DEBUG BlockManager: Put block broadcast_0 locally took 51 ms 20/01/23 18:53:57.389 Thread-4 DEBUG BlockManager: Putting block broadcast_0 without replication took 55 ms 20/01/23 18:53:57.411 Thread-4 DEBUG Analyzer$ResolveReferences: Resolving 'features to features#4 20/01/23 18:53:57.462 Thread-4 INFO MemoryStore: Block broadcast_0_piece0 stored as bytes in memory (estimated size 264.0 B, free 366.3 MiB) 20/01/23 18:53:57.465 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added broadcast_0_piece0 in memory on 192.168.1.225:41233 (size: 264.0 B, free: 366.3 MiB) 20/01/23 18:53:57.467 Thread-4 DEBUG BlockManagerMaster: Updated info of block broadcast_0_piece0 20/01/23 18:53:57.467 Thread-4 DEBUG BlockManager: Told master about block broadcast_0_piece0 20/01/23 18:53:57.468 Thread-4 DEBUG BlockManager: Put block broadcast_0_piece0 locally took 10 ms 20/01/23 18:53:57.468 Thread-4 DEBUG BlockManager: Putting block broadcast_0_piece0 without replication took 10 ms 20/01/23 18:53:57.474 Thread-4 INFO SparkContext: Created broadcast 0 from broadcast at CountVectorizer.scala:306 20/01/23 18:53:57.485 Thread-4 DEBUG Analyzer$ResolveReferences: Resolving 'user to user#0L 20/01/23 18:53:57.486 Thread-4 DEBUG Analyzer$ResolveReferences: Resolving 'item to item#1L 20/01/23 18:53:57.486 Thread-4 DEBUG Analyzer$ResolveReferences: Resolving 'rating to rating#2 20/01/23 18:53:57.934 Thread-4 DEBUG Analyzer$ResolveReferences: Resolving 'words to words#1 20/01/23 18:53:57.953 Thread-4 DEBUG WholeStageCodegenExec: /* 001 */ public Object generate(Object[] references) { /* 002 */ return new GeneratedIteratorForCodegenStage2(references); /* 003 */ } /* 004 */ /* 005 */ // codegenStageId=2 /* 006 */ final class GeneratedIteratorForCodegenStage2 extends org.apache.spark.sql.execution.BufferedRowIterator { /* 007 */ private Object[] references; /* 008 */ private scala.collection.Iterator[] inputs; /* 009 */ private scala.collection.Iterator inputadapter_input_0; /* 010 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[] project_mutableStateArray_0 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[1]; /* 011 */ /* 012 */ public GeneratedIteratorForCodegenStage2(Object[] references) { /* 013 */ this.references = references; /* 014 */ } /* 015 */ /* 016 */ public void init(int index, scala.collection.Iterator[] inputs) { /* 017 */ partitionIndex = index; /* 018 */ this.inputs = inputs; /* 019 */ inputadapter_input_0 = inputs[0]; /* 020 */ project_mutableStateArray_0[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(2, 0); /* 021 */ /* 022 */ } /* 023 */ /* 024 */ protected void processNext() throws java.io.IOException { /* 025 */ while ( inputadapter_input_0.hasNext()) { /* 026 */ InternalRow inputadapter_row_0 = (InternalRow) inputadapter_input_0.next(); /* 027 */ /* 028 */ long inputadapter_value_0 = inputadapter_row_0.getLong(0); /* 029 */ boolean project_isNull_0 = false; /* 030 */ double project_value_0 = -1.0; /* 031 */ if (!false) { /* 032 */ project_value_0 = (double) inputadapter_value_0; /* 033 */ } /* 034 */ boolean inputadapter_isNull_1 = inputadapter_row_0.isNullAt(1); /* 035 */ double inputadapter_value_1 = inputadapter_isNull_1 ? /* 036 */ -1.0 : (inputadapter_row_0.getDouble(1)); /* 037 */ project_mutableStateArray_0[0].reset(); /* 038 */ /* 039 */ project_mutableStateArray_0[0].zeroOutNullBytes(); /* 040 */ /* 041 */ project_mutableStateArray_0[0].write(0, project_value_0); /* 042 */ /* 043 */ if (inputadapter_isNull_1) { /* 044 */ project_mutableStateArray_0[0].setNullAt(1); /* 045 */ } else { /* 046 */ project_mutableStateArray_0[0].write(1, inputadapter_value_1); /* 047 */ } /* 048 */ append((project_mutableStateArray_0[0].getRow())); /* 049 */ if (shouldStop()) return; /* 050 */ } /* 051 */ } /* 052 */ /* 053 */ } 20/01/23 18:53:58.011 Thread-4 DEBUG CodeGenerator: /* 001 */ public Object generate(Object[] references) { /* 002 */ return new GeneratedIteratorForCodegenStage2(references); /* 003 */ } /* 004 */ /* 005 */ // codegenStageId=2 /* 006 */ final class GeneratedIteratorForCodegenStage2 extends org.apache.spark.sql.execution.BufferedRowIterator { /* 007 */ private Object[] references; /* 008 */ private scala.collection.Iterator[] inputs; /* 009 */ private scala.collection.Iterator inputadapter_input_0; /* 010 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[] project_mutableStateArray_0 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[1]; /* 011 */ /* 012 */ public GeneratedIteratorForCodegenStage2(Object[] references) { /* 013 */ this.references = references; /* 014 */ } /* 015 */ /* 016 */ public void init(int index, scala.collection.Iterator[] inputs) { /* 017 */ partitionIndex = index; /* 018 */ this.inputs = inputs; /* 019 */ inputadapter_input_0 = inputs[0]; /* 020 */ project_mutableStateArray_0[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(2, 0); /* 021 */ /* 022 */ } /* 023 */ /* 024 */ protected void processNext() throws java.io.IOException { /* 025 */ while ( inputadapter_input_0.hasNext()) { /* 026 */ InternalRow inputadapter_row_0 = (InternalRow) inputadapter_input_0.next(); /* 027 */ /* 028 */ long inputadapter_value_0 = inputadapter_row_0.getLong(0); /* 029 */ boolean project_isNull_0 = false; /* 030 */ double project_value_0 = -1.0; /* 031 */ if (!false) { /* 032 */ project_value_0 = (double) inputadapter_value_0; /* 033 */ } /* 034 */ boolean inputadapter_isNull_1 = inputadapter_row_0.isNullAt(1); /* 035 */ double inputadapter_value_1 = inputadapter_isNull_1 ? /* 036 */ -1.0 : (inputadapter_row_0.getDouble(1)); /* 037 */ project_mutableStateArray_0[0].reset(); /* 038 */ /* 039 */ project_mutableStateArray_0[0].zeroOutNullBytes(); /* 040 */ /* 041 */ project_mutableStateArray_0[0].write(0, project_value_0); /* 042 */ /* 043 */ if (inputadapter_isNull_1) { /* 044 */ project_mutableStateArray_0[0].setNullAt(1); /* 045 */ } else { /* 046 */ project_mutableStateArray_0[0].write(1, inputadapter_value_1); /* 047 */ } /* 048 */ append((project_mutableStateArray_0[0].getRow())); /* 049 */ if (shouldStop()) return; /* 050 */ } /* 051 */ } /* 052 */ /* 053 */ } 20/01/23 18:53:58.037 Thread-4 DEBUG Analyzer$ResolveReferences: Resolving 'features to features#7 20/01/23 18:53:58.038 Thread-4 DEBUG Analyzer$ResolveReferences: Resolving 'expected to expected#2 20/01/23 18:53:58.288 Thread-4 DEBUG WholeStageCodegenExec: /* 001 */ public Object generate(Object[] references) { /* 002 */ return new GeneratedIteratorForCodegenStage1(references); /* 003 */ } /* 004 */ /* 005 */ // codegenStageId=1 /* 006 */ final class GeneratedIteratorForCodegenStage1 extends org.apache.spark.sql.execution.BufferedRowIterator { /* 007 */ private Object[] references; /* 008 */ private scala.collection.Iterator[] inputs; /* 009 */ private scala.collection.Iterator rdd_input_0; /* 010 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[] rdd_mutableStateArray_0 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[4]; /* 011 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[] rdd_mutableStateArray_1 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[4]; /* 012 */ /* 013 */ public GeneratedIteratorForCodegenStage1(Object[] references) { /* 014 */ this.references = references; /* 015 */ } /* 016 */ /* 017 */ public void init(int index, scala.collection.Iterator[] inputs) { /* 018 */ partitionIndex = index; /* 019 */ this.inputs = inputs; /* 020 */ rdd_input_0 = inputs[0]; /* 021 */ rdd_mutableStateArray_0[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(2, 32); /* 022 */ rdd_mutableStateArray_0[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(rdd_mutableStateArray_0[0], 4); /* 023 */ rdd_mutableStateArray_1[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[1], 4); /* 024 */ rdd_mutableStateArray_1[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[1], 8); /* 025 */ rdd_mutableStateArray_0[2] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(1, 32); /* 026 */ rdd_mutableStateArray_0[3] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(rdd_mutableStateArray_0[2], 4); /* 027 */ rdd_mutableStateArray_1[2] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[3], 4); /* 028 */ rdd_mutableStateArray_1[3] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[3], 8); /* 029 */ /* 030 */ } /* 031 */ /* 032 */ private void project_writeFields_0_1(InternalRow project_tmpInput_0) { /* 033 */ if ((project_tmpInput_0.isNullAt(3))) { /* 034 */ rdd_mutableStateArray_0[3].setNullAt(3); /* 035 */ } else { /* 036 */ // Remember the current cursor so that we can calculate how many bytes are /* 037 */ // written later. /* 038 */ final int project_previousCursor_2 = rdd_mutableStateArray_0[3].cursor(); /* 039 */ /* 040 */ final ArrayData project_tmpInput_2 = (project_tmpInput_0.getArray(3)); /* 041 */ if (project_tmpInput_2 instanceof UnsafeArrayData) { /* 042 */ rdd_mutableStateArray_0[3].write((UnsafeArrayData) project_tmpInput_2); /* 043 */ } else { /* 044 */ final int project_numElements_1 = project_tmpInput_2.numElements(); /* 045 */ rdd_mutableStateArray_1[3].initialize(project_numElements_1); /* 046 */ /* 047 */ for (int project_index_1 = 0; project_index_1 < project_numElements_1; project_index_1++) { /* 048 */ rdd_mutableStateArray_1[3].write(project_index_1, project_tmpInput_2.getDouble(project_index_1)); /* 049 */ } /* 050 */ } /* 051 */ /* 052 */ rdd_mutableStateArray_0[3].setOffsetAndSizeFromPreviousCursor(3, project_previousCursor_2); /* 053 */ } /* 054 */ /* 055 */ } /* 056 */ /* 057 */ private void project_writeFields_0_0(InternalRow project_tmpInput_0) { /* 058 */ rdd_mutableStateArray_0[3].write(0, (project_tmpInput_0.getByte(0))); /* 059 */ /* 060 */ if ((project_tmpInput_0.isNullAt(1))) { /* 061 */ rdd_mutableStateArray_0[3].setNullAt(1); /* 062 */ } else { /* 063 */ rdd_mutableStateArray_0[3].write(1, (project_tmpInput_0.getInt(1))); /* 064 */ } /* 065 */ /* 066 */ if ((project_tmpInput_0.isNullAt(2))) { /* 067 */ rdd_mutableStateArray_0[3].setNullAt(2); /* 068 */ } else { /* 069 */ // Remember the current cursor so that we can calculate how many bytes are /* 070 */ // written later. /* 071 */ final int project_previousCursor_1 = rdd_mutableStateArray_0[3].cursor(); /* 072 */ /* 073 */ final ArrayData project_tmpInput_1 = (project_tmpInput_0.getArray(2)); /* 074 */ if (project_tmpInput_1 instanceof UnsafeArrayData) { /* 075 */ rdd_mutableStateArray_0[3].write((UnsafeArrayData) project_tmpInput_1); /* 076 */ } else { /* 077 */ final int project_numElements_0 = project_tmpInput_1.numElements(); /* 078 */ rdd_mutableStateArray_1[2].initialize(project_numElements_0); /* 079 */ /* 080 */ for (int project_index_0 = 0; project_index_0 < project_numElements_0; project_index_0++) { /* 081 */ rdd_mutableStateArray_1[2].write(project_index_0, project_tmpInput_1.getInt(project_index_0)); /* 082 */ } /* 083 */ } /* 084 */ /* 085 */ rdd_mutableStateArray_0[3].setOffsetAndSizeFromPreviousCursor(2, project_previousCursor_1); /* 086 */ } /* 087 */ /* 088 */ } /* 089 */ /* 090 */ protected void processNext() throws java.io.IOException { /* 091 */ while ( rdd_input_0.hasNext()) { /* 092 */ InternalRow rdd_row_0 = (InternalRow) rdd_input_0.next(); /* 093 */ ((org.apache.spark.sql.execution.metric.SQLMetric) references[0] /* numOutputRows */).add(1); /* 094 */ boolean rdd_isNull_0 = rdd_row_0.isNullAt(0); /* 095 */ InternalRow rdd_value_0 = rdd_isNull_0 ? /* 096 */ null : (rdd_row_0.getStruct(0, 4)); /* 097 */ rdd_mutableStateArray_0[2].reset(); /* 098 */ /* 099 */ rdd_mutableStateArray_0[2].zeroOutNullBytes(); /* 100 */ /* 101 */ if (rdd_isNull_0) { /* 102 */ rdd_mutableStateArray_0[2].setNullAt(0); /* 103 */ } else { /* 104 */ final InternalRow project_tmpInput_0 = rdd_value_0; /* 105 */ if (project_tmpInput_0 instanceof UnsafeRow) { /* 106 */ rdd_mutableStateArray_0[2].write(0, (UnsafeRow) project_tmpInput_0); /* 107 */ } else { /* 108 */ // Remember the current cursor so that we can calculate how many bytes are /* 109 */ // written later. /* 110 */ final int project_previousCursor_0 = rdd_mutableStateArray_0[2].cursor(); /* 111 */ /* 112 */ rdd_mutableStateArray_0[3].resetRowWriter(); /* 113 */ project_writeFields_0_0(project_tmpInput_0); /* 114 */ project_writeFields_0_1(project_tmpInput_0); /* 115 */ /* 116 */ rdd_mutableStateArray_0[2].setOffsetAndSizeFromPreviousCursor(0, project_previousCursor_0); /* 117 */ } /* 118 */ } /* 119 */ append((rdd_mutableStateArray_0[2].getRow())); /* 120 */ if (shouldStop()) return; /* 121 */ } /* 122 */ } /* 123 */ /* 124 */ } 20/01/23 18:53:58.346 Thread-4 DEBUG CodeGenerator: /* 001 */ public Object generate(Object[] references) { /* 002 */ return new GeneratedIteratorForCodegenStage1(references); /* 003 */ } /* 004 */ /* 005 */ // codegenStageId=1 /* 006 */ final class GeneratedIteratorForCodegenStage1 extends org.apache.spark.sql.execution.BufferedRowIterator { /* 007 */ private Object[] references; /* 008 */ private scala.collection.Iterator[] inputs; /* 009 */ private scala.collection.Iterator rdd_input_0; /* 010 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[] rdd_mutableStateArray_0 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[4]; /* 011 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[] rdd_mutableStateArray_1 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[4]; /* 012 */ /* 013 */ public GeneratedIteratorForCodegenStage1(Object[] references) { /* 014 */ this.references = references; /* 015 */ } /* 016 */ /* 017 */ public void init(int index, scala.collection.Iterator[] inputs) { /* 018 */ partitionIndex = index; /* 019 */ this.inputs = inputs; /* 020 */ rdd_input_0 = inputs[0]; /* 021 */ rdd_mutableStateArray_0[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(2, 32); /* 022 */ rdd_mutableStateArray_0[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(rdd_mutableStateArray_0[0], 4); /* 023 */ rdd_mutableStateArray_1[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[1], 4); /* 024 */ rdd_mutableStateArray_1[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[1], 8); /* 025 */ rdd_mutableStateArray_0[2] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(1, 32); /* 026 */ rdd_mutableStateArray_0[3] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(rdd_mutableStateArray_0[2], 4); /* 027 */ rdd_mutableStateArray_1[2] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[3], 4); /* 028 */ rdd_mutableStateArray_1[3] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[3], 8); /* 029 */ /* 030 */ } /* 031 */ /* 032 */ private void project_writeFields_0_1(InternalRow project_tmpInput_0) { /* 033 */ if ((project_tmpInput_0.isNullAt(3))) { /* 034 */ rdd_mutableStateArray_0[3].setNullAt(3); /* 035 */ } else { /* 036 */ // Remember the current cursor so that we can calculate how many bytes are /* 037 */ // written later. /* 038 */ final int project_previousCursor_2 = rdd_mutableStateArray_0[3].cursor(); /* 039 */ /* 040 */ final ArrayData project_tmpInput_2 = (project_tmpInput_0.getArray(3)); /* 041 */ if (project_tmpInput_2 instanceof UnsafeArrayData) { /* 042 */ rdd_mutableStateArray_0[3].write((UnsafeArrayData) project_tmpInput_2); /* 043 */ } else { /* 044 */ final int project_numElements_1 = project_tmpInput_2.numElements(); /* 045 */ rdd_mutableStateArray_1[3].initialize(project_numElements_1); /* 046 */ /* 047 */ for (int project_index_1 = 0; project_index_1 < project_numElements_1; project_index_1++) { /* 048 */ rdd_mutableStateArray_1[3].write(project_index_1, project_tmpInput_2.getDouble(project_index_1)); /* 049 */ } /* 050 */ } /* 051 */ /* 052 */ rdd_mutableStateArray_0[3].setOffsetAndSizeFromPreviousCursor(3, project_previousCursor_2); /* 053 */ } /* 054 */ /* 055 */ } /* 056 */ /* 057 */ private void project_writeFields_0_0(InternalRow project_tmpInput_0) { /* 058 */ rdd_mutableStateArray_0[3].write(0, (project_tmpInput_0.getByte(0))); /* 059 */ /* 060 */ if ((project_tmpInput_0.isNullAt(1))) { /* 061 */ rdd_mutableStateArray_0[3].setNullAt(1); /* 062 */ } else { /* 063 */ rdd_mutableStateArray_0[3].write(1, (project_tmpInput_0.getInt(1))); /* 064 */ } /* 065 */ /* 066 */ if ((project_tmpInput_0.isNullAt(2))) { /* 067 */ rdd_mutableStateArray_0[3].setNullAt(2); /* 068 */ } else { /* 069 */ // Remember the current cursor so that we can calculate how many bytes are /* 070 */ // written later. /* 071 */ final int project_previousCursor_1 = rdd_mutableStateArray_0[3].cursor(); /* 072 */ /* 073 */ final ArrayData project_tmpInput_1 = (project_tmpInput_0.getArray(2)); /* 074 */ if (project_tmpInput_1 instanceof UnsafeArrayData) { /* 075 */ rdd_mutableStateArray_0[3].write((UnsafeArrayData) project_tmpInput_1); /* 076 */ } else { /* 077 */ final int project_numElements_0 = project_tmpInput_1.numElements(); /* 078 */ rdd_mutableStateArray_1[2].initialize(project_numElements_0); /* 079 */ /* 080 */ for (int project_index_0 = 0; project_index_0 < project_numElements_0; project_index_0++) { /* 081 */ rdd_mutableStateArray_1[2].write(project_index_0, project_tmpInput_1.getInt(project_index_0)); /* 082 */ } /* 083 */ } /* 084 */ /* 085 */ rdd_mutableStateArray_0[3].setOffsetAndSizeFromPreviousCursor(2, project_previousCursor_1); /* 086 */ } /* 087 */ /* 088 */ } /* 089 */ /* 090 */ protected void processNext() throws java.io.IOException { /* 091 */ while ( rdd_input_0.hasNext()) { /* 092 */ InternalRow rdd_row_0 = (InternalRow) rdd_input_0.next(); /* 093 */ ((org.apache.spark.sql.execution.metric.SQLMetric) references[0] /* numOutputRows */).add(1); /* 094 */ boolean rdd_isNull_0 = rdd_row_0.isNullAt(0); /* 095 */ InternalRow rdd_value_0 = rdd_isNull_0 ? /* 096 */ null : (rdd_row_0.getStruct(0, 4)); /* 097 */ rdd_mutableStateArray_0[2].reset(); /* 098 */ /* 099 */ rdd_mutableStateArray_0[2].zeroOutNullBytes(); /* 100 */ /* 101 */ if (rdd_isNull_0) { /* 102 */ rdd_mutableStateArray_0[2].setNullAt(0); /* 103 */ } else { /* 104 */ final InternalRow project_tmpInput_0 = rdd_value_0; /* 105 */ if (project_tmpInput_0 instanceof UnsafeRow) { /* 106 */ rdd_mutableStateArray_0[2].write(0, (UnsafeRow) project_tmpInput_0); /* 107 */ } else { /* 108 */ // Remember the current cursor so that we can calculate how many bytes are /* 109 */ // written later. /* 110 */ final int project_previousCursor_0 = rdd_mutableStateArray_0[2].cursor(); /* 111 */ /* 112 */ rdd_mutableStateArray_0[3].resetRowWriter(); /* 113 */ project_writeFields_0_0(project_tmpInput_0); /* 114 */ project_writeFields_0_1(project_tmpInput_0); /* 115 */ /* 116 */ rdd_mutableStateArray_0[2].setOffsetAndSizeFromPreviousCursor(0, project_previousCursor_0); /* 117 */ } /* 118 */ } /* 119 */ append((rdd_mutableStateArray_0[2].getRow())); /* 120 */ if (shouldStop()) return; /* 121 */ } /* 122 */ } /* 123 */ /* 124 */ } 20/01/23 18:53:58.362 Thread-4 DEBUG WholeStageCodegenExec: /* 001 */ public Object generate(Object[] references) { /* 002 */ return new GeneratedIteratorForCodegenStage1(references); /* 003 */ } /* 004 */ /* 005 */ // codegenStageId=1 /* 006 */ final class GeneratedIteratorForCodegenStage1 extends org.apache.spark.sql.execution.BufferedRowIterator { /* 007 */ private Object[] references; /* 008 */ private scala.collection.Iterator[] inputs; /* 009 */ private scala.collection.Iterator rdd_input_0; /* 010 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[] rdd_mutableStateArray_0 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[2]; /* 011 */ /* 012 */ public GeneratedIteratorForCodegenStage1(Object[] references) { /* 013 */ this.references = references; /* 014 */ } /* 015 */ /* 016 */ public void init(int index, scala.collection.Iterator[] inputs) { /* 017 */ partitionIndex = index; /* 018 */ this.inputs = inputs; /* 019 */ rdd_input_0 = inputs[0]; /* 020 */ rdd_mutableStateArray_0[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(3, 0); /* 021 */ rdd_mutableStateArray_0[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(3, 0); /* 022 */ /* 023 */ } /* 024 */ /* 025 */ protected void processNext() throws java.io.IOException { /* 026 */ while ( rdd_input_0.hasNext()) { /* 027 */ InternalRow rdd_row_0 = (InternalRow) rdd_input_0.next(); /* 028 */ ((org.apache.spark.sql.execution.metric.SQLMetric) references[0] /* numOutputRows */).add(1); /* 029 */ boolean rdd_isNull_0 = rdd_row_0.isNullAt(0); /* 030 */ long rdd_value_0 = rdd_isNull_0 ? /* 031 */ -1L : (rdd_row_0.getLong(0)); /* 032 */ /* 033 */ Long project_conv_0 = rdd_value_0; /* 034 */ Object project_arg_0 = rdd_isNull_0 ? null : project_conv_0; /* 035 */ /* 036 */ Integer project_result_0 = null; /* 037 */ try { /* 038 */ project_result_0 = (Integer)((scala.Function1) references[3] /* udf */).apply(project_arg_0); /* 039 */ } catch (Exception e) { /* 040 */ throw new org.apache.spark.SparkException(((java.lang.String) references[2] /* errMsg */), e); /* 041 */ } /* 042 */ /* 043 */ boolean project_isNull_0 = project_result_0 == null; /* 044 */ int project_value_0 = -1; /* 045 */ if (!project_isNull_0) { /* 046 */ project_value_0 = project_result_0; /* 047 */ } /* 048 */ boolean rdd_isNull_1 = rdd_row_0.isNullAt(1); /* 049 */ long rdd_value_1 = rdd_isNull_1 ? /* 050 */ -1L : (rdd_row_0.getLong(1)); /* 051 */ /* 052 */ Long project_conv_1 = rdd_value_1; /* 053 */ Object project_arg_1 = rdd_isNull_1 ? null : project_conv_1; /* 054 */ /* 055 */ Integer project_result_1 = null; /* 056 */ try { /* 057 */ project_result_1 = (Integer)((scala.Function1) references[6] /* udf */).apply(project_arg_1); /* 058 */ } catch (Exception e) { /* 059 */ throw new org.apache.spark.SparkException(((java.lang.String) references[5] /* errMsg */), e); /* 060 */ } /* 061 */ /* 062 */ boolean project_isNull_2 = project_result_1 == null; /* 063 */ int project_value_2 = -1; /* 064 */ if (!project_isNull_2) { /* 065 */ project_value_2 = project_result_1; /* 066 */ } /* 067 */ boolean rdd_isNull_2 = rdd_row_0.isNullAt(2); /* 068 */ double rdd_value_2 = rdd_isNull_2 ? /* 069 */ -1.0 : (rdd_row_0.getDouble(2)); /* 070 */ boolean project_isNull_4 = rdd_isNull_2; /* 071 */ float project_value_4 = -1.0f; /* 072 */ if (!rdd_isNull_2) { /* 073 */ project_value_4 = (float) rdd_value_2; /* 074 */ } /* 075 */ rdd_mutableStateArray_0[1].reset(); /* 076 */ /* 077 */ rdd_mutableStateArray_0[1].zeroOutNullBytes(); /* 078 */ /* 079 */ rdd_mutableStateArray_0[1].write(0, project_value_0); /* 080 */ /* 081 */ rdd_mutableStateArray_0[1].write(1, project_value_2); /* 082 */ /* 083 */ if (project_isNull_4) { /* 084 */ rdd_mutableStateArray_0[1].setNullAt(2); /* 085 */ } else { /* 086 */ rdd_mutableStateArray_0[1].write(2, project_value_4); /* 087 */ } /* 088 */ append((rdd_mutableStateArray_0[1].getRow())); /* 089 */ if (shouldStop()) return; /* 090 */ } /* 091 */ } /* 092 */ /* 093 */ } 20/01/23 18:53:58.396 Thread-4 DEBUG CodeGenerator: /* 001 */ public Object generate(Object[] references) { /* 002 */ return new GeneratedIteratorForCodegenStage1(references); /* 003 */ } /* 004 */ /* 005 */ // codegenStageId=1 /* 006 */ final class GeneratedIteratorForCodegenStage1 extends org.apache.spark.sql.execution.BufferedRowIterator { /* 007 */ private Object[] references; /* 008 */ private scala.collection.Iterator[] inputs; /* 009 */ private scala.collection.Iterator rdd_input_0; /* 010 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[] rdd_mutableStateArray_0 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[2]; /* 011 */ /* 012 */ public GeneratedIteratorForCodegenStage1(Object[] references) { /* 013 */ this.references = references; /* 014 */ } /* 015 */ /* 016 */ public void init(int index, scala.collection.Iterator[] inputs) { /* 017 */ partitionIndex = index; /* 018 */ this.inputs = inputs; /* 019 */ rdd_input_0 = inputs[0]; /* 020 */ rdd_mutableStateArray_0[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(3, 0); /* 021 */ rdd_mutableStateArray_0[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(3, 0); /* 022 */ /* 023 */ } /* 024 */ /* 025 */ protected void processNext() throws java.io.IOException { /* 026 */ while ( rdd_input_0.hasNext()) { /* 027 */ InternalRow rdd_row_0 = (InternalRow) rdd_input_0.next(); /* 028 */ ((org.apache.spark.sql.execution.metric.SQLMetric) references[0] /* numOutputRows */).add(1); /* 029 */ boolean rdd_isNull_0 = rdd_row_0.isNullAt(0); /* 030 */ long rdd_value_0 = rdd_isNull_0 ? /* 031 */ -1L : (rdd_row_0.getLong(0)); /* 032 */ /* 033 */ Long project_conv_0 = rdd_value_0; /* 034 */ Object project_arg_0 = rdd_isNull_0 ? null : project_conv_0; /* 035 */ /* 036 */ Integer project_result_0 = null; /* 037 */ try { /* 038 */ project_result_0 = (Integer)((scala.Function1) references[3] /* udf */).apply(project_arg_0); /* 039 */ } catch (Exception e) { /* 040 */ throw new org.apache.spark.SparkException(((java.lang.String) references[2] /* errMsg */), e); /* 041 */ } /* 042 */ /* 043 */ boolean project_isNull_0 = project_result_0 == null; /* 044 */ int project_value_0 = -1; /* 045 */ if (!project_isNull_0) { /* 046 */ project_value_0 = project_result_0; /* 047 */ } /* 048 */ boolean rdd_isNull_1 = rdd_row_0.isNullAt(1); /* 049 */ long rdd_value_1 = rdd_isNull_1 ? /* 050 */ -1L : (rdd_row_0.getLong(1)); /* 051 */ /* 052 */ Long project_conv_1 = rdd_value_1; /* 053 */ Object project_arg_1 = rdd_isNull_1 ? null : project_conv_1; /* 054 */ /* 055 */ Integer project_result_1 = null; /* 056 */ try { /* 057 */ project_result_1 = (Integer)((scala.Function1) references[6] /* udf */).apply(project_arg_1); /* 058 */ } catch (Exception e) { /* 059 */ throw new org.apache.spark.SparkException(((java.lang.String) references[5] /* errMsg */), e); /* 060 */ } /* 061 */ /* 062 */ boolean project_isNull_2 = project_result_1 == null; /* 063 */ int project_value_2 = -1; /* 064 */ if (!project_isNull_2) { /* 065 */ project_value_2 = project_result_1; /* 066 */ } /* 067 */ boolean rdd_isNull_2 = rdd_row_0.isNullAt(2); /* 068 */ double rdd_value_2 = rdd_isNull_2 ? /* 069 */ -1.0 : (rdd_row_0.getDouble(2)); /* 070 */ boolean project_isNull_4 = rdd_isNull_2; /* 071 */ float project_value_4 = -1.0f; /* 072 */ if (!rdd_isNull_2) { /* 073 */ project_value_4 = (float) rdd_value_2; /* 074 */ } /* 075 */ rdd_mutableStateArray_0[1].reset(); /* 076 */ /* 077 */ rdd_mutableStateArray_0[1].zeroOutNullBytes(); /* 078 */ /* 079 */ rdd_mutableStateArray_0[1].write(0, project_value_0); /* 080 */ /* 081 */ rdd_mutableStateArray_0[1].write(1, project_value_2); /* 082 */ /* 083 */ if (project_isNull_4) { /* 084 */ rdd_mutableStateArray_0[1].setNullAt(2); /* 085 */ } else { /* 086 */ rdd_mutableStateArray_0[1].write(2, project_value_4); /* 087 */ } /* 088 */ append((rdd_mutableStateArray_0[1].getRow())); /* 089 */ if (shouldStop()) return; /* 090 */ } /* 091 */ } /* 092 */ /* 093 */ } 20/01/23 18:53:58.438 Thread-4 INFO CodeGenerator: Code generated in 483.388794 ms 20/01/23 18:53:58.459 Thread-4 DEBUG WholeStageCodegenExec: /* 001 */ public Object generate(Object[] references) { /* 002 */ return new GeneratedIteratorForCodegenStage1(references); /* 003 */ } /* 004 */ /* 005 */ // codegenStageId=1 /* 006 */ final class GeneratedIteratorForCodegenStage1 extends org.apache.spark.sql.execution.BufferedRowIterator { /* 007 */ private Object[] references; /* 008 */ private scala.collection.Iterator[] inputs; /* 009 */ private boolean range_initRange_0; /* 010 */ private long range_nextIndex_0; /* 011 */ private TaskContext range_taskContext_0; /* 012 */ private InputMetrics range_inputMetrics_0; /* 013 */ private long range_batchEnd_0; /* 014 */ private long range_numElementsTodo_0; /* 015 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[] range_mutableStateArray_0 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[1]; /* 016 */ /* 017 */ public GeneratedIteratorForCodegenStage1(Object[] references) { /* 018 */ this.references = references; /* 019 */ } /* 020 */ /* 021 */ public void init(int index, scala.collection.Iterator[] inputs) { /* 022 */ partitionIndex = index; /* 023 */ this.inputs = inputs; /* 024 */ /* 025 */ range_taskContext_0 = TaskContext.get(); /* 026 */ range_inputMetrics_0 = range_taskContext_0.taskMetrics().inputMetrics(); /* 027 */ range_mutableStateArray_0[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(1, 0); /* 028 */ /* 029 */ } /* 030 */ /* 031 */ private void initRange(int idx) { /* 032 */ java.math.BigInteger index = java.math.BigInteger.valueOf(idx); /* 033 */ java.math.BigInteger numSlice = java.math.BigInteger.valueOf(4L); /* 034 */ java.math.BigInteger numElement = java.math.BigInteger.valueOf(10L); /* 035 */ java.math.BigInteger step = java.math.BigInteger.valueOf(1L); /* 036 */ java.math.BigInteger start = java.math.BigInteger.valueOf(0L); /* 037 */ long partitionEnd; /* 038 */ /* 039 */ java.math.BigInteger st = index.multiply(numElement).divide(numSlice).multiply(step).add(start); /* 040 */ if (st.compareTo(java.math.BigInteger.valueOf(Long.MAX_VALUE)) > 0) { /* 041 */ range_nextIndex_0 = Long.MAX_VALUE; /* 042 */ } else if (st.compareTo(java.math.BigInteger.valueOf(Long.MIN_VALUE)) < 0) { /* 043 */ range_nextIndex_0 = Long.MIN_VALUE; /* 044 */ } else { /* 045 */ range_nextIndex_0 = st.longValue(); /* 046 */ } /* 047 */ range_batchEnd_0 = range_nextIndex_0; /* 048 */ /* 049 */ java.math.BigInteger end = index.add(java.math.BigInteger.ONE).multiply(numElement).divide(numSlice) /* 050 */ .multiply(step).add(start); /* 051 */ if (end.compareTo(java.math.BigInteger.valueOf(Long.MAX_VALUE)) > 0) { /* 052 */ partitionEnd = Long.MAX_VALUE; /* 053 */ } else if (end.compareTo(java.math.BigInteger.valueOf(Long.MIN_VALUE)) < 0) { /* 054 */ partitionEnd = Long.MIN_VALUE; /* 055 */ } else { /* 056 */ partitionEnd = end.longValue(); /* 057 */ } /* 058 */ /* 059 */ java.math.BigInteger startToEnd = java.math.BigInteger.valueOf(partitionEnd).subtract( /* 060 */ java.math.BigInteger.valueOf(range_nextIndex_0)); /* 061 */ range_numElementsTodo_0 = startToEnd.divide(step).longValue(); /* 062 */ if (range_numElementsTodo_0 < 0) { /* 063 */ range_numElementsTodo_0 = 0; /* 064 */ } else if (startToEnd.remainder(step).compareTo(java.math.BigInteger.valueOf(0L)) != 0) { /* 065 */ range_numElementsTodo_0++; /* 066 */ } /* 067 */ } /* 068 */ /* 069 */ protected void processNext() throws java.io.IOException { /* 070 */ // initialize Range /* 071 */ if (!range_initRange_0) { /* 072 */ range_initRange_0 = true; /* 073 */ initRange(partitionIndex); /* 074 */ } /* 075 */ /* 076 */ while (true) { /* 077 */ if (range_nextIndex_0 == range_batchEnd_0) { /* 078 */ long range_nextBatchTodo_0; /* 079 */ if (range_numElementsTodo_0 > 1000L) { /* 080 */ range_nextBatchTodo_0 = 1000L; /* 081 */ range_numElementsTodo_0 -= 1000L; /* 082 */ } else { /* 083 */ range_nextBatchTodo_0 = range_numElementsTodo_0; /* 084 */ range_numElementsTodo_0 = 0; /* 085 */ if (range_nextBatchTodo_0 == 0) break; /* 086 */ } /* 087 */ range_batchEnd_0 += range_nextBatchTodo_0 * 1L; /* 088 */ } /* 089 */ /* 090 */ int range_localEnd_0 = (int)((range_batchEnd_0 - range_nextIndex_0) / 1L); /* 091 */ for (int range_localIdx_0 = 0; range_localIdx_0 < range_localEnd_0; range_localIdx_0++) { /* 092 */ long range_value_0 = ((long)range_localIdx_0 * 1L) + range_nextIndex_0; /* 093 */ /* 094 */ range_mutableStateArray_0[0].reset(); /* 095 */ /* 096 */ range_mutableStateArray_0[0].write(0, range_value_0); /* 097 */ append((range_mutableStateArray_0[0].getRow())); /* 098 */ /* 099 */ if (shouldStop()) { /* 100 */ range_nextIndex_0 = range_value_0 + 1L; /* 101 */ ((org.apache.spark.sql.execution.metric.SQLMetric) references[0] /* numOutputRows */).add(range_localIdx_0 + 1); /* 102 */ range_inputMetrics_0.incRecordsRead(range_localIdx_0 + 1); /* 103 */ return; /* 104 */ } /* 105 */ /* 106 */ } /* 107 */ range_nextIndex_0 = range_batchEnd_0; /* 108 */ ((org.apache.spark.sql.execution.metric.SQLMetric) references[0] /* numOutputRows */).add(range_localEnd_0); /* 109 */ range_inputMetrics_0.incRecordsRead(range_localEnd_0); /* 110 */ range_taskContext_0.killTaskIfInterrupted(); /* 111 */ } /* 112 */ } /* 113 */ /* 114 */ } 20/01/23 18:53:58.474 Thread-4 DEBUG CodeGenerator: /* 001 */ public Object generate(Object[] references) { /* 002 */ return new GeneratedIteratorForCodegenStage1(references); /* 003 */ } /* 004 */ /* 005 */ // codegenStageId=1 /* 006 */ final class GeneratedIteratorForCodegenStage1 extends org.apache.spark.sql.execution.BufferedRowIterator { /* 007 */ private Object[] references; /* 008 */ private scala.collection.Iterator[] inputs; /* 009 */ private boolean range_initRange_0; /* 010 */ private long range_nextIndex_0; /* 011 */ private TaskContext range_taskContext_0; /* 012 */ private InputMetrics range_inputMetrics_0; /* 013 */ private long range_batchEnd_0; /* 014 */ private long range_numElementsTodo_0; /* 015 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[] range_mutableStateArray_0 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[1]; /* 016 */ /* 017 */ public GeneratedIteratorForCodegenStage1(Object[] references) { /* 018 */ this.references = references; /* 019 */ } /* 020 */ /* 021 */ public void init(int index, scala.collection.Iterator[] inputs) { /* 022 */ partitionIndex = index; /* 023 */ this.inputs = inputs; /* 024 */ /* 025 */ range_taskContext_0 = TaskContext.get(); /* 026 */ range_inputMetrics_0 = range_taskContext_0.taskMetrics().inputMetrics(); /* 027 */ range_mutableStateArray_0[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(1, 0); /* 028 */ /* 029 */ } /* 030 */ /* 031 */ private void initRange(int idx) { /* 032 */ java.math.BigInteger index = java.math.BigInteger.valueOf(idx); /* 033 */ java.math.BigInteger numSlice = java.math.BigInteger.valueOf(4L); /* 034 */ java.math.BigInteger numElement = java.math.BigInteger.valueOf(10L); /* 035 */ java.math.BigInteger step = java.math.BigInteger.valueOf(1L); /* 036 */ java.math.BigInteger start = java.math.BigInteger.valueOf(0L); /* 037 */ long partitionEnd; /* 038 */ /* 039 */ java.math.BigInteger st = index.multiply(numElement).divide(numSlice).multiply(step).add(start); /* 040 */ if (st.compareTo(java.math.BigInteger.valueOf(Long.MAX_VALUE)) > 0) { /* 041 */ range_nextIndex_0 = Long.MAX_VALUE; /* 042 */ } else if (st.compareTo(java.math.BigInteger.valueOf(Long.MIN_VALUE)) < 0) { /* 043 */ range_nextIndex_0 = Long.MIN_VALUE; /* 044 */ } else { /* 045 */ range_nextIndex_0 = st.longValue(); /* 046 */ } /* 047 */ range_batchEnd_0 = range_nextIndex_0; /* 048 */ /* 049 */ java.math.BigInteger end = index.add(java.math.BigInteger.ONE).multiply(numElement).divide(numSlice) /* 050 */ .multiply(step).add(start); /* 051 */ if (end.compareTo(java.math.BigInteger.valueOf(Long.MAX_VALUE)) > 0) { /* 052 */ partitionEnd = Long.MAX_VALUE; /* 053 */ } else if (end.compareTo(java.math.BigInteger.valueOf(Long.MIN_VALUE)) < 0) { /* 054 */ partitionEnd = Long.MIN_VALUE; /* 055 */ } else { /* 056 */ partitionEnd = end.longValue(); /* 057 */ } /* 058 */ /* 059 */ java.math.BigInteger startToEnd = java.math.BigInteger.valueOf(partitionEnd).subtract( /* 060 */ java.math.BigInteger.valueOf(range_nextIndex_0)); /* 061 */ range_numElementsTodo_0 = startToEnd.divide(step).longValue(); /* 062 */ if (range_numElementsTodo_0 < 0) { /* 063 */ range_numElementsTodo_0 = 0; /* 064 */ } else if (startToEnd.remainder(step).compareTo(java.math.BigInteger.valueOf(0L)) != 0) { /* 065 */ range_numElementsTodo_0++; /* 066 */ } /* 067 */ } /* 068 */ /* 069 */ protected void processNext() throws java.io.IOException { /* 070 */ // initialize Range /* 071 */ if (!range_initRange_0) { /* 072 */ range_initRange_0 = true; /* 073 */ initRange(partitionIndex); /* 074 */ } /* 075 */ /* 076 */ while (true) { /* 077 */ if (range_nextIndex_0 == range_batchEnd_0) { /* 078 */ long range_nextBatchTodo_0; /* 079 */ if (range_numElementsTodo_0 > 1000L) { /* 080 */ range_nextBatchTodo_0 = 1000L; /* 081 */ range_numElementsTodo_0 -= 1000L; /* 082 */ } else { /* 083 */ range_nextBatchTodo_0 = range_numElementsTodo_0; /* 084 */ range_numElementsTodo_0 = 0; /* 085 */ if (range_nextBatchTodo_0 == 0) break; /* 086 */ } /* 087 */ range_batchEnd_0 += range_nextBatchTodo_0 * 1L; /* 088 */ } /* 089 */ /* 090 */ int range_localEnd_0 = (int)((range_batchEnd_0 - range_nextIndex_0) / 1L); /* 091 */ for (int range_localIdx_0 = 0; range_localIdx_0 < range_localEnd_0; range_localIdx_0++) { /* 092 */ long range_value_0 = ((long)range_localIdx_0 * 1L) + range_nextIndex_0; /* 093 */ /* 094 */ range_mutableStateArray_0[0].reset(); /* 095 */ /* 096 */ range_mutableStateArray_0[0].write(0, range_value_0); /* 097 */ append((range_mutableStateArray_0[0].getRow())); /* 098 */ /* 099 */ if (shouldStop()) { /* 100 */ range_nextIndex_0 = range_value_0 + 1L; /* 101 */ ((org.apache.spark.sql.execution.metric.SQLMetric) references[0] /* numOutputRows */).add(range_localIdx_0 + 1); /* 102 */ range_inputMetrics_0.incRecordsRead(range_localIdx_0 + 1); /* 103 */ return; /* 104 */ } /* 105 */ /* 106 */ } /* 107 */ range_nextIndex_0 = range_batchEnd_0; /* 108 */ ((org.apache.spark.sql.execution.metric.SQLMetric) references[0] /* numOutputRows */).add(range_localEnd_0); /* 109 */ range_inputMetrics_0.incRecordsRead(range_localEnd_0); /* 110 */ range_taskContext_0.killTaskIfInterrupted(); /* 111 */ } /* 112 */ } /* 113 */ /* 114 */ } 20/01/23 18:53:58.544 Thread-4 INFO CodeGenerator: Code generated in 84.286896 ms 20/01/23 18:53:58.574 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$inputRDDs$1$adapted 20/01/23 18:53:58.596 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$inputRDDs$1$adapted) is now cleaned +++ 20/01/23 18:53:58.614 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$doExecute$4$adapted 20/01/23 18:53:58.623 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$doExecute$4$adapted) is now cleaned +++ 20/01/23 18:53:58.696 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$doExecute$1 20/01/23 18:53:58.699 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$doExecute$1) is now cleaned +++ 20/01/23 18:53:58.706 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$doExecute$2 20/01/23 18:53:58.707 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$doExecute$2) is now cleaned +++ 20/01/23 18:53:58.725 Thread-4 INFO CodeGenerator: Code generated in 434.452528 ms 20/01/23 18:53:58.727 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$doExecute$4$adapted 20/01/23 18:53:58.730 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$doExecute$4$adapted) is now cleaned +++ 20/01/23 18:53:58.753 Thread-4 INFO CodeGenerator: Code generated in 388.808114 ms 20/01/23 18:53:58.755 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$doExecute$4$adapted 20/01/23 18:53:58.757 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$doExecute$4$adapted) is now cleaned +++ 20/01/23 18:53:58.774 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$doExecute$4$adapted 20/01/23 18:53:58.776 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$doExecute$4$adapted) is now cleaned +++ 20/01/23 18:53:58.791 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$executeTake$2 20/01/23 18:53:58.791 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$collect$2 20/01/23 18:53:58.793 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$executeTake$2) is now cleaned +++ 20/01/23 18:53:58.795 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$runJob$5 20/01/23 18:53:58.799 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$collect$2) is now cleaned +++ 20/01/23 18:53:58.802 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$runJob$5) is now cleaned +++ 20/01/23 18:53:58.805 Thread-4 INFO SparkContext: Starting job: head at MetadataUtils.scala:51 20/01/23 18:53:58.830 dag-scheduler-event-loop INFO DAGScheduler: Got job 0 (head at MetadataUtils.scala:51) with 1 output partitions 20/01/23 18:53:58.831 dag-scheduler-event-loop INFO DAGScheduler: Final stage: ResultStage 0 (head at MetadataUtils.scala:51) 20/01/23 18:53:58.832 dag-scheduler-event-loop INFO DAGScheduler: Parents of final stage: List() 20/01/23 18:53:58.834 dag-scheduler-event-loop INFO DAGScheduler: Missing parents: List() 20/01/23 18:53:58.839 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$rdd$1 20/01/23 18:53:58.841 dag-scheduler-event-loop DEBUG DAGScheduler: submitStage(ResultStage 0 (name=head at MetadataUtils.scala:51;jobs=0)) 20/01/23 18:53:58.841 Thread-4 DEBUG WholeStageCodegenExec: /* 001 */ public Object generate(Object[] references) { /* 002 */ return new GeneratedIteratorForCodegenStage1(references); /* 003 */ } /* 004 */ /* 005 */ // codegenStageId=1 /* 006 */ final class GeneratedIteratorForCodegenStage1 extends org.apache.spark.sql.execution.BufferedRowIterator { /* 007 */ private Object[] references; /* 008 */ private scala.collection.Iterator[] inputs; /* 009 */ private scala.collection.Iterator rdd_input_0; /* 010 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[] rdd_mutableStateArray_0 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[5]; /* 011 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[] rdd_mutableStateArray_1 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[7]; /* 012 */ /* 013 */ public GeneratedIteratorForCodegenStage1(Object[] references) { /* 014 */ this.references = references; /* 015 */ } /* 016 */ /* 017 */ public void init(int index, scala.collection.Iterator[] inputs) { /* 018 */ partitionIndex = index; /* 019 */ this.inputs = inputs; /* 020 */ wholestagecodegen_init_0_0(); /* 021 */ wholestagecodegen_init_0_1(); /* 022 */ /* 023 */ } /* 024 */ /* 025 */ private void project_writeFields_1_1(InternalRow project_tmpInput_3) { /* 026 */ if ((project_tmpInput_3.isNullAt(3))) { /* 027 */ rdd_mutableStateArray_0[4].setNullAt(3); /* 028 */ } else { /* 029 */ // Remember the current cursor so that we can calculate how many bytes are /* 030 */ // written later. /* 031 */ final int project_previousCursor_5 = rdd_mutableStateArray_0[4].cursor(); /* 032 */ /* 033 */ final ArrayData project_tmpInput_5 = (project_tmpInput_3.getArray(3)); /* 034 */ if (project_tmpInput_5 instanceof UnsafeArrayData) { /* 035 */ rdd_mutableStateArray_0[4].write((UnsafeArrayData) project_tmpInput_5); /* 036 */ } else { /* 037 */ final int project_numElements_3 = project_tmpInput_5.numElements(); /* 038 */ rdd_mutableStateArray_1[6].initialize(project_numElements_3); /* 039 */ /* 040 */ for (int project_index_3 = 0; project_index_3 < project_numElements_3; project_index_3++) { /* 041 */ rdd_mutableStateArray_1[6].write(project_index_3, project_tmpInput_5.getDouble(project_index_3)); /* 042 */ } /* 043 */ } /* 044 */ /* 045 */ rdd_mutableStateArray_0[4].setOffsetAndSizeFromPreviousCursor(3, project_previousCursor_5); /* 046 */ } /* 047 */ /* 048 */ } /* 049 */ /* 050 */ private void project_writeFields_1_0(InternalRow project_tmpInput_3) { /* 051 */ rdd_mutableStateArray_0[4].write(0, (project_tmpInput_3.getByte(0))); /* 052 */ /* 053 */ if ((project_tmpInput_3.isNullAt(1))) { /* 054 */ rdd_mutableStateArray_0[4].setNullAt(1); /* 055 */ } else { /* 056 */ rdd_mutableStateArray_0[4].write(1, (project_tmpInput_3.getInt(1))); /* 057 */ } /* 058 */ /* 059 */ if ((project_tmpInput_3.isNullAt(2))) { /* 060 */ rdd_mutableStateArray_0[4].setNullAt(2); /* 061 */ } else { /* 062 */ // Remember the current cursor so that we can calculate how many bytes are /* 063 */ // written later. /* 064 */ final int project_previousCursor_4 = rdd_mutableStateArray_0[4].cursor(); /* 065 */ /* 066 */ final ArrayData project_tmpInput_4 = (project_tmpInput_3.getArray(2)); /* 067 */ if (project_tmpInput_4 instanceof UnsafeArrayData) { /* 068 */ rdd_mutableStateArray_0[4].write((UnsafeArrayData) project_tmpInput_4); /* 069 */ } else { /* 070 */ final int project_numElements_2 = project_tmpInput_4.numElements(); /* 071 */ rdd_mutableStateArray_1[5].initialize(project_numElements_2); /* 072 */ /* 073 */ for (int project_index_2 = 0; project_index_2 < project_numElements_2; project_index_2++) { /* 074 */ rdd_mutableStateArray_1[5].write(project_index_2, project_tmpInput_4.getInt(project_index_2)); /* 075 */ } /* 076 */ } /* 077 */ /* 078 */ rdd_mutableStateArray_0[4].setOffsetAndSizeFromPreviousCursor(2, project_previousCursor_4); /* 079 */ } /* 080 */ /* 081 */ } /* 082 */ /* 083 */ private void project_writeFields_0_1(InternalRow project_tmpInput_0) { /* 084 */ if ((project_tmpInput_0.isNullAt(3))) { /* 085 */ rdd_mutableStateArray_0[3].setNullAt(3); /* 086 */ } else { /* 087 */ // Remember the current cursor so that we can calculate how many bytes are /* 088 */ // written later. /* 089 */ final int project_previousCursor_2 = rdd_mutableStateArray_0[3].cursor(); /* 090 */ /* 091 */ final ArrayData project_tmpInput_2 = (project_tmpInput_0.getArray(3)); /* 092 */ if (project_tmpInput_2 instanceof UnsafeArrayData) { /* 093 */ rdd_mutableStateArray_0[3].write((UnsafeArrayData) project_tmpInput_2); /* 094 */ } else { /* 095 */ final int project_numElements_1 = project_tmpInput_2.numElements(); /* 096 */ rdd_mutableStateArray_1[4].initialize(project_numElements_1); /* 097 */ /* 098 */ for (int project_index_1 = 0; project_index_1 < project_numElements_1; project_index_1++) { /* 099 */ rdd_mutableStateArray_1[4].write(project_index_1, project_tmpInput_2.getDouble(project_index_1)); /* 100 */ } /* 101 */ } /* 102 */ /* 103 */ rdd_mutableStateArray_0[3].setOffsetAndSizeFromPreviousCursor(3, project_previousCursor_2); /* 104 */ } /* 105 */ /* 106 */ } /* 107 */ /* 108 */ private void wholestagecodegen_init_0_1() { /* 109 */ rdd_mutableStateArray_1[4] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[3], 8); /* 110 */ rdd_mutableStateArray_0[4] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(rdd_mutableStateArray_0[2], 4); /* 111 */ rdd_mutableStateArray_1[5] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[4], 4); /* 112 */ rdd_mutableStateArray_1[6] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[4], 8); /* 113 */ /* 114 */ } /* 115 */ /* 116 */ private void project_writeFields_0_0(InternalRow project_tmpInput_0) { /* 117 */ rdd_mutableStateArray_0[3].write(0, (project_tmpInput_0.getByte(0))); /* 118 */ /* 119 */ if ((project_tmpInput_0.isNullAt(1))) { /* 120 */ rdd_mutableStateArray_0[3].setNullAt(1); /* 121 */ } else { /* 122 */ rdd_mutableStateArray_0[3].write(1, (project_tmpInput_0.getInt(1))); /* 123 */ } /* 124 */ /* 125 */ if ((project_tmpInput_0.isNullAt(2))) { /* 126 */ rdd_mutableStateArray_0[3].setNullAt(2); /* 127 */ } else { /* 128 */ // Remember the current cursor so that we can calculate how many bytes are /* 129 */ // written later. /* 130 */ final int project_previousCursor_1 = rdd_mutableStateArray_0[3].cursor(); /* 131 */ /* 132 */ final ArrayData project_tmpInput_1 = (project_tmpInput_0.getArray(2)); /* 133 */ if (project_tmpInput_1 instanceof UnsafeArrayData) { /* 134 */ rdd_mutableStateArray_0[3].write((UnsafeArrayData) project_tmpInput_1); /* 135 */ } else { /* 136 */ final int project_numElements_0 = project_tmpInput_1.numElements(); /* 137 */ rdd_mutableStateArray_1[3].initialize(project_numElements_0); /* 138 */ /* 139 */ for (int project_index_0 = 0; project_index_0 < project_numElements_0; project_index_0++) { /* 140 */ rdd_mutableStateArray_1[3].write(project_index_0, project_tmpInput_1.getInt(project_index_0)); /* 141 */ } /* 142 */ } /* 143 */ /* 144 */ rdd_mutableStateArray_0[3].setOffsetAndSizeFromPreviousCursor(2, project_previousCursor_1); /* 145 */ } /* 146 */ /* 147 */ } /* 148 */ /* 149 */ protected void processNext() throws java.io.IOException { /* 150 */ while ( rdd_input_0.hasNext()) { /* 151 */ InternalRow rdd_row_0 = (InternalRow) rdd_input_0.next(); /* 152 */ ((org.apache.spark.sql.execution.metric.SQLMetric) references[0] /* numOutputRows */).add(1); /* 153 */ boolean rdd_isNull_1 = rdd_row_0.isNullAt(1); /* 154 */ ArrayData rdd_value_1 = rdd_isNull_1 ? /* 155 */ null : (rdd_row_0.getArray(1)); /* 156 */ Object project_arg_0 = rdd_isNull_1 ? null : ((scala.Function1[]) references[1] /* converters */)[0].apply(rdd_value_1); /* 157 */ /* 158 */ InternalRow project_result_0 = null; /* 159 */ try { /* 160 */ project_result_0 = (InternalRow)((scala.Function1[]) references[1] /* converters */)[1].apply(((scala.Function1) references[3] /* udf */).apply(project_arg_0)); /* 161 */ } catch (Exception e) { /* 162 */ throw new org.apache.spark.SparkException(((java.lang.String) references[2] /* errMsg */), e); /* 163 */ } /* 164 */ /* 165 */ boolean project_isNull_0 = project_result_0 == null; /* 166 */ InternalRow project_value_0 = null; /* 167 */ if (!project_isNull_0) { /* 168 */ project_value_0 = project_result_0; /* 169 */ } /* 170 */ boolean rdd_isNull_2 = rdd_row_0.isNullAt(2); /* 171 */ InternalRow rdd_value_2 = rdd_isNull_2 ? /* 172 */ null : (rdd_row_0.getStruct(2, 4)); /* 173 */ rdd_mutableStateArray_0[2].reset(); /* 174 */ /* 175 */ rdd_mutableStateArray_0[2].zeroOutNullBytes(); /* 176 */ /* 177 */ if (project_isNull_0) { /* 178 */ rdd_mutableStateArray_0[2].setNullAt(0); /* 179 */ } else { /* 180 */ final InternalRow project_tmpInput_0 = project_value_0; /* 181 */ if (project_tmpInput_0 instanceof UnsafeRow) { /* 182 */ rdd_mutableStateArray_0[2].write(0, (UnsafeRow) project_tmpInput_0); /* 183 */ } else { /* 184 */ // Remember the current cursor so that we can calculate how many bytes are /* 185 */ // written later. /* 186 */ final int project_previousCursor_0 = rdd_mutableStateArray_0[2].cursor(); /* 187 */ /* 188 */ rdd_mutableStateArray_0[3].resetRowWriter(); /* 189 */ project_writeFields_0_0(project_tmpInput_0); /* 190 */ project_writeFields_0_1(project_tmpInput_0); /* 191 */ /* 192 */ rdd_mutableStateArray_0[2].setOffsetAndSizeFromPreviousCursor(0, project_previousCursor_0); /* 193 */ } /* 194 */ } /* 195 */ /* 196 */ if (rdd_isNull_2) { /* 197 */ rdd_mutableStateArray_0[2].setNullAt(1); /* 198 */ } else { /* 199 */ final InternalRow project_tmpInput_3 = rdd_value_2; /* 200 */ if (project_tmpInput_3 instanceof UnsafeRow) { /* 201 */ rdd_mutableStateArray_0[2].write(1, (UnsafeRow) project_tmpInput_3); /* 202 */ } else { /* 203 */ // Remember the current cursor so that we can calculate how many bytes are /* 204 */ // written later. /* 205 */ final int project_previousCursor_3 = rdd_mutableStateArray_0[2].cursor(); /* 206 */ /* 207 */ rdd_mutableStateArray_0[4].resetRowWriter(); /* 208 */ project_writeFields_1_0(project_tmpInput_3); /* 209 */ project_writeFields_1_1(project_tmpInput_3); /* 210 */ /* 211 */ rdd_mutableStateArray_0[2].setOffsetAndSizeFromPreviousCursor(1, project_previousCursor_3); /* 212 */ } /* 213 */ } /* 214 */ append((rdd_mutableStateArray_0[2].getRow())); /* 215 */ if (shouldStop()) return; /* 216 */ } /* 217 */ } /* 218 */ /* 219 */ private void wholestagecodegen_init_0_0() { /* 220 */ rdd_input_0 = inputs[0]; /* 221 */ rdd_mutableStateArray_0[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(3, 64); /* 222 */ rdd_mutableStateArray_1[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[0], 8); /* 223 */ rdd_mutableStateArray_0[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(rdd_mutableStateArray_0[0], 4); /* 224 */ rdd_mutableStateArray_1[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[1], 4); /* 225 */ rdd_mutableStateArray_1[2] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[1], 8); /* 226 */ rdd_mutableStateArray_0[2] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(2, 64); /* 227 */ rdd_mutableStateArray_0[3] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(rdd_mutableStateArray_0[2], 4); /* 228 */ rdd_mutableStateArray_1[3] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[3], 4); /* 229 */ /* 230 */ } /* 231 */ /* 232 */ } 20/01/23 18:53:58.843 dag-scheduler-event-loop DEBUG DAGScheduler: missing: List() 20/01/23 18:53:58.844 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$rdd$1) is now cleaned +++ 20/01/23 18:53:58.855 dag-scheduler-event-loop INFO DAGScheduler: Submitting ResultStage 0 (MapPartitionsRDD[6] at head at MetadataUtils.scala:51), which has no missing parents 20/01/23 18:53:58.856 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$runJob$5 20/01/23 18:53:58.861 dag-scheduler-event-loop DEBUG DAGScheduler: submitMissingTasks(ResultStage 0) 20/01/23 18:53:58.863 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$runJob$5) is now cleaned +++ 20/01/23 18:53:58.866 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$fit$2 20/01/23 18:53:58.867 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$fit$2) is now cleaned +++ 20/01/23 18:53:58.868 Thread-4 INFO SparkContext: Starting job: at /usr/lib/python3.6/runpy.py:85 20/01/23 18:53:58.871 Thread-4 INFO Instrumentation: [3e99e817] Stage class: ALS 20/01/23 18:53:58.871 Thread-4 INFO Instrumentation: [3e99e817] Stage uid: ALS_56177bd83433 20/01/23 18:53:58.909 dag-scheduler-event-loop INFO DAGScheduler: Got job 0 ( at /usr/lib/python3.6/runpy.py:85) with 4 output partitions 20/01/23 18:53:58.910 dag-scheduler-event-loop INFO DAGScheduler: Final stage: ResultStage 0 ( at /usr/lib/python3.6/runpy.py:85) 20/01/23 18:53:58.910 dag-scheduler-event-loop INFO DAGScheduler: Parents of final stage: List() 20/01/23 18:53:58.912 dag-scheduler-event-loop INFO DAGScheduler: Missing parents: List() 20/01/23 18:53:58.916 dag-scheduler-event-loop DEBUG DAGScheduler: submitStage(ResultStage 0 (name= at /usr/lib/python3.6/runpy.py:85;jobs=0)) 20/01/23 18:53:58.921 Thread-4 DEBUG WholeStageCodegenExec: /* 001 */ public Object generate(Object[] references) { /* 002 */ return new GeneratedIteratorForCodegenStage1(references); /* 003 */ } /* 004 */ /* 005 */ // codegenStageId=1 /* 006 */ final class GeneratedIteratorForCodegenStage1 extends org.apache.spark.sql.execution.BufferedRowIterator { /* 007 */ private Object[] references; /* 008 */ private scala.collection.Iterator[] inputs; /* 009 */ private scala.collection.Iterator rdd_input_0; /* 010 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[] rdd_mutableStateArray_0 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[1]; /* 011 */ /* 012 */ public GeneratedIteratorForCodegenStage1(Object[] references) { /* 013 */ this.references = references; /* 014 */ } /* 015 */ /* 016 */ public void init(int index, scala.collection.Iterator[] inputs) { /* 017 */ partitionIndex = index; /* 018 */ this.inputs = inputs; /* 019 */ rdd_input_0 = inputs[0]; /* 020 */ rdd_mutableStateArray_0[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(3, 0); /* 021 */ /* 022 */ } /* 023 */ /* 024 */ protected void processNext() throws java.io.IOException { /* 025 */ while ( rdd_input_0.hasNext()) { /* 026 */ InternalRow rdd_row_0 = (InternalRow) rdd_input_0.next(); /* 027 */ ((org.apache.spark.sql.execution.metric.SQLMetric) references[0] /* numOutputRows */).add(1); /* 028 */ boolean rdd_isNull_0 = rdd_row_0.isNullAt(0); /* 029 */ long rdd_value_0 = rdd_isNull_0 ? /* 030 */ -1L : (rdd_row_0.getLong(0)); /* 031 */ boolean rdd_isNull_1 = rdd_row_0.isNullAt(1); /* 032 */ long rdd_value_1 = rdd_isNull_1 ? /* 033 */ -1L : (rdd_row_0.getLong(1)); /* 034 */ boolean rdd_isNull_2 = rdd_row_0.isNullAt(2); /* 035 */ double rdd_value_2 = rdd_isNull_2 ? /* 036 */ -1.0 : (rdd_row_0.getDouble(2)); /* 037 */ rdd_mutableStateArray_0[0].reset(); /* 038 */ /* 039 */ rdd_mutableStateArray_0[0].zeroOutNullBytes(); /* 040 */ /* 041 */ if (rdd_isNull_0) { /* 042 */ rdd_mutableStateArray_0[0].setNullAt(0); /* 043 */ } else { /* 044 */ rdd_mutableStateArray_0[0].write(0, rdd_value_0); /* 045 */ } /* 046 */ /* 047 */ if (rdd_isNull_1) { /* 048 */ rdd_mutableStateArray_0[0].setNullAt(1); /* 049 */ } else { /* 050 */ rdd_mutableStateArray_0[0].write(1, rdd_value_1); /* 051 */ } /* 052 */ /* 053 */ if (rdd_isNull_2) { /* 054 */ rdd_mutableStateArray_0[0].setNullAt(2); /* 055 */ } else { /* 056 */ rdd_mutableStateArray_0[0].write(2, rdd_value_2); /* 057 */ } /* 058 */ append((rdd_mutableStateArray_0[0].getRow())); /* 059 */ if (shouldStop()) return; /* 060 */ } /* 061 */ } /* 062 */ /* 063 */ } 20/01/23 18:53:58.922 dag-scheduler-event-loop DEBUG DAGScheduler: missing: List() 20/01/23 18:53:58.923 Thread-4 DEBUG CodeGenerator: /* 001 */ public Object generate(Object[] references) { /* 002 */ return new GeneratedIteratorForCodegenStage1(references); /* 003 */ } /* 004 */ /* 005 */ // codegenStageId=1 /* 006 */ final class GeneratedIteratorForCodegenStage1 extends org.apache.spark.sql.execution.BufferedRowIterator { /* 007 */ private Object[] references; /* 008 */ private scala.collection.Iterator[] inputs; /* 009 */ private scala.collection.Iterator rdd_input_0; /* 010 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[] rdd_mutableStateArray_0 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[5]; /* 011 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[] rdd_mutableStateArray_1 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[7]; /* 012 */ /* 013 */ public GeneratedIteratorForCodegenStage1(Object[] references) { /* 014 */ this.references = references; /* 015 */ } /* 016 */ /* 017 */ public void init(int index, scala.collection.Iterator[] inputs) { /* 018 */ partitionIndex = index; /* 019 */ this.inputs = inputs; /* 020 */ wholestagecodegen_init_0_0(); /* 021 */ wholestagecodegen_init_0_1(); /* 022 */ /* 023 */ } /* 024 */ /* 025 */ private void project_writeFields_1_1(InternalRow project_tmpInput_3) { /* 026 */ if ((project_tmpInput_3.isNullAt(3))) { /* 027 */ rdd_mutableStateArray_0[4].setNullAt(3); /* 028 */ } else { /* 029 */ // Remember the current cursor so that we can calculate how many bytes are /* 030 */ // written later. /* 031 */ final int project_previousCursor_5 = rdd_mutableStateArray_0[4].cursor(); /* 032 */ /* 033 */ final ArrayData project_tmpInput_5 = (project_tmpInput_3.getArray(3)); /* 034 */ if (project_tmpInput_5 instanceof UnsafeArrayData) { /* 035 */ rdd_mutableStateArray_0[4].write((UnsafeArrayData) project_tmpInput_5); /* 036 */ } else { /* 037 */ final int project_numElements_3 = project_tmpInput_5.numElements(); /* 038 */ rdd_mutableStateArray_1[6].initialize(project_numElements_3); /* 039 */ /* 040 */ for (int project_index_3 = 0; project_index_3 < project_numElements_3; project_index_3++) { /* 041 */ rdd_mutableStateArray_1[6].write(project_index_3, project_tmpInput_5.getDouble(project_index_3)); /* 042 */ } /* 043 */ } /* 044 */ /* 045 */ rdd_mutableStateArray_0[4].setOffsetAndSizeFromPreviousCursor(3, project_previousCursor_5); /* 046 */ } /* 047 */ /* 048 */ } /* 049 */ /* 050 */ private void project_writeFields_1_0(InternalRow project_tmpInput_3) { /* 051 */ rdd_mutableStateArray_0[4].write(0, (project_tmpInput_3.getByte(0))); /* 052 */ /* 053 */ if ((project_tmpInput_3.isNullAt(1))) { /* 054 */ rdd_mutableStateArray_0[4].setNullAt(1); /* 055 */ } else { /* 056 */ rdd_mutableStateArray_0[4].write(1, (project_tmpInput_3.getInt(1))); /* 057 */ } /* 058 */ /* 059 */ if ((project_tmpInput_3.isNullAt(2))) { /* 060 */ rdd_mutableStateArray_0[4].setNullAt(2); /* 061 */ } else { /* 062 */ // Remember the current cursor so that we can calculate how many bytes are /* 063 */ // written later. /* 064 */ final int project_previousCursor_4 = rdd_mutableStateArray_0[4].cursor(); /* 065 */ /* 066 */ final ArrayData project_tmpInput_4 = (project_tmpInput_3.getArray(2)); /* 067 */ if (project_tmpInput_4 instanceof UnsafeArrayData) { /* 068 */ rdd_mutableStateArray_0[4].write((UnsafeArrayData) project_tmpInput_4); /* 069 */ } else { /* 070 */ final int project_numElements_2 = project_tmpInput_4.numElements(); /* 071 */ rdd_mutableStateArray_1[5].initialize(project_numElements_2); /* 072 */ /* 073 */ for (int project_index_2 = 0; project_index_2 < project_numElements_2; project_index_2++) { /* 074 */ rdd_mutableStateArray_1[5].write(project_index_2, project_tmpInput_4.getInt(project_index_2)); /* 075 */ } /* 076 */ } /* 077 */ /* 078 */ rdd_mutableStateArray_0[4].setOffsetAndSizeFromPreviousCursor(2, project_previousCursor_4); /* 079 */ } /* 080 */ /* 081 */ } /* 082 */ /* 083 */ private void project_writeFields_0_1(InternalRow project_tmpInput_0) { /* 084 */ if ((project_tmpInput_0.isNullAt(3))) { /* 085 */ rdd_mutableStateArray_0[3].setNullAt(3); /* 086 */ } else { /* 087 */ // Remember the current cursor so that we can calculate how many bytes are /* 088 */ // written later. /* 089 */ final int project_previousCursor_2 = rdd_mutableStateArray_0[3].cursor(); /* 090 */ /* 091 */ final ArrayData project_tmpInput_2 = (project_tmpInput_0.getArray(3)); /* 092 */ if (project_tmpInput_2 instanceof UnsafeArrayData) { /* 093 */ rdd_mutableStateArray_0[3].write((UnsafeArrayData) project_tmpInput_2); /* 094 */ } else { /* 095 */ final int project_numElements_1 = project_tmpInput_2.numElements(); /* 096 */ rdd_mutableStateArray_1[4].initialize(project_numElements_1); /* 097 */ /* 098 */ for (int project_index_1 = 0; project_index_1 < project_numElements_1; project_index_1++) { /* 099 */ rdd_mutableStateArray_1[4].write(project_index_1, project_tmpInput_2.getDouble(project_index_1)); /* 100 */ } /* 101 */ } /* 102 */ /* 103 */ rdd_mutableStateArray_0[3].setOffsetAndSizeFromPreviousCursor(3, project_previousCursor_2); /* 104 */ } /* 105 */ /* 106 */ } /* 107 */ /* 108 */ private void wholestagecodegen_init_0_1() { /* 109 */ rdd_mutableStateArray_1[4] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[3], 8); /* 110 */ rdd_mutableStateArray_0[4] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(rdd_mutableStateArray_0[2], 4); /* 111 */ rdd_mutableStateArray_1[5] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[4], 4); /* 112 */ rdd_mutableStateArray_1[6] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[4], 8); /* 113 */ /* 114 */ } /* 115 */ /* 116 */ private void project_writeFields_0_0(InternalRow project_tmpInput_0) { /* 117 */ rdd_mutableStateArray_0[3].write(0, (project_tmpInput_0.getByte(0))); /* 118 */ /* 119 */ if ((project_tmpInput_0.isNullAt(1))) { /* 120 */ rdd_mutableStateArray_0[3].setNullAt(1); /* 121 */ } else { /* 122 */ rdd_mutableStateArray_0[3].write(1, (project_tmpInput_0.getInt(1))); /* 123 */ } /* 124 */ /* 125 */ if ((project_tmpInput_0.isNullAt(2))) { /* 126 */ rdd_mutableStateArray_0[3].setNullAt(2); /* 127 */ } else { /* 128 */ // Remember the current cursor so that we can calculate how many bytes are /* 129 */ // written later. /* 130 */ final int project_previousCursor_1 = rdd_mutableStateArray_0[3].cursor(); /* 131 */ /* 132 */ final ArrayData project_tmpInput_1 = (project_tmpInput_0.getArray(2)); /* 133 */ if (project_tmpInput_1 instanceof UnsafeArrayData) { /* 134 */ rdd_mutableStateArray_0[3].write((UnsafeArrayData) project_tmpInput_1); /* 135 */ } else { /* 136 */ final int project_numElements_0 = project_tmpInput_1.numElements(); /* 137 */ rdd_mutableStateArray_1[3].initialize(project_numElements_0); /* 138 */ /* 139 */ for (int project_index_0 = 0; project_index_0 < project_numElements_0; project_index_0++) { /* 140 */ rdd_mutableStateArray_1[3].write(project_index_0, project_tmpInput_1.getInt(project_index_0)); /* 141 */ } /* 142 */ } /* 143 */ /* 144 */ rdd_mutableStateArray_0[3].setOffsetAndSizeFromPreviousCursor(2, project_previousCursor_1); /* 145 */ } /* 146 */ /* 147 */ } /* 148 */ /* 149 */ protected void processNext() throws java.io.IOException { /* 150 */ while ( rdd_input_0.hasNext()) { /* 151 */ InternalRow rdd_row_0 = (InternalRow) rdd_input_0.next(); /* 152 */ ((org.apache.spark.sql.execution.metric.SQLMetric) references[0] /* numOutputRows */).add(1); /* 153 */ boolean rdd_isNull_1 = rdd_row_0.isNullAt(1); /* 154 */ ArrayData rdd_value_1 = rdd_isNull_1 ? /* 155 */ null : (rdd_row_0.getArray(1)); /* 156 */ Object project_arg_0 = rdd_isNull_1 ? null : ((scala.Function1[]) references[1] /* converters */)[0].apply(rdd_value_1); /* 157 */ /* 158 */ InternalRow project_result_0 = null; /* 159 */ try { /* 160 */ project_result_0 = (InternalRow)((scala.Function1[]) references[1] /* converters */)[1].apply(((scala.Function1) references[3] /* udf */).apply(project_arg_0)); /* 161 */ } catch (Exception e) { /* 162 */ throw new org.apache.spark.SparkException(((java.lang.String) references[2] /* errMsg */), e); /* 163 */ } /* 164 */ /* 165 */ boolean project_isNull_0 = project_result_0 == null; /* 166 */ InternalRow project_value_0 = null; /* 167 */ if (!project_isNull_0) { /* 168 */ project_value_0 = project_result_0; /* 169 */ } /* 170 */ boolean rdd_isNull_2 = rdd_row_0.isNullAt(2); /* 171 */ InternalRow rdd_value_2 = rdd_isNull_2 ? /* 172 */ null : (rdd_row_0.getStruct(2, 4)); /* 173 */ rdd_mutableStateArray_0[2].reset(); /* 174 */ /* 175 */ rdd_mutableStateArray_0[2].zeroOutNullBytes(); /* 176 */ /* 177 */ if (project_isNull_0) { /* 178 */ rdd_mutableStateArray_0[2].setNullAt(0); /* 179 */ } else { /* 180 */ final InternalRow project_tmpInput_0 = project_value_0; /* 181 */ if (project_tmpInput_0 instanceof UnsafeRow) { /* 182 */ rdd_mutableStateArray_0[2].write(0, (UnsafeRow) project_tmpInput_0); /* 183 */ } else { /* 184 */ // Remember the current cursor so that we can calculate how many bytes are /* 185 */ // written later. /* 186 */ final int project_previousCursor_0 = rdd_mutableStateArray_0[2].cursor(); /* 187 */ /* 188 */ rdd_mutableStateArray_0[3].resetRowWriter(); /* 189 */ project_writeFields_0_0(project_tmpInput_0); /* 190 */ project_writeFields_0_1(project_tmpInput_0); /* 191 */ /* 192 */ rdd_mutableStateArray_0[2].setOffsetAndSizeFromPreviousCursor(0, project_previousCursor_0); /* 193 */ } /* 194 */ } /* 195 */ /* 196 */ if (rdd_isNull_2) { /* 197 */ rdd_mutableStateArray_0[2].setNullAt(1); /* 198 */ } else { /* 199 */ final InternalRow project_tmpInput_3 = rdd_value_2; /* 200 */ if (project_tmpInput_3 instanceof UnsafeRow) { /* 201 */ rdd_mutableStateArray_0[2].write(1, (UnsafeRow) project_tmpInput_3); /* 202 */ } else { /* 203 */ // Remember the current cursor so that we can calculate how many bytes are /* 204 */ // written later. /* 205 */ final int project_previousCursor_3 = rdd_mutableStateArray_0[2].cursor(); /* 206 */ /* 207 */ rdd_mutableStateArray_0[4].resetRowWriter(); /* 208 */ project_writeFields_1_0(project_tmpInput_3); /* 209 */ project_writeFields_1_1(project_tmpInput_3); /* 210 */ /* 211 */ rdd_mutableStateArray_0[2].setOffsetAndSizeFromPreviousCursor(1, project_previousCursor_3); /* 212 */ } /* 213 */ } /* 214 */ append((rdd_mutableStateArray_0[2].getRow())); /* 215 */ if (shouldStop()) return; /* 216 */ } /* 217 */ } /* 218 */ /* 219 */ private void wholestagecodegen_init_0_0() { /* 220 */ rdd_input_0 = inputs[0]; /* 221 */ rdd_mutableStateArray_0[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(3, 64); /* 222 */ rdd_mutableStateArray_1[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[0], 8); /* 223 */ rdd_mutableStateArray_0[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(rdd_mutableStateArray_0[0], 4); /* 224 */ rdd_mutableStateArray_1[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[1], 4); /* 225 */ rdd_mutableStateArray_1[2] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[1], 8); /* 226 */ rdd_mutableStateArray_0[2] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(2, 64); /* 227 */ rdd_mutableStateArray_0[3] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(rdd_mutableStateArray_0[2], 4); /* 228 */ rdd_mutableStateArray_1[3] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[3], 4); /* 229 */ /* 230 */ } /* 231 */ /* 232 */ } 20/01/23 18:53:58.924 dag-scheduler-event-loop INFO DAGScheduler: Submitting ResultStage 0 (MapPartitionsRDD[6] at at /usr/lib/python3.6/runpy.py:85), which has no missing parents 20/01/23 18:53:58.925 dag-scheduler-event-loop DEBUG DAGScheduler: submitMissingTasks(ResultStage 0) 20/01/23 18:53:58.925 Thread-4 DEBUG CodeGenerator: /* 001 */ public Object generate(Object[] references) { /* 002 */ return new GeneratedIteratorForCodegenStage1(references); /* 003 */ } /* 004 */ /* 005 */ // codegenStageId=1 /* 006 */ final class GeneratedIteratorForCodegenStage1 extends org.apache.spark.sql.execution.BufferedRowIterator { /* 007 */ private Object[] references; /* 008 */ private scala.collection.Iterator[] inputs; /* 009 */ private scala.collection.Iterator rdd_input_0; /* 010 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[] rdd_mutableStateArray_0 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[1]; /* 011 */ /* 012 */ public GeneratedIteratorForCodegenStage1(Object[] references) { /* 013 */ this.references = references; /* 014 */ } /* 015 */ /* 016 */ public void init(int index, scala.collection.Iterator[] inputs) { /* 017 */ partitionIndex = index; /* 018 */ this.inputs = inputs; /* 019 */ rdd_input_0 = inputs[0]; /* 020 */ rdd_mutableStateArray_0[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(3, 0); /* 021 */ /* 022 */ } /* 023 */ /* 024 */ protected void processNext() throws java.io.IOException { /* 025 */ while ( rdd_input_0.hasNext()) { /* 026 */ InternalRow rdd_row_0 = (InternalRow) rdd_input_0.next(); /* 027 */ ((org.apache.spark.sql.execution.metric.SQLMetric) references[0] /* numOutputRows */).add(1); /* 028 */ boolean rdd_isNull_0 = rdd_row_0.isNullAt(0); /* 029 */ long rdd_value_0 = rdd_isNull_0 ? /* 030 */ -1L : (rdd_row_0.getLong(0)); /* 031 */ boolean rdd_isNull_1 = rdd_row_0.isNullAt(1); /* 032 */ long rdd_value_1 = rdd_isNull_1 ? /* 033 */ -1L : (rdd_row_0.getLong(1)); /* 034 */ boolean rdd_isNull_2 = rdd_row_0.isNullAt(2); /* 035 */ double rdd_value_2 = rdd_isNull_2 ? /* 036 */ -1.0 : (rdd_row_0.getDouble(2)); /* 037 */ rdd_mutableStateArray_0[0].reset(); /* 038 */ /* 039 */ rdd_mutableStateArray_0[0].zeroOutNullBytes(); /* 040 */ /* 041 */ if (rdd_isNull_0) { /* 042 */ rdd_mutableStateArray_0[0].setNullAt(0); /* 043 */ } else { /* 044 */ rdd_mutableStateArray_0[0].write(0, rdd_value_0); /* 045 */ } /* 046 */ /* 047 */ if (rdd_isNull_1) { /* 048 */ rdd_mutableStateArray_0[0].setNullAt(1); /* 049 */ } else { /* 050 */ rdd_mutableStateArray_0[0].write(1, rdd_value_1); /* 051 */ } /* 052 */ /* 053 */ if (rdd_isNull_2) { /* 054 */ rdd_mutableStateArray_0[0].setNullAt(2); /* 055 */ } else { /* 056 */ rdd_mutableStateArray_0[0].write(2, rdd_value_2); /* 057 */ } /* 058 */ append((rdd_mutableStateArray_0[0].getRow())); /* 059 */ if (shouldStop()) return; /* 060 */ } /* 061 */ } /* 062 */ /* 063 */ } 20/01/23 18:53:58.945 Thread-4 INFO CodeGenerator: Code generated in 24.04469 ms 20/01/23 18:53:58.946 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$doExecute$4$adapted 20/01/23 18:53:58.947 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$doExecute$4$adapted) is now cleaned +++ 20/01/23 18:53:58.950 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$rdd$1 20/01/23 18:53:58.954 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$rdd$1) is now cleaned +++ 20/01/23 18:53:58.973 Thread-4 INFO Instrumentation: [3e99e817] training: numPartitions=4 storageLevel=StorageLevel(1 replicas) 20/01/23 18:53:59.018 Thread-4 INFO Instrumentation: [3e99e817] {"rank":1,"maxIter":1} 20/01/23 18:53:59.037 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_0 stored as values in memory (estimated size 16.5 KiB, free 366.3 MiB) 20/01/23 18:53:59.039 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_0 locally took 63 ms 20/01/23 18:53:59.042 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_0 without replication took 65 ms 20/01/23 18:53:59.043 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$take$2 20/01/23 18:53:59.047 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$take$2) is now cleaned +++ 20/01/23 18:53:59.051 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_0 stored as values in memory (estimated size 14.9 KiB, free 366.3 MiB) 20/01/23 18:53:59.053 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_0 locally took 69 ms 20/01/23 18:53:59.056 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_0 without replication took 72 ms 20/01/23 18:53:59.098 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_0_piece0 stored as bytes in memory (estimated size 7.7 KiB, free 366.3 MiB) 20/01/23 18:53:59.106 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_0_piece0 stored as bytes in memory (estimated size 7.0 KiB, free 366.3 MiB) 20/01/23 18:53:59.106 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added broadcast_0_piece0 in memory on 192.168.1.225:42431 (size: 7.7 KiB, free: 366.3 MiB) 20/01/23 18:53:59.108 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added broadcast_0_piece0 in memory on 192.168.1.225:45461 (size: 7.0 KiB, free: 366.3 MiB) 20/01/23 18:53:59.110 dag-scheduler-event-loop DEBUG BlockManagerMaster: Updated info of block broadcast_0_piece0 20/01/23 18:53:59.111 dag-scheduler-event-loop DEBUG BlockManager: Told master about block broadcast_0_piece0 20/01/23 18:53:59.112 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_0_piece0 locally took 8 ms 20/01/23 18:53:59.112 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_0_piece0 without replication took 9 ms 20/01/23 18:53:59.113 dag-scheduler-event-loop INFO SparkContext: Created broadcast 0 from broadcast at DAGScheduler.scala:1200 20/01/23 18:53:59.114 dag-scheduler-event-loop DEBUG BlockManagerMaster: Updated info of block broadcast_0_piece0 20/01/23 18:53:59.115 dag-scheduler-event-loop DEBUG BlockManager: Told master about block broadcast_0_piece0 20/01/23 18:53:59.116 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_0_piece0 locally took 20 ms 20/01/23 18:53:59.116 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_0_piece0 without replication took 20 ms 20/01/23 18:53:59.117 dag-scheduler-event-loop INFO SparkContext: Created broadcast 0 from broadcast at DAGScheduler.scala:1200 20/01/23 18:53:59.142 dag-scheduler-event-loop INFO DAGScheduler: Submitting 1 missing tasks from ResultStage 0 (MapPartitionsRDD[6] at head at MetadataUtils.scala:51) (first 15 tasks are for partitions Vector(0)) 20/01/23 18:53:59.143 dag-scheduler-event-loop INFO DAGScheduler: Submitting 4 missing tasks from ResultStage 0 (MapPartitionsRDD[6] at at /usr/lib/python3.6/runpy.py:85) (first 15 tasks are for partitions Vector(0, 1, 2, 3)) 20/01/23 18:53:59.143 dag-scheduler-event-loop INFO TaskSchedulerImpl: Adding task set 0.0 with 1 tasks 20/01/23 18:53:59.144 dag-scheduler-event-loop INFO TaskSchedulerImpl: Adding task set 0.0 with 4 tasks 20/01/23 18:53:59.177 dag-scheduler-event-loop DEBUG TaskSetManager: Epoch for TaskSet 0.0: 0 20/01/23 18:53:59.187 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$runJob$5 20/01/23 18:53:59.189 dag-scheduler-event-loop DEBUG TaskSetManager: Adding pending tasks took 7 ms 20/01/23 18:53:59.192 dag-scheduler-event-loop DEBUG TaskSetManager: Valid locality levels for TaskSet 0.0: NO_PREF, ANY 20/01/23 18:53:59.195 dag-scheduler-event-loop DEBUG TaskSetManager: Epoch for TaskSet 0.0: 0 20/01/23 18:53:59.198 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$runJob$5) is now cleaned +++ 20/01/23 18:53:59.208 dag-scheduler-event-loop DEBUG TaskSetManager: Adding pending tasks took 10 ms 20/01/23 18:53:59.210 dag-scheduler-event-loop DEBUG TaskSetManager: Valid locality levels for TaskSet 0.0: NO_PREF, ANY 20/01/23 18:53:59.211 dispatcher-event-loop-2 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_0.0, runningTasks: 0 20/01/23 18:53:59.212 dispatcher-event-loop-2 DEBUG TaskSetManager: Valid locality levels for TaskSet 0.0: NO_PREF, ANY 20/01/23 18:53:59.216 Thread-4 INFO SparkContext: Starting job: isEmpty at ALS.scala:931 20/01/23 18:53:59.228 dispatcher-event-loop-2 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_0.0, runningTasks: 0 20/01/23 18:53:59.229 dispatcher-event-loop-2 DEBUG TaskSetManager: Valid locality levels for TaskSet 0.0: NO_PREF, ANY 20/01/23 18:53:59.246 dag-scheduler-event-loop INFO DAGScheduler: Got job 0 (isEmpty at ALS.scala:931) with 1 output partitions 20/01/23 18:53:59.247 dag-scheduler-event-loop INFO DAGScheduler: Final stage: ResultStage 0 (isEmpty at ALS.scala:931) 20/01/23 18:53:59.248 dag-scheduler-event-loop INFO DAGScheduler: Parents of final stage: List() 20/01/23 18:53:59.251 dag-scheduler-event-loop INFO DAGScheduler: Missing parents: List() 20/01/23 18:53:59.259 dag-scheduler-event-loop DEBUG DAGScheduler: submitStage(ResultStage 0 (name=isEmpty at ALS.scala:931;jobs=0)) 20/01/23 18:53:59.261 dag-scheduler-event-loop DEBUG DAGScheduler: missing: List() 20/01/23 18:53:59.263 dag-scheduler-event-loop INFO DAGScheduler: Submitting ResultStage 0 (MapPartitionsRDD[9] at map at ALS.scala:678), which has no missing parents 20/01/23 18:53:59.265 dag-scheduler-event-loop DEBUG DAGScheduler: submitMissingTasks(ResultStage 0) 20/01/23 18:53:59.279 dispatcher-event-loop-2 INFO TaskSetManager: Starting task 0.0 in stage 0.0 (TID 0, 192.168.1.225, executor driver, partition 0, PROCESS_LOCAL, 7393 bytes) 20/01/23 18:53:59.294 dispatcher-event-loop-2 INFO TaskSetManager: Starting task 1.0 in stage 0.0 (TID 1, 192.168.1.225, executor driver, partition 1, PROCESS_LOCAL, 7393 bytes) 20/01/23 18:53:59.295 dispatcher-event-loop-2 INFO TaskSetManager: Starting task 2.0 in stage 0.0 (TID 2, 192.168.1.225, executor driver, partition 2, PROCESS_LOCAL, 7393 bytes) 20/01/23 18:53:59.295 dispatcher-event-loop-2 INFO TaskSetManager: Starting task 3.0 in stage 0.0 (TID 3, 192.168.1.225, executor driver, partition 3, PROCESS_LOCAL, 7393 bytes) 20/01/23 18:53:59.300 dispatcher-event-loop-2 INFO TaskSetManager: Starting task 0.0 in stage 0.0 (TID 0, 192.168.1.225, executor driver, partition 0, PROCESS_LOCAL, 7398 bytes) 20/01/23 18:53:59.323 dispatcher-event-loop-2 DEBUG TaskSetManager: No tasks for locality level NO_PREF, so moving to locality level ANY 20/01/23 18:53:59.333 Executor task launch worker for task 1 INFO Executor: Running task 1.0 in stage 0.0 (TID 1) 20/01/23 18:53:59.341 Executor task launch worker for task 3 INFO Executor: Running task 3.0 in stage 0.0 (TID 3) 20/01/23 18:53:59.341 Executor task launch worker for task 2 INFO Executor: Running task 2.0 in stage 0.0 (TID 2) 20/01/23 18:53:59.333 Executor task launch worker for task 0 INFO Executor: Running task 0.0 in stage 0.0 (TID 0) 20/01/23 18:53:59.347 Executor task launch worker for task 0 INFO Executor: Running task 0.0 in stage 0.0 (TID 0) 20/01/23 18:53:59.362 Executor task launch worker for task 1 DEBUG ExecutorMetricsPoller: stageTCMP: (0, 0) -> 1 20/01/23 18:53:59.365 Executor task launch worker for task 2 DEBUG ExecutorMetricsPoller: stageTCMP: (0, 0) -> 3 20/01/23 18:53:59.363 Executor task launch worker for task 0 DEBUG ExecutorMetricsPoller: stageTCMP: (0, 0) -> 2 20/01/23 18:53:59.368 Executor task launch worker for task 3 DEBUG ExecutorMetricsPoller: stageTCMP: (0, 0) -> 4 20/01/23 18:53:59.370 Executor task launch worker for task 0 DEBUG ExecutorMetricsPoller: stageTCMP: (0, 0) -> 1 20/01/23 18:53:59.390 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_0 stored as values in memory (estimated size 24.3 KiB, free 366.3 MiB) 20/01/23 18:53:59.398 Thread-4 INFO CodeGenerator: Code generated in 548.164487 ms 20/01/23 18:53:59.399 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_0 locally took 67 ms 20/01/23 18:53:59.400 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$doExecute$4$adapted 20/01/23 18:53:59.402 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_0 without replication took 70 ms 20/01/23 18:53:59.403 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$doExecute$4$adapted) is now cleaned +++ 20/01/23 18:53:59.433 Executor task launch worker for task 3 DEBUG BlockManager: Getting local block broadcast_0 20/01/23 18:53:59.435 Executor task launch worker for task 3 DEBUG BlockManager: Level for block broadcast_0 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:53:59.444 Executor task launch worker for task 0 DEBUG BlockManager: Getting local block broadcast_0 20/01/23 18:53:59.446 Executor task launch worker for task 0 DEBUG BlockManager: Level for block broadcast_0 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:53:59.450 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_0_piece0 stored as bytes in memory (estimated size 11.9 KiB, free 366.3 MiB) 20/01/23 18:53:59.453 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added broadcast_0_piece0 in memory on 192.168.1.225:35863 (size: 11.9 KiB, free: 366.3 MiB) 20/01/23 18:53:59.461 dag-scheduler-event-loop DEBUG BlockManagerMaster: Updated info of block broadcast_0_piece0 20/01/23 18:53:59.474 dag-scheduler-event-loop DEBUG BlockManager: Told master about block broadcast_0_piece0 20/01/23 18:53:59.474 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_0_piece0 locally took 28 ms 20/01/23 18:53:59.475 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_0_piece0 without replication took 28 ms 20/01/23 18:53:59.475 dag-scheduler-event-loop INFO SparkContext: Created broadcast 0 from broadcast at DAGScheduler.scala:1200 20/01/23 18:53:59.498 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$collect$2 20/01/23 18:53:59.498 dag-scheduler-event-loop INFO DAGScheduler: Submitting 1 missing tasks from ResultStage 0 (MapPartitionsRDD[9] at map at ALS.scala:678) (first 15 tasks are for partitions Vector(0)) 20/01/23 18:53:59.500 dag-scheduler-event-loop INFO TaskSchedulerImpl: Adding task set 0.0 with 1 tasks 20/01/23 18:53:59.505 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$collect$2) is now cleaned +++ 20/01/23 18:53:59.526 dag-scheduler-event-loop DEBUG TaskSetManager: Epoch for TaskSet 0.0: 0 20/01/23 18:53:59.530 dag-scheduler-event-loop DEBUG TaskSetManager: Adding pending tasks took 3 ms 20/01/23 18:53:59.532 dag-scheduler-event-loop DEBUG TaskSetManager: Valid locality levels for TaskSet 0.0: NO_PREF, ANY 20/01/23 18:53:59.551 dispatcher-event-loop-2 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_0.0, runningTasks: 0 20/01/23 18:53:59.552 dispatcher-event-loop-2 DEBUG TaskSetManager: Valid locality levels for TaskSet 0.0: NO_PREF, ANY 20/01/23 18:53:59.557 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$runJob$5 20/01/23 18:53:59.583 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$runJob$5) is now cleaned +++ 20/01/23 18:53:59.598 Thread-4 INFO SparkContext: Starting job: at /usr/lib/python3.6/runpy.py:85 20/01/23 18:53:59.631 dag-scheduler-event-loop INFO DAGScheduler: Got job 0 ( at /usr/lib/python3.6/runpy.py:85) with 4 output partitions 20/01/23 18:53:59.633 dag-scheduler-event-loop INFO DAGScheduler: Final stage: ResultStage 0 ( at /usr/lib/python3.6/runpy.py:85) 20/01/23 18:53:59.633 dag-scheduler-event-loop INFO DAGScheduler: Parents of final stage: List() 20/01/23 18:53:59.635 dag-scheduler-event-loop INFO DAGScheduler: Missing parents: List() 20/01/23 18:53:59.639 dispatcher-event-loop-2 INFO TaskSetManager: Starting task 0.0 in stage 0.0 (TID 0, 192.168.1.225, executor driver, partition 0, PROCESS_LOCAL, 7373 bytes) 20/01/23 18:53:59.652 dispatcher-event-loop-2 DEBUG TaskSetManager: No tasks for locality level NO_PREF, so moving to locality level ANY 20/01/23 18:53:59.658 dag-scheduler-event-loop DEBUG DAGScheduler: submitStage(ResultStage 0 (name= at /usr/lib/python3.6/runpy.py:85;jobs=0)) 20/01/23 18:53:59.660 dag-scheduler-event-loop DEBUG DAGScheduler: missing: List() 20/01/23 18:53:59.662 dag-scheduler-event-loop INFO DAGScheduler: Submitting ResultStage 0 (MapPartitionsRDD[6] at at /usr/lib/python3.6/runpy.py:85), which has no missing parents 20/01/23 18:53:59.672 dag-scheduler-event-loop DEBUG DAGScheduler: submitMissingTasks(ResultStage 0) 20/01/23 18:53:59.696 Executor task launch worker for task 0 INFO Executor: Running task 0.0 in stage 0.0 (TID 0) 20/01/23 18:53:59.711 Executor task launch worker for task 0 DEBUG ExecutorMetricsPoller: stageTCMP: (0, 0) -> 1 20/01/23 18:53:59.718 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_1 stored as values in memory (estimated size 24.9 KiB, free 366.3 MiB) 20/01/23 18:53:59.720 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_1 locally took 19 ms 20/01/23 18:53:59.721 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_1 without replication took 20 ms 20/01/23 18:53:59.727 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_1_piece0 stored as bytes in memory (estimated size 10.5 KiB, free 366.3 MiB) 20/01/23 18:53:59.727 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added broadcast_1_piece0 in memory on 192.168.1.225:41233 (size: 10.5 KiB, free: 366.3 MiB) 20/01/23 18:53:59.728 dag-scheduler-event-loop DEBUG BlockManagerMaster: Updated info of block broadcast_1_piece0 20/01/23 18:53:59.728 dag-scheduler-event-loop DEBUG BlockManager: Told master about block broadcast_1_piece0 20/01/23 18:53:59.728 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_1_piece0 locally took 2 ms 20/01/23 18:53:59.728 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_1_piece0 without replication took 2 ms 20/01/23 18:53:59.729 dag-scheduler-event-loop INFO SparkContext: Created broadcast 1 from broadcast at DAGScheduler.scala:1200 20/01/23 18:53:59.763 dag-scheduler-event-loop INFO DAGScheduler: Submitting 4 missing tasks from ResultStage 0 (MapPartitionsRDD[6] at at /usr/lib/python3.6/runpy.py:85) (first 15 tasks are for partitions Vector(0, 1, 2, 3)) 20/01/23 18:53:59.764 dag-scheduler-event-loop INFO TaskSchedulerImpl: Adding task set 0.0 with 4 tasks 20/01/23 18:53:59.809 Executor task launch worker for task 0 DEBUG BlockManager: Getting local block broadcast_0 20/01/23 18:53:59.810 Executor task launch worker for task 0 DEBUG BlockManager: Level for block broadcast_0 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:53:59.849 dag-scheduler-event-loop DEBUG TaskSetManager: Epoch for TaskSet 0.0: 0 20/01/23 18:53:59.855 dag-scheduler-event-loop DEBUG TaskSetManager: Adding pending tasks took 4 ms 20/01/23 18:53:59.857 dag-scheduler-event-loop DEBUG TaskSetManager: Valid locality levels for TaskSet 0.0: NO_PREF, ANY 20/01/23 18:53:59.872 dispatcher-event-loop-2 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_0.0, runningTasks: 0 20/01/23 18:53:59.876 dispatcher-event-loop-2 DEBUG TaskSetManager: Valid locality levels for TaskSet 0.0: NO_PREF, ANY 20/01/23 18:53:59.911 dispatcher-event-loop-2 INFO TaskSetManager: Starting task 0.0 in stage 0.0 (TID 0, 192.168.1.225, executor driver, partition 0, PROCESS_LOCAL, 7333 bytes) 20/01/23 18:53:59.928 dispatcher-event-loop-2 INFO TaskSetManager: Starting task 1.0 in stage 0.0 (TID 1, 192.168.1.225, executor driver, partition 1, PROCESS_LOCAL, 7421 bytes) 20/01/23 18:53:59.929 dispatcher-event-loop-2 INFO TaskSetManager: Starting task 2.0 in stage 0.0 (TID 2, 192.168.1.225, executor driver, partition 2, PROCESS_LOCAL, 7396 bytes) 20/01/23 18:53:59.929 dispatcher-event-loop-2 INFO TaskSetManager: Starting task 3.0 in stage 0.0 (TID 3, 192.168.1.225, executor driver, partition 3, PROCESS_LOCAL, 7385 bytes) 20/01/23 18:53:59.987 Executor task launch worker for task 0 INFO Executor: Running task 0.0 in stage 0.0 (TID 0) 20/01/23 18:53:59.989 Executor task launch worker for task 2 INFO Executor: Running task 2.0 in stage 0.0 (TID 2) 20/01/23 18:53:59.990 Executor task launch worker for task 1 INFO Executor: Running task 1.0 in stage 0.0 (TID 1) 20/01/23 18:53:59.991 Executor task launch worker for task 3 INFO Executor: Running task 3.0 in stage 0.0 (TID 3) 20/01/23 18:54:00.003 Executor task launch worker for task 2 DEBUG ExecutorMetricsPoller: stageTCMP: (0, 0) -> 1 20/01/23 18:54:00.006 Executor task launch worker for task 1 DEBUG ExecutorMetricsPoller: stageTCMP: (0, 0) -> 3 20/01/23 18:54:00.007 Executor task launch worker for task 3 DEBUG ExecutorMetricsPoller: stageTCMP: (0, 0) -> 4 20/01/23 18:54:00.003 Executor task launch worker for task 0 DEBUG ExecutorMetricsPoller: stageTCMP: (0, 0) -> 2 20/01/23 18:54:00.110 Executor task launch worker for task 1 DEBUG BlockManager: Getting local block broadcast_1 20/01/23 18:54:00.112 Executor task launch worker for task 1 DEBUG BlockManager: Level for block broadcast_1 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:00.127 Executor task launch worker for task 3 DEBUG GenerateMutableProjection: code for cast(input[0, bigint, false] as double): /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificMutableProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificMutableProjection extends org.apache.spark.sql.catalyst.expressions.codegen.BaseMutableProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private InternalRow mutableRow; /* 009 */ private double value_2; /* 010 */ /* 011 */ public SpecificMutableProjection(Object[] references) { /* 012 */ this.references = references; /* 013 */ mutableRow = new org.apache.spark.sql.catalyst.expressions.GenericInternalRow(1); /* 014 */ /* 015 */ /* 016 */ } /* 017 */ /* 018 */ public void initialize(int partitionIndex) { /* 019 */ /* 020 */ } /* 021 */ /* 022 */ public org.apache.spark.sql.catalyst.expressions.codegen.BaseMutableProjection target(InternalRow row) { /* 023 */ mutableRow = row; /* 024 */ return this; /* 025 */ } /* 026 */ /* 027 */ /* Provide immutable access to the last projected row. */ /* 028 */ public InternalRow currentValue() { /* 029 */ return (InternalRow) mutableRow; /* 030 */ } /* 031 */ /* 032 */ public java.lang.Object apply(java.lang.Object _i) { /* 033 */ InternalRow i = (InternalRow) _i; /* 034 */ /* 035 */ /* 036 */ long value_1 = i.getLong(0); /* 037 */ boolean isNull_0 = false; /* 038 */ double value_0 = -1.0; /* 039 */ if (!false) { /* 040 */ value_0 = (double) value_1; /* 041 */ } /* 042 */ value_2 = value_0; /* 043 */ /* 044 */ // copy all the results into MutableRow /* 045 */ mutableRow.setDouble(0, value_2); /* 046 */ return mutableRow; /* 047 */ } /* 048 */ /* 049 */ /* 050 */ } 20/01/23 18:54:00.129 Executor task launch worker for task 1 DEBUG GenerateMutableProjection: code for cast(input[0, bigint, false] as double): /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificMutableProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificMutableProjection extends org.apache.spark.sql.catalyst.expressions.codegen.BaseMutableProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private InternalRow mutableRow; /* 009 */ private double value_2; /* 010 */ /* 011 */ public SpecificMutableProjection(Object[] references) { /* 012 */ this.references = references; /* 013 */ mutableRow = new org.apache.spark.sql.catalyst.expressions.GenericInternalRow(1); /* 014 */ /* 015 */ /* 016 */ } /* 017 */ /* 018 */ public void initialize(int partitionIndex) { /* 019 */ /* 020 */ } /* 021 */ /* 022 */ public org.apache.spark.sql.catalyst.expressions.codegen.BaseMutableProjection target(InternalRow row) { /* 023 */ mutableRow = row; /* 024 */ return this; /* 025 */ } /* 026 */ /* 027 */ /* Provide immutable access to the last projected row. */ /* 028 */ public InternalRow currentValue() { /* 029 */ return (InternalRow) mutableRow; /* 030 */ } /* 031 */ /* 032 */ public java.lang.Object apply(java.lang.Object _i) { /* 033 */ InternalRow i = (InternalRow) _i; /* 034 */ /* 035 */ /* 036 */ long value_1 = i.getLong(0); /* 037 */ boolean isNull_0 = false; /* 038 */ double value_0 = -1.0; /* 039 */ if (!false) { /* 040 */ value_0 = (double) value_1; /* 041 */ } /* 042 */ value_2 = value_0; /* 043 */ /* 044 */ // copy all the results into MutableRow /* 045 */ mutableRow.setDouble(0, value_2); /* 046 */ return mutableRow; /* 047 */ } /* 048 */ /* 049 */ /* 050 */ } 20/01/23 18:54:00.147 Executor task launch worker for task 0 DEBUG GenerateMutableProjection: code for cast(input[0, bigint, false] as double): /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificMutableProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificMutableProjection extends org.apache.spark.sql.catalyst.expressions.codegen.BaseMutableProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private InternalRow mutableRow; /* 009 */ private double value_2; /* 010 */ /* 011 */ public SpecificMutableProjection(Object[] references) { /* 012 */ this.references = references; /* 013 */ mutableRow = new org.apache.spark.sql.catalyst.expressions.GenericInternalRow(1); /* 014 */ /* 015 */ /* 016 */ } /* 017 */ /* 018 */ public void initialize(int partitionIndex) { /* 019 */ /* 020 */ } /* 021 */ /* 022 */ public org.apache.spark.sql.catalyst.expressions.codegen.BaseMutableProjection target(InternalRow row) { /* 023 */ mutableRow = row; /* 024 */ return this; /* 025 */ } /* 026 */ /* 027 */ /* Provide immutable access to the last projected row. */ /* 028 */ public InternalRow currentValue() { /* 029 */ return (InternalRow) mutableRow; /* 030 */ } /* 031 */ /* 032 */ public java.lang.Object apply(java.lang.Object _i) { /* 033 */ InternalRow i = (InternalRow) _i; /* 034 */ /* 035 */ /* 036 */ long value_1 = i.getLong(0); /* 037 */ boolean isNull_0 = false; /* 038 */ double value_0 = -1.0; /* 039 */ if (!false) { /* 040 */ value_0 = (double) value_1; /* 041 */ } /* 042 */ value_2 = value_0; /* 043 */ /* 044 */ // copy all the results into MutableRow /* 045 */ mutableRow.setDouble(0, value_2); /* 046 */ return mutableRow; /* 047 */ } /* 048 */ /* 049 */ /* 050 */ } 20/01/23 18:54:00.141 Executor task launch worker for task 3 DEBUG CodeGenerator: /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificMutableProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificMutableProjection extends org.apache.spark.sql.catalyst.expressions.codegen.BaseMutableProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private InternalRow mutableRow; /* 009 */ private double value_2; /* 010 */ /* 011 */ public SpecificMutableProjection(Object[] references) { /* 012 */ this.references = references; /* 013 */ mutableRow = new org.apache.spark.sql.catalyst.expressions.GenericInternalRow(1); /* 014 */ /* 015 */ /* 016 */ } /* 017 */ /* 018 */ public void initialize(int partitionIndex) { /* 019 */ /* 020 */ } /* 021 */ /* 022 */ public org.apache.spark.sql.catalyst.expressions.codegen.BaseMutableProjection target(InternalRow row) { /* 023 */ mutableRow = row; /* 024 */ return this; /* 025 */ } /* 026 */ /* 027 */ /* Provide immutable access to the last projected row. */ /* 028 */ public InternalRow currentValue() { /* 029 */ return (InternalRow) mutableRow; /* 030 */ } /* 031 */ /* 032 */ public java.lang.Object apply(java.lang.Object _i) { /* 033 */ InternalRow i = (InternalRow) _i; /* 034 */ /* 035 */ /* 036 */ long value_1 = i.getLong(0); /* 037 */ boolean isNull_0 = false; /* 038 */ double value_0 = -1.0; /* 039 */ if (!false) { /* 040 */ value_0 = (double) value_1; /* 041 */ } /* 042 */ value_2 = value_0; /* 043 */ /* 044 */ // copy all the results into MutableRow /* 045 */ mutableRow.setDouble(0, value_2); /* 046 */ return mutableRow; /* 047 */ } /* 048 */ /* 049 */ /* 050 */ } 20/01/23 18:54:00.158 Executor task launch worker for task 2 DEBUG GenerateMutableProjection: code for cast(input[0, bigint, false] as double): /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificMutableProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificMutableProjection extends org.apache.spark.sql.catalyst.expressions.codegen.BaseMutableProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private InternalRow mutableRow; /* 009 */ private double value_2; /* 010 */ /* 011 */ public SpecificMutableProjection(Object[] references) { /* 012 */ this.references = references; /* 013 */ mutableRow = new org.apache.spark.sql.catalyst.expressions.GenericInternalRow(1); /* 014 */ /* 015 */ /* 016 */ } /* 017 */ /* 018 */ public void initialize(int partitionIndex) { /* 019 */ /* 020 */ } /* 021 */ /* 022 */ public org.apache.spark.sql.catalyst.expressions.codegen.BaseMutableProjection target(InternalRow row) { /* 023 */ mutableRow = row; /* 024 */ return this; /* 025 */ } /* 026 */ /* 027 */ /* Provide immutable access to the last projected row. */ /* 028 */ public InternalRow currentValue() { /* 029 */ return (InternalRow) mutableRow; /* 030 */ } /* 031 */ /* 032 */ public java.lang.Object apply(java.lang.Object _i) { /* 033 */ InternalRow i = (InternalRow) _i; /* 034 */ /* 035 */ /* 036 */ long value_1 = i.getLong(0); /* 037 */ boolean isNull_0 = false; /* 038 */ double value_0 = -1.0; /* 039 */ if (!false) { /* 040 */ value_0 = (double) value_1; /* 041 */ } /* 042 */ value_2 = value_0; /* 043 */ /* 044 */ // copy all the results into MutableRow /* 045 */ mutableRow.setDouble(0, value_2); /* 046 */ return mutableRow; /* 047 */ } /* 048 */ /* 049 */ /* 050 */ } 20/01/23 18:54:00.174 Executor task launch worker for task 3 INFO CodeGenerator: Code generated in 46.748452 ms 20/01/23 18:54:00.876 stdout writer for /home/jenkins/venv/py36/bin/python3.6 DEBUG TaskMemoryManager: Task 0 acquired 4.0 MiB for HybridRowQueue(org.apache.spark.memory.TaskMemoryManager@7c9615c9,/home/jenkins/workspace/spark-master-test-python-arm/python/target/22a04de0-c639-4801-aead-76fbc7eb088f/spark-9be496c1-624c-4a61-9fa3-b7ee419d20cf,1,org.apache.spark.serializer.SerializerManager@2b0ea9d1) 20/01/23 18:54:00.878 stdout writer for /home/jenkins/venv/py36/bin/python3.6 DEBUG TaskMemoryManager: Task 3 acquired 4.0 MiB for HybridRowQueue(org.apache.spark.memory.TaskMemoryManager@678f2472,/home/jenkins/workspace/spark-master-test-python-arm/python/target/22a04de0-c639-4801-aead-76fbc7eb088f/spark-9be496c1-624c-4a61-9fa3-b7ee419d20cf,1,org.apache.spark.serializer.SerializerManager@2b0ea9d1) 20/01/23 18:54:00.885 stdout writer for /home/jenkins/venv/py36/bin/python3.6 DEBUG TaskMemoryManager: Task 2 acquired 4.0 MiB for HybridRowQueue(org.apache.spark.memory.TaskMemoryManager@6f4728b,/home/jenkins/workspace/spark-master-test-python-arm/python/target/22a04de0-c639-4801-aead-76fbc7eb088f/spark-9be496c1-624c-4a61-9fa3-b7ee419d20cf,1,org.apache.spark.serializer.SerializerManager@2b0ea9d1) 20/01/23 18:54:00.894 Executor task launch worker for task 3 DEBUG GenerateUnsafeProjection: code for input[0, bigint, false],input[1, double, true]: /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificUnsafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificUnsafeProjection extends org.apache.spark.sql.catalyst.expressions.UnsafeProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[] mutableStateArray_0 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[1]; /* 009 */ /* 010 */ public SpecificUnsafeProjection(Object[] references) { /* 011 */ this.references = references; /* 012 */ mutableStateArray_0[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(2, 0); /* 013 */ /* 014 */ } /* 015 */ /* 016 */ public void initialize(int partitionIndex) { /* 017 */ /* 018 */ } /* 019 */ /* 020 */ // Scala.Function1 need this /* 021 */ public java.lang.Object apply(java.lang.Object row) { /* 022 */ return apply((InternalRow) row); /* 023 */ } /* 024 */ /* 025 */ public UnsafeRow apply(InternalRow i) { /* 026 */ mutableStateArray_0[0].reset(); /* 027 */ /* 028 */ /* 029 */ mutableStateArray_0[0].zeroOutNullBytes(); /* 030 */ /* 031 */ long value_0 = i.getLong(0); /* 032 */ mutableStateArray_0[0].write(0, value_0); /* 033 */ /* 034 */ boolean isNull_1 = i.isNullAt(1); /* 035 */ double value_1 = isNull_1 ? /* 036 */ -1.0 : (i.getDouble(1)); /* 037 */ if (isNull_1) { /* 038 */ mutableStateArray_0[0].setNullAt(1); /* 039 */ } else { /* 040 */ mutableStateArray_0[0].write(1, value_1); /* 041 */ } /* 042 */ return (mutableStateArray_0[0].getRow()); /* 043 */ } /* 044 */ /* 045 */ /* 046 */ } 20/01/23 18:54:00.904 stdout writer for /home/jenkins/venv/py36/bin/python3.6 DEBUG TaskMemoryManager: Task 1 acquired 4.0 MiB for HybridRowQueue(org.apache.spark.memory.TaskMemoryManager@1f547e3d,/home/jenkins/workspace/spark-master-test-python-arm/python/target/22a04de0-c639-4801-aead-76fbc7eb088f/spark-9be496c1-624c-4a61-9fa3-b7ee419d20cf,1,org.apache.spark.serializer.SerializerManager@2b0ea9d1) 20/01/23 18:54:00.922 Executor task launch worker for task 1 DEBUG GenerateUnsafeProjection: code for input[0, bigint, false],input[1, double, true]: /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificUnsafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificUnsafeProjection extends org.apache.spark.sql.catalyst.expressions.UnsafeProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[] mutableStateArray_0 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[1]; /* 009 */ /* 010 */ public SpecificUnsafeProjection(Object[] references) { /* 011 */ this.references = references; /* 012 */ mutableStateArray_0[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(2, 0); /* 013 */ /* 014 */ } /* 015 */ /* 016 */ public void initialize(int partitionIndex) { /* 017 */ /* 018 */ } /* 019 */ /* 020 */ // Scala.Function1 need this /* 021 */ public java.lang.Object apply(java.lang.Object row) { /* 022 */ return apply((InternalRow) row); /* 023 */ } /* 024 */ /* 025 */ public UnsafeRow apply(InternalRow i) { /* 026 */ mutableStateArray_0[0].reset(); /* 027 */ /* 028 */ /* 029 */ mutableStateArray_0[0].zeroOutNullBytes(); /* 030 */ /* 031 */ long value_0 = i.getLong(0); /* 032 */ mutableStateArray_0[0].write(0, value_0); /* 033 */ /* 034 */ boolean isNull_1 = i.isNullAt(1); /* 035 */ double value_1 = isNull_1 ? /* 036 */ -1.0 : (i.getDouble(1)); /* 037 */ if (isNull_1) { /* 038 */ mutableStateArray_0[0].setNullAt(1); /* 039 */ } else { /* 040 */ mutableStateArray_0[0].write(1, value_1); /* 041 */ } /* 042 */ return (mutableStateArray_0[0].getRow()); /* 043 */ } /* 044 */ /* 045 */ /* 046 */ } 20/01/23 18:54:00.924 Executor task launch worker for task 2 DEBUG GenerateUnsafeProjection: code for input[0, bigint, false],input[1, double, true]: /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificUnsafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificUnsafeProjection extends org.apache.spark.sql.catalyst.expressions.UnsafeProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[] mutableStateArray_0 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[1]; /* 009 */ /* 010 */ public SpecificUnsafeProjection(Object[] references) { /* 011 */ this.references = references; /* 012 */ mutableStateArray_0[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(2, 0); /* 013 */ /* 014 */ } /* 015 */ /* 016 */ public void initialize(int partitionIndex) { /* 017 */ /* 018 */ } /* 019 */ /* 020 */ // Scala.Function1 need this /* 021 */ public java.lang.Object apply(java.lang.Object row) { /* 022 */ return apply((InternalRow) row); /* 023 */ } /* 024 */ /* 025 */ public UnsafeRow apply(InternalRow i) { /* 026 */ mutableStateArray_0[0].reset(); /* 027 */ /* 028 */ /* 029 */ mutableStateArray_0[0].zeroOutNullBytes(); /* 030 */ /* 031 */ long value_0 = i.getLong(0); /* 032 */ mutableStateArray_0[0].write(0, value_0); /* 033 */ /* 034 */ boolean isNull_1 = i.isNullAt(1); /* 035 */ double value_1 = isNull_1 ? /* 036 */ -1.0 : (i.getDouble(1)); /* 037 */ if (isNull_1) { /* 038 */ mutableStateArray_0[0].setNullAt(1); /* 039 */ } else { /* 040 */ mutableStateArray_0[0].write(1, value_1); /* 041 */ } /* 042 */ return (mutableStateArray_0[0].getRow()); /* 043 */ } /* 044 */ /* 045 */ /* 046 */ } 20/01/23 18:54:00.924 Executor task launch worker for task 0 DEBUG GenerateUnsafeProjection: code for input[0, bigint, false],input[1, double, true]: /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificUnsafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificUnsafeProjection extends org.apache.spark.sql.catalyst.expressions.UnsafeProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[] mutableStateArray_0 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[1]; /* 009 */ /* 010 */ public SpecificUnsafeProjection(Object[] references) { /* 011 */ this.references = references; /* 012 */ mutableStateArray_0[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(2, 0); /* 013 */ /* 014 */ } /* 015 */ /* 016 */ public void initialize(int partitionIndex) { /* 017 */ /* 018 */ } /* 019 */ /* 020 */ // Scala.Function1 need this /* 021 */ public java.lang.Object apply(java.lang.Object row) { /* 022 */ return apply((InternalRow) row); /* 023 */ } /* 024 */ /* 025 */ public UnsafeRow apply(InternalRow i) { /* 026 */ mutableStateArray_0[0].reset(); /* 027 */ /* 028 */ /* 029 */ mutableStateArray_0[0].zeroOutNullBytes(); /* 030 */ /* 031 */ long value_0 = i.getLong(0); /* 032 */ mutableStateArray_0[0].write(0, value_0); /* 033 */ /* 034 */ boolean isNull_1 = i.isNullAt(1); /* 035 */ double value_1 = isNull_1 ? /* 036 */ -1.0 : (i.getDouble(1)); /* 037 */ if (isNull_1) { /* 038 */ mutableStateArray_0[0].setNullAt(1); /* 039 */ } else { /* 040 */ mutableStateArray_0[0].write(1, value_1); /* 041 */ } /* 042 */ return (mutableStateArray_0[0].getRow()); /* 043 */ } /* 044 */ /* 045 */ /* 046 */ } 20/01/23 18:54:00.930 Executor task launch worker for task 3 DEBUG CodeGenerator: /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificUnsafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificUnsafeProjection extends org.apache.spark.sql.catalyst.expressions.UnsafeProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[] mutableStateArray_0 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[1]; /* 009 */ /* 010 */ public SpecificUnsafeProjection(Object[] references) { /* 011 */ this.references = references; /* 012 */ mutableStateArray_0[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(2, 0); /* 013 */ /* 014 */ } /* 015 */ /* 016 */ public void initialize(int partitionIndex) { /* 017 */ /* 018 */ } /* 019 */ /* 020 */ // Scala.Function1 need this /* 021 */ public java.lang.Object apply(java.lang.Object row) { /* 022 */ return apply((InternalRow) row); /* 023 */ } /* 024 */ /* 025 */ public UnsafeRow apply(InternalRow i) { /* 026 */ mutableStateArray_0[0].reset(); /* 027 */ /* 028 */ /* 029 */ mutableStateArray_0[0].zeroOutNullBytes(); /* 030 */ /* 031 */ long value_0 = i.getLong(0); /* 032 */ mutableStateArray_0[0].write(0, value_0); /* 033 */ /* 034 */ boolean isNull_1 = i.isNullAt(1); /* 035 */ double value_1 = isNull_1 ? /* 036 */ -1.0 : (i.getDouble(1)); /* 037 */ if (isNull_1) { /* 038 */ mutableStateArray_0[0].setNullAt(1); /* 039 */ } else { /* 040 */ mutableStateArray_0[0].write(1, value_1); /* 041 */ } /* 042 */ return (mutableStateArray_0[0].getRow()); /* 043 */ } /* 044 */ /* 045 */ /* 046 */ } 20/01/23 18:54:00.952 Executor task launch worker for task 3 INFO CodeGenerator: Code generated in 57.78031 ms 20/01/23 18:54:00.952 Executor task launch worker for task 0 INFO Executor: Finished task 0.0 in stage 0.0 (TID 0). 2023 bytes result sent to driver 20/01/23 18:54:00.954 Executor task launch worker for task 0 DEBUG ExecutorMetricsPoller: removing (0, 0) from stageTCMP 20/01/23 18:54:00.965 dispatcher-event-loop-0 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_0.0, runningTasks: 0 20/01/23 18:54:00.963 Executor task launch worker for task 1 INFO PythonUDFRunner: Times: total = 717, boot = 693, init = 24, finish = 0 20/01/23 18:54:00.967 Executor task launch worker for task 3 INFO PythonUDFRunner: Times: total = 700, boot = 670, init = 30, finish = 0 20/01/23 18:54:00.963 Executor task launch worker for task 0 INFO PythonUDFRunner: Times: total = 699, boot = 665, init = 34, finish = 0 20/01/23 18:54:00.963 Executor task launch worker for task 2 INFO PythonUDFRunner: Times: total = 700, boot = 680, init = 20, finish = 0 20/01/23 18:54:00.970 Executor task launch worker for task 2 DEBUG TaskMemoryManager: Task 2 release 4.0 MiB from HybridRowQueue(org.apache.spark.memory.TaskMemoryManager@6f4728b,/home/jenkins/workspace/spark-master-test-python-arm/python/target/22a04de0-c639-4801-aead-76fbc7eb088f/spark-9be496c1-624c-4a61-9fa3-b7ee419d20cf,1,org.apache.spark.serializer.SerializerManager@2b0ea9d1) 20/01/23 18:54:00.972 Executor task launch worker for task 0 DEBUG TaskMemoryManager: Task 0 release 4.0 MiB from HybridRowQueue(org.apache.spark.memory.TaskMemoryManager@7c9615c9,/home/jenkins/workspace/spark-master-test-python-arm/python/target/22a04de0-c639-4801-aead-76fbc7eb088f/spark-9be496c1-624c-4a61-9fa3-b7ee419d20cf,1,org.apache.spark.serializer.SerializerManager@2b0ea9d1) 20/01/23 18:54:00.972 Executor task launch worker for task 3 DEBUG TaskMemoryManager: Task 3 release 4.0 MiB from HybridRowQueue(org.apache.spark.memory.TaskMemoryManager@678f2472,/home/jenkins/workspace/spark-master-test-python-arm/python/target/22a04de0-c639-4801-aead-76fbc7eb088f/spark-9be496c1-624c-4a61-9fa3-b7ee419d20cf,1,org.apache.spark.serializer.SerializerManager@2b0ea9d1) 20/01/23 18:54:00.973 Executor task launch worker for task 1 DEBUG TaskMemoryManager: Task 1 release 4.0 MiB from HybridRowQueue(org.apache.spark.memory.TaskMemoryManager@1f547e3d,/home/jenkins/workspace/spark-master-test-python-arm/python/target/22a04de0-c639-4801-aead-76fbc7eb088f/spark-9be496c1-624c-4a61-9fa3-b7ee419d20cf,1,org.apache.spark.serializer.SerializerManager@2b0ea9d1) 20/01/23 18:54:00.975 task-result-getter-0 INFO TaskSetManager: Finished task 0.0 in stage 0.0 (TID 0) in 1721 ms on 192.168.1.225 (executor driver) (1/1) 20/01/23 18:54:00.979 task-result-getter-0 INFO TaskSchedulerImpl: Removed TaskSet 0.0, whose tasks have all completed, from pool 20/01/23 18:54:00.984 dag-scheduler-event-loop INFO PythonAccumulatorV2: Connected to AccumulatorServer at host: 127.0.0.1 port: 39457 20/01/23 18:54:00.992 dag-scheduler-event-loop INFO DAGScheduler: ResultStage 0 (head at MetadataUtils.scala:51) finished in 2.104 s 20/01/23 18:54:00.993 Executor task launch worker for task 0 INFO Executor: Finished task 0.0 in stage 0.0 (TID 0). 2236 bytes result sent to driver 20/01/23 18:54:00.996 dag-scheduler-event-loop DEBUG DAGScheduler: After removal of stage 0, remaining stages = 0 20/01/23 18:54:00.996 dag-scheduler-event-loop INFO DAGScheduler: Job 0 is finished. Cancelling potential speculative or zombie tasks for this job 20/01/23 18:54:00.997 Executor task launch worker for task 0 DEBUG ExecutorMetricsPoller: stageTCMP: (0, 0) -> 3 20/01/23 18:54:00.998 Executor task launch worker for task 3 INFO Executor: Finished task 3.0 in stage 0.0 (TID 3). 2258 bytes result sent to driver 20/01/23 18:54:00.998 Executor task launch worker for task 3 DEBUG ExecutorMetricsPoller: stageTCMP: (0, 0) -> 2 20/01/23 18:54:00.998 Executor task launch worker for task 1 INFO Executor: Finished task 1.0 in stage 0.0 (TID 1). 2256 bytes result sent to driver 20/01/23 18:54:00.998 Executor task launch worker for task 1 DEBUG ExecutorMetricsPoller: stageTCMP: (0, 0) -> 1 20/01/23 18:54:00.998 dispatcher-event-loop-3 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_0.0, runningTasks: 3 20/01/23 18:54:00.999 dispatcher-event-loop-3 DEBUG TaskSetManager: No tasks for locality level NO_PREF, so moving to locality level ANY 20/01/23 18:54:01.001 dag-scheduler-event-loop INFO TaskSchedulerImpl: Killing all running tasks in stage 0: Stage finished 20/01/23 18:54:01.004 dispatcher-event-loop-3 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_0.0, runningTasks: 2 20/01/23 18:54:00.993 Executor task launch worker for task 2 INFO Executor: Finished task 2.0 in stage 0.0 (TID 2). 2244 bytes result sent to driver 20/01/23 18:54:01.006 Executor task launch worker for task 2 DEBUG ExecutorMetricsPoller: removing (0, 0) from stageTCMP 20/01/23 18:54:01.013 dispatcher-event-loop-3 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_0.0, runningTasks: 1 20/01/23 18:54:01.017 Thread-4 INFO DAGScheduler: Job 0 finished: head at MetadataUtils.scala:51, took 2.211876 s 20/01/23 18:54:01.020 dispatcher-event-loop-3 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_0.0, runningTasks: 0 20/01/23 18:54:01.021 task-result-getter-0 INFO TaskSetManager: Finished task 0.0 in stage 0.0 (TID 0) in 1783 ms on 192.168.1.225 (executor driver) (1/4) 20/01/23 18:54:01.023 task-result-getter-2 INFO TaskSetManager: Finished task 1.0 in stage 0.0 (TID 1) in 1729 ms on 192.168.1.225 (executor driver) (2/4) 20/01/23 18:54:01.025 task-result-getter-1 INFO TaskSetManager: Finished task 3.0 in stage 0.0 (TID 3) in 1730 ms on 192.168.1.225 (executor driver) (3/4) 20/01/23 18:54:01.025 task-result-getter-3 INFO TaskSetManager: Finished task 2.0 in stage 0.0 (TID 2) in 1730 ms on 192.168.1.225 (executor driver) (4/4) 20/01/23 18:54:01.026 task-result-getter-3 INFO TaskSchedulerImpl: Removed TaskSet 0.0, whose tasks have all completed, from pool 20/01/23 18:54:01.027 dag-scheduler-event-loop INFO PythonAccumulatorV2: Connected to AccumulatorServer at host: 127.0.0.1 port: 53019 20/01/23 18:54:01.039 dag-scheduler-event-loop INFO DAGScheduler: ResultStage 0 ( at /usr/lib/python3.6/runpy.py:85) finished in 2.101 s 20/01/23 18:54:01.043 dag-scheduler-event-loop DEBUG DAGScheduler: After removal of stage 0, remaining stages = 0 20/01/23 18:54:01.044 dag-scheduler-event-loop INFO DAGScheduler: Job 0 is finished. Cancelling potential speculative or zombie tasks for this job 20/01/23 18:54:01.044 dag-scheduler-event-loop INFO TaskSchedulerImpl: Killing all running tasks in stage 0: Stage finished 20/01/23 18:54:01.051 Thread-4 INFO DAGScheduler: Job 0 finished: at /usr/lib/python3.6/runpy.py:85, took 2.182880 s 20/01/23 18:54:01.061 Thread-4 DEBUG GenerateSafeProjection: code for createexternalrow(newInstance(class org.apache.spark.ml.linalg.VectorUDT).deserialize, StructField(features,org.apache.spark.ml.linalg.VectorUDT@3bfc3ba7,true)): /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificSafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificSafeProjection extends org.apache.spark.sql.catalyst.expressions.codegen.BaseProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private InternalRow mutableRow; /* 009 */ private boolean resultIsNull_0; /* 010 */ private InternalRow[] mutableStateArray_0 = new InternalRow[1]; /* 011 */ /* 012 */ public SpecificSafeProjection(Object[] references) { /* 013 */ this.references = references; /* 014 */ mutableRow = (InternalRow) references[references.length - 1]; /* 015 */ /* 016 */ /* 017 */ } /* 018 */ /* 019 */ public void initialize(int partitionIndex) { /* 020 */ /* 021 */ } /* 022 */ /* 023 */ public java.lang.Object apply(java.lang.Object _i) { /* 024 */ InternalRow i = (InternalRow) _i; /* 025 */ org.apache.spark.sql.Row value_4 = CreateExternalRow_0(i); /* 026 */ if (false) { /* 027 */ mutableRow.setNullAt(0); /* 028 */ } else { /* 029 */ /* 030 */ mutableRow.update(0, value_4); /* 031 */ } /* 032 */ /* 033 */ return mutableRow; /* 034 */ } /* 035 */ /* 036 */ /* 037 */ private org.apache.spark.sql.Row CreateExternalRow_0(InternalRow i) { /* 038 */ Object[] values_0 = new Object[1]; /* 039 */ /* 040 */ final org.apache.spark.ml.linalg.VectorUDT value_2 = false ? /* 041 */ null : new org.apache.spark.ml.linalg.VectorUDT(); /* 042 */ boolean isNull_1 = true; /* 043 */ org.apache.spark.ml.linalg.Vector value_1 = null; /* 044 */ if (!false) { /* 045 */ resultIsNull_0 = false; /* 046 */ if (!resultIsNull_0) { /* 047 */ boolean isNull_3 = i.isNullAt(0); /* 048 */ InternalRow value_3 = isNull_3 ? /* 049 */ null : (i.getStruct(0, 4)); /* 050 */ resultIsNull_0 = isNull_3; /* 051 */ mutableStateArray_0[0] = value_3; /* 052 */ } /* 053 */ /* 054 */ isNull_1 = resultIsNull_0; /* 055 */ if (!isNull_1) { /* 056 */ /* 057 */ Object funcResult_0 = null; /* 058 */ funcResult_0 = value_2.deserialize(mutableStateArray_0[0]); /* 059 */ /* 060 */ if (funcResult_0 != null) { /* 061 */ value_1 = (org.apache.spark.ml.linalg.Vector) funcResult_0; /* 062 */ } else { /* 063 */ isNull_1 = true; /* 064 */ } /* 065 */ /* 066 */ /* 067 */ } /* 068 */ } /* 069 */ if (isNull_1) { /* 070 */ values_0[0] = null; /* 071 */ } else { /* 072 */ values_0[0] = value_1; /* 073 */ } /* 074 */ /* 075 */ final org.apache.spark.sql.Row value_0 = new org.apache.spark.sql.catalyst.expressions.GenericRowWithSchema(values_0, ((org.apache.spark.sql.types.StructType) references[0] /* schema */)); /* 076 */ /* 077 */ return value_0; /* 078 */ } /* 079 */ /* 080 */ } 20/01/23 18:54:01.065 Thread-4 DEBUG CodeGenerator: /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificSafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificSafeProjection extends org.apache.spark.sql.catalyst.expressions.codegen.BaseProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private InternalRow mutableRow; /* 009 */ private boolean resultIsNull_0; /* 010 */ private InternalRow[] mutableStateArray_0 = new InternalRow[1]; /* 011 */ /* 012 */ public SpecificSafeProjection(Object[] references) { /* 013 */ this.references = references; /* 014 */ mutableRow = (InternalRow) references[references.length - 1]; /* 015 */ /* 016 */ /* 017 */ } /* 018 */ /* 019 */ public void initialize(int partitionIndex) { /* 020 */ /* 021 */ } /* 022 */ /* 023 */ public java.lang.Object apply(java.lang.Object _i) { /* 024 */ InternalRow i = (InternalRow) _i; /* 025 */ org.apache.spark.sql.Row value_4 = CreateExternalRow_0(i); /* 026 */ if (false) { /* 027 */ mutableRow.setNullAt(0); /* 028 */ } else { /* 029 */ /* 030 */ mutableRow.update(0, value_4); /* 031 */ } /* 032 */ /* 033 */ return mutableRow; /* 034 */ } /* 035 */ /* 036 */ /* 037 */ private org.apache.spark.sql.Row CreateExternalRow_0(InternalRow i) { /* 038 */ Object[] values_0 = new Object[1]; /* 039 */ /* 040 */ final org.apache.spark.ml.linalg.VectorUDT value_2 = false ? /* 041 */ null : new org.apache.spark.ml.linalg.VectorUDT(); /* 042 */ boolean isNull_1 = true; /* 043 */ org.apache.spark.ml.linalg.Vector value_1 = null; /* 044 */ if (!false) { /* 045 */ resultIsNull_0 = false; /* 046 */ if (!resultIsNull_0) { /* 047 */ boolean isNull_3 = i.isNullAt(0); /* 048 */ InternalRow value_3 = isNull_3 ? /* 049 */ null : (i.getStruct(0, 4)); /* 050 */ resultIsNull_0 = isNull_3; /* 051 */ mutableStateArray_0[0] = value_3; /* 052 */ } /* 053 */ /* 054 */ isNull_1 = resultIsNull_0; /* 055 */ if (!isNull_1) { /* 056 */ /* 057 */ Object funcResult_0 = null; /* 058 */ funcResult_0 = value_2.deserialize(mutableStateArray_0[0]); /* 059 */ /* 060 */ if (funcResult_0 != null) { /* 061 */ value_1 = (org.apache.spark.ml.linalg.Vector) funcResult_0; /* 062 */ } else { /* 063 */ isNull_1 = true; /* 064 */ } /* 065 */ /* 066 */ /* 067 */ } /* 068 */ } /* 069 */ if (isNull_1) { /* 070 */ values_0[0] = null; /* 071 */ } else { /* 072 */ values_0[0] = value_1; /* 073 */ } /* 074 */ /* 075 */ final org.apache.spark.sql.Row value_0 = new org.apache.spark.sql.catalyst.expressions.GenericRowWithSchema(values_0, ((org.apache.spark.sql.types.StructType) references[0] /* schema */)); /* 076 */ /* 077 */ return value_0; /* 078 */ } /* 079 */ /* 080 */ } 20/01/23 18:54:01.092 Thread-4 INFO CodeGenerator: Code generated in 30.49898 ms 20/01/23 18:54:01.115 dispatcher-event-loop-2 INFO MapOutputTrackerMasterEndpoint: MapOutputTrackerMasterEndpoint stopped! 20/01/23 18:54:01.163 Thread-4 DEBUG Analyzer$ResolveReferences: Resolving 'prediction to prediction#1 20/01/23 18:54:01.164 Thread-4 DEBUG Analyzer$ResolveReferences: Resolving 'normalizedFeatures to normalizedFeatures#7 20/01/23 18:54:01.224 Thread-4 INFO MemoryStore: MemoryStore cleared 20/01/23 18:54:01.224 Thread-4 INFO BlockManager: BlockManager stopped 20/01/23 18:54:01.241 Thread-4 INFO BlockManagerMaster: BlockManagerMaster stopped 20/01/23 18:54:01.247 dispatcher-event-loop-3 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint: OutputCommitCoordinator stopped! 20/01/23 18:54:01.265 Thread-4 INFO SparkContext: Successfully stopped SparkContext 20/01/23 18:54:01.279 Executor task launch worker for task 0 DEBUG GenerateSafeProjection: code for createexternalrow(input[0, int, false], input[1, int, false], input[2, float, true], StructField(UDF(user),IntegerType,false), StructField(UDF(item),IntegerType,false), StructField(rating,FloatType,true)): /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificSafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificSafeProjection extends org.apache.spark.sql.catalyst.expressions.codegen.BaseProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private InternalRow mutableRow; /* 009 */ /* 010 */ /* 011 */ public SpecificSafeProjection(Object[] references) { /* 012 */ this.references = references; /* 013 */ mutableRow = (InternalRow) references[references.length - 1]; /* 014 */ /* 015 */ } /* 016 */ /* 017 */ public void initialize(int partitionIndex) { /* 018 */ /* 019 */ } /* 020 */ /* 021 */ public java.lang.Object apply(java.lang.Object _i) { /* 022 */ InternalRow i = (InternalRow) _i; /* 023 */ Object[] values_0 = new Object[3]; /* 024 */ /* 025 */ int value_1 = i.getInt(0); /* 026 */ if (false) { /* 027 */ values_0[0] = null; /* 028 */ } else { /* 029 */ values_0[0] = value_1; /* 030 */ } /* 031 */ /* 032 */ int value_2 = i.getInt(1); /* 033 */ if (false) { /* 034 */ values_0[1] = null; /* 035 */ } else { /* 036 */ values_0[1] = value_2; /* 037 */ } /* 038 */ /* 039 */ boolean isNull_3 = i.isNullAt(2); /* 040 */ float value_3 = isNull_3 ? /* 041 */ -1.0f : (i.getFloat(2)); /* 042 */ if (isNull_3) { /* 043 */ values_0[2] = null; /* 044 */ } else { /* 045 */ values_0[2] = value_3; /* 046 */ } /* 047 */ /* 048 */ final org.apache.spark.sql.Row value_0 = new org.apache.spark.sql.catalyst.expressions.GenericRowWithSchema(values_0, ((org.apache.spark.sql.types.StructType) references[0] /* schema */)); /* 049 */ if (false) { /* 050 */ mutableRow.setNullAt(0); /* 051 */ } else { /* 052 */ /* 053 */ mutableRow.update(0, value_0); /* 054 */ } /* 055 */ /* 056 */ return mutableRow; /* 057 */ } /* 058 */ /* 059 */ /* 060 */ } 20/01/23 18:54:01.284 Executor task launch worker for task 0 DEBUG CodeGenerator: /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificSafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificSafeProjection extends org.apache.spark.sql.catalyst.expressions.codegen.BaseProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private InternalRow mutableRow; /* 009 */ /* 010 */ /* 011 */ public SpecificSafeProjection(Object[] references) { /* 012 */ this.references = references; /* 013 */ mutableRow = (InternalRow) references[references.length - 1]; /* 014 */ /* 015 */ } /* 016 */ /* 017 */ public void initialize(int partitionIndex) { /* 018 */ /* 019 */ } /* 020 */ /* 021 */ public java.lang.Object apply(java.lang.Object _i) { /* 022 */ InternalRow i = (InternalRow) _i; /* 023 */ Object[] values_0 = new Object[3]; /* 024 */ /* 025 */ int value_1 = i.getInt(0); /* 026 */ if (false) { /* 027 */ values_0[0] = null; /* 028 */ } else { /* 029 */ values_0[0] = value_1; /* 030 */ } /* 031 */ /* 032 */ int value_2 = i.getInt(1); /* 033 */ if (false) { /* 034 */ values_0[1] = null; /* 035 */ } else { /* 036 */ values_0[1] = value_2; /* 037 */ } /* 038 */ /* 039 */ boolean isNull_3 = i.isNullAt(2); /* 040 */ float value_3 = isNull_3 ? /* 041 */ -1.0f : (i.getFloat(2)); /* 042 */ if (isNull_3) { /* 043 */ values_0[2] = null; /* 044 */ } else { /* 045 */ values_0[2] = value_3; /* 046 */ } /* 047 */ /* 048 */ final org.apache.spark.sql.Row value_0 = new org.apache.spark.sql.catalyst.expressions.GenericRowWithSchema(values_0, ((org.apache.spark.sql.types.StructType) references[0] /* schema */)); /* 049 */ if (false) { /* 050 */ mutableRow.setNullAt(0); /* 051 */ } else { /* 052 */ /* 053 */ mutableRow.update(0, value_0); /* 054 */ } /* 055 */ /* 056 */ return mutableRow; /* 057 */ } /* 058 */ /* 059 */ /* 060 */ } 20/01/23 18:54:01.303 Executor task launch worker for task 0 INFO CodeGenerator: Code generated in 23.777938 ms 20/01/23 18:54:01.328 Executor task launch worker for task 0 INFO PythonRunner: Times: total = 618, boot = 575, init = 43, finish = 0 20/01/23 18:54:01.335 Executor task launch worker for task 3 DEBUG BlockManager: Getting local block broadcast_0 20/01/23 18:54:01.341 Executor task launch worker for task 0 INFO Executor: Finished task 0.0 in stage 0.0 (TID 0). 2092 bytes result sent to driver 20/01/23 18:54:01.342 Executor task launch worker for task 0 DEBUG ExecutorMetricsPoller: removing (0, 0) from stageTCMP 20/01/23 18:54:01.343 Executor task launch worker for task 3 DEBUG BlockManager: Level for block broadcast_0 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:01.345 dispatcher-event-loop-0 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_0.0, runningTasks: 0 20/01/23 18:54:01.354 task-result-getter-0 INFO TaskSetManager: Finished task 0.0 in stage 0.0 (TID 0) in 1767 ms on 192.168.1.225 (executor driver) (1/1) 20/01/23 18:54:01.356 task-result-getter-0 INFO TaskSchedulerImpl: Removed TaskSet 0.0, whose tasks have all completed, from pool 20/01/23 18:54:01.359 Executor task launch worker for task 0 INFO Executor: Finished task 0.0 in stage 0.0 (TID 0). 2137 bytes result sent to driver 20/01/23 18:54:01.360 Executor task launch worker for task 0 DEBUG ExecutorMetricsPoller: stageTCMP: (0, 0) -> 3 20/01/23 18:54:01.360 dag-scheduler-event-loop INFO PythonAccumulatorV2: Connected to AccumulatorServer at host: 127.0.0.1 port: 36719 20/01/23 18:54:01.362 dispatcher-event-loop-3 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_0.0, runningTasks: 3 20/01/23 18:54:01.362 dispatcher-event-loop-3 DEBUG TaskSetManager: No tasks for locality level NO_PREF, so moving to locality level ANY 20/01/23 18:54:01.365 dag-scheduler-event-loop INFO DAGScheduler: ResultStage 0 (isEmpty at ALS.scala:931) finished in 2.060 s 20/01/23 18:54:01.369 dag-scheduler-event-loop DEBUG DAGScheduler: After removal of stage 0, remaining stages = 0 20/01/23 18:54:01.370 dag-scheduler-event-loop INFO DAGScheduler: Job 0 is finished. Cancelling potential speculative or zombie tasks for this job 20/01/23 18:54:01.370 dag-scheduler-event-loop INFO TaskSchedulerImpl: Killing all running tasks in stage 0: Stage finished 20/01/23 18:54:01.370 Executor task launch worker for task 1 INFO PythonRunner: Times: total = 609, boot = 580, init = 29, finish = 0 20/01/23 18:54:01.372 Thread-4 INFO DAGScheduler: Job 0 finished: isEmpty at ALS.scala:931, took 2.156006 s 20/01/23 18:54:01.370 Executor task launch worker for task 3 INFO PythonRunner: Times: total = 612, boot = 591, init = 21, finish = 0 20/01/23 18:54:01.370 Executor task launch worker for task 2 INFO PythonRunner: Times: total = 609, boot = 584, init = 25, finish = 0 20/01/23 18:54:01.377 Executor task launch worker for task 1 INFO Executor: Finished task 1.0 in stage 0.0 (TID 1). 2169 bytes result sent to driver 20/01/23 18:54:01.378 Executor task launch worker for task 1 DEBUG ExecutorMetricsPoller: stageTCMP: (0, 0) -> 2 20/01/23 18:54:01.378 Executor task launch worker for task 3 INFO Executor: Finished task 3.0 in stage 0.0 (TID 3). 2143 bytes result sent to driver 20/01/23 18:54:01.378 Executor task launch worker for task 3 DEBUG ExecutorMetricsPoller: stageTCMP: (0, 0) -> 1 20/01/23 18:54:01.379 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$partitionRatings$1 20/01/23 18:54:01.381 Executor task launch worker for task 2 INFO Executor: Finished task 2.0 in stage 0.0 (TID 2). 2155 bytes result sent to driver 20/01/23 18:54:01.381 dispatcher-event-loop-0 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_0.0, runningTasks: 2 20/01/23 18:54:01.382 Executor task launch worker for task 2 DEBUG ExecutorMetricsPoller: removing (0, 0) from stageTCMP 20/01/23 18:54:01.382 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$partitionRatings$1) is now cleaned +++ 20/01/23 18:54:01.383 dispatcher-event-loop-0 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_0.0, runningTasks: 1 20/01/23 18:54:01.389 task-result-getter-0 INFO TaskSetManager: Finished task 0.0 in stage 0.0 (TID 0) in 1502 ms on 192.168.1.225 (executor driver) (1/4) 20/01/23 18:54:01.395 dag-scheduler-event-loop INFO PythonAccumulatorV2: Connected to AccumulatorServer at host: 127.0.0.1 port: 56015 20/01/23 18:54:01.397 task-result-getter-1 INFO TaskSetManager: Finished task 1.0 in stage 0.0 (TID 1) in 1469 ms on 192.168.1.225 (executor driver) (2/4) 20/01/23 18:54:01.398 dispatcher-event-loop-0 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_0.0, runningTasks: 0 20/01/23 18:54:01.398 task-result-getter-2 INFO TaskSetManager: Finished task 3.0 in stage 0.0 (TID 3) in 1469 ms on 192.168.1.225 (executor driver) (3/4) 20/01/23 18:54:01.398 task-result-getter-3 INFO TaskSetManager: Finished task 2.0 in stage 0.0 (TID 2) in 1470 ms on 192.168.1.225 (executor driver) (4/4) 20/01/23 18:54:01.400 task-result-getter-3 INFO TaskSchedulerImpl: Removed TaskSet 0.0, whose tasks have all completed, from pool 20/01/23 18:54:01.404 dag-scheduler-event-loop INFO DAGScheduler: ResultStage 0 ( at /usr/lib/python3.6/runpy.py:85) finished in 1.709 s 20/01/23 18:54:01.407 dag-scheduler-event-loop DEBUG DAGScheduler: After removal of stage 0, remaining stages = 0 20/01/23 18:54:01.408 dag-scheduler-event-loop INFO DAGScheduler: Job 0 is finished. Cancelling potential speculative or zombie tasks for this job 20/01/23 18:54:01.409 dag-scheduler-event-loop INFO TaskSchedulerImpl: Killing all running tasks in stage 0: Stage finished 20/01/23 18:54:01.417 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$groupByKey$2 20/01/23 18:54:01.417 Thread-4 DEBUG WholeStageCodegenExec: /* 001 */ public Object generate(Object[] references) { /* 002 */ return new GeneratedIteratorForCodegenStage1(references); /* 003 */ } /* 004 */ /* 005 */ // codegenStageId=1 /* 006 */ final class GeneratedIteratorForCodegenStage1 extends org.apache.spark.sql.execution.BufferedRowIterator { /* 007 */ private Object[] references; /* 008 */ private scala.collection.Iterator[] inputs; /* 009 */ private scala.collection.Iterator rdd_input_0; /* 010 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[] rdd_mutableStateArray_0 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[4]; /* 011 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[] rdd_mutableStateArray_1 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[4]; /* 012 */ /* 013 */ public GeneratedIteratorForCodegenStage1(Object[] references) { /* 014 */ this.references = references; /* 015 */ } /* 016 */ /* 017 */ public void init(int index, scala.collection.Iterator[] inputs) { /* 018 */ partitionIndex = index; /* 019 */ this.inputs = inputs; /* 020 */ rdd_input_0 = inputs[0]; /* 021 */ rdd_mutableStateArray_0[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(2, 32); /* 022 */ rdd_mutableStateArray_0[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(rdd_mutableStateArray_0[0], 4); /* 023 */ rdd_mutableStateArray_1[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[1], 4); /* 024 */ rdd_mutableStateArray_1[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[1], 8); /* 025 */ rdd_mutableStateArray_0[2] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(2, 32); /* 026 */ rdd_mutableStateArray_0[3] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(rdd_mutableStateArray_0[2], 4); /* 027 */ rdd_mutableStateArray_1[2] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[3], 4); /* 028 */ rdd_mutableStateArray_1[3] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[3], 8); /* 029 */ /* 030 */ } /* 031 */ /* 032 */ private void project_writeFields_0_1(InternalRow project_tmpInput_0) { /* 033 */ if ((project_tmpInput_0.isNullAt(3))) { /* 034 */ rdd_mutableStateArray_0[3].setNullAt(3); /* 035 */ } else { /* 036 */ // Remember the current cursor so that we can calculate how many bytes are /* 037 */ // written later. /* 038 */ final int project_previousCursor_2 = rdd_mutableStateArray_0[3].cursor(); /* 039 */ /* 040 */ final ArrayData project_tmpInput_2 = (project_tmpInput_0.getArray(3)); /* 041 */ if (project_tmpInput_2 instanceof UnsafeArrayData) { /* 042 */ rdd_mutableStateArray_0[3].write((UnsafeArrayData) project_tmpInput_2); /* 043 */ } else { /* 044 */ final int project_numElements_1 = project_tmpInput_2.numElements(); /* 045 */ rdd_mutableStateArray_1[3].initialize(project_numElements_1); /* 046 */ /* 047 */ for (int project_index_1 = 0; project_index_1 < project_numElements_1; project_index_1++) { /* 048 */ rdd_mutableStateArray_1[3].write(project_index_1, project_tmpInput_2.getDouble(project_index_1)); /* 049 */ } /* 050 */ } /* 051 */ /* 052 */ rdd_mutableStateArray_0[3].setOffsetAndSizeFromPreviousCursor(3, project_previousCursor_2); /* 053 */ } /* 054 */ /* 055 */ } /* 056 */ /* 057 */ private void project_writeFields_0_0(InternalRow project_tmpInput_0) { /* 058 */ rdd_mutableStateArray_0[3].write(0, (project_tmpInput_0.getByte(0))); /* 059 */ /* 060 */ if ((project_tmpInput_0.isNullAt(1))) { /* 061 */ rdd_mutableStateArray_0[3].setNullAt(1); /* 062 */ } else { /* 063 */ rdd_mutableStateArray_0[3].write(1, (project_tmpInput_0.getInt(1))); /* 064 */ } /* 065 */ /* 066 */ if ((project_tmpInput_0.isNullAt(2))) { /* 067 */ rdd_mutableStateArray_0[3].setNullAt(2); /* 068 */ } else { /* 069 */ // Remember the current cursor so that we can calculate how many bytes are /* 070 */ // written later. /* 071 */ final int project_previousCursor_1 = rdd_mutableStateArray_0[3].cursor(); /* 072 */ /* 073 */ final ArrayData project_tmpInput_1 = (project_tmpInput_0.getArray(2)); /* 074 */ if (project_tmpInput_1 instanceof UnsafeArrayData) { /* 075 */ rdd_mutableStateArray_0[3].write((UnsafeArrayData) project_tmpInput_1); /* 076 */ } else { /* 077 */ final int project_numElements_0 = project_tmpInput_1.numElements(); /* 078 */ rdd_mutableStateArray_1[2].initialize(project_numElements_0); /* 079 */ /* 080 */ for (int project_index_0 = 0; project_index_0 < project_numElements_0; project_index_0++) { /* 081 */ rdd_mutableStateArray_1[2].write(project_index_0, project_tmpInput_1.getInt(project_index_0)); /* 082 */ } /* 083 */ } /* 084 */ /* 085 */ rdd_mutableStateArray_0[3].setOffsetAndSizeFromPreviousCursor(2, project_previousCursor_1); /* 086 */ } /* 087 */ /* 088 */ } /* 089 */ /* 090 */ protected void processNext() throws java.io.IOException { /* 091 */ while ( rdd_input_0.hasNext()) { /* 092 */ InternalRow rdd_row_0 = (InternalRow) rdd_input_0.next(); /* 093 */ ((org.apache.spark.sql.execution.metric.SQLMetric) references[0] /* numOutputRows */).add(1); /* 094 */ boolean rdd_isNull_1 = rdd_row_0.isNullAt(1); /* 095 */ double rdd_value_1 = rdd_isNull_1 ? /* 096 */ -1.0 : (rdd_row_0.getDouble(1)); /* 097 */ boolean rdd_isNull_0 = rdd_row_0.isNullAt(0); /* 098 */ InternalRow rdd_value_0 = rdd_isNull_0 ? /* 099 */ null : (rdd_row_0.getStruct(0, 4)); /* 100 */ Object project_arg_0 = rdd_isNull_0 ? null : ((scala.Function1[]) references[1] /* converters */)[0].apply(rdd_value_0); /* 101 */ /* 102 */ InternalRow project_result_0 = null; /* 103 */ try { /* 104 */ project_result_0 = (InternalRow)((scala.Function1[]) references[1] /* converters */)[1].apply(((scala.Function1) references[3] /* udf */).apply(project_arg_0)); /* 105 */ } catch (Exception e) { /* 106 */ throw new org.apache.spark.SparkException(((java.lang.String) references[2] /* errMsg */), e); /* 107 */ } /* 108 */ /* 109 */ boolean project_isNull_1 = project_result_0 == null; /* 110 */ InternalRow project_value_1 = null; /* 111 */ if (!project_isNull_1) { /* 112 */ project_value_1 = project_result_0; /* 113 */ } /* 114 */ rdd_mutableStateArray_0[2].reset(); /* 115 */ /* 116 */ rdd_mutableStateArray_0[2].zeroOutNullBytes(); /* 117 */ /* 118 */ if (rdd_isNull_1) { /* 119 */ rdd_mutableStateArray_0[2].setNullAt(0); /* 120 */ } else { /* 121 */ rdd_mutableStateArray_0[2].write(0, rdd_value_1); /* 122 */ } /* 123 */ /* 124 */ if (project_isNull_1) { /* 125 */ rdd_mutableStateArray_0[2].setNullAt(1); /* 126 */ } else { /* 127 */ final InternalRow project_tmpInput_0 = project_value_1; /* 128 */ if (project_tmpInput_0 instanceof UnsafeRow) { /* 129 */ rdd_mutableStateArray_0[2].write(1, (UnsafeRow) project_tmpInput_0); /* 130 */ } else { /* 131 */ // Remember the current cursor so that we can calculate how many bytes are /* 132 */ // written later. /* 133 */ final int project_previousCursor_0 = rdd_mutableStateArray_0[2].cursor(); /* 134 */ /* 135 */ rdd_mutableStateArray_0[3].resetRowWriter(); /* 136 */ project_writeFields_0_0(project_tmpInput_0); /* 137 */ project_writeFields_0_1(project_tmpInput_0); /* 138 */ /* 139 */ rdd_mutableStateArray_0[2].setOffsetAndSizeFromPreviousCursor(1, project_previousCursor_0); /* 140 */ } /* 141 */ } /* 142 */ append((rdd_mutableStateArray_0[2].getRow())); /* 143 */ if (shouldStop()) return; /* 144 */ } /* 145 */ } /* 146 */ /* 147 */ } 20/01/23 18:54:01.417 Thread-4 INFO DAGScheduler: Job 0 finished: at /usr/lib/python3.6/runpy.py:85, took 1.818490 s 20/01/23 18:54:01.422 Thread-4 DEBUG CodeGenerator: /* 001 */ public Object generate(Object[] references) { /* 002 */ return new GeneratedIteratorForCodegenStage1(references); /* 003 */ } /* 004 */ /* 005 */ // codegenStageId=1 /* 006 */ final class GeneratedIteratorForCodegenStage1 extends org.apache.spark.sql.execution.BufferedRowIterator { /* 007 */ private Object[] references; /* 008 */ private scala.collection.Iterator[] inputs; /* 009 */ private scala.collection.Iterator rdd_input_0; /* 010 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[] rdd_mutableStateArray_0 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[4]; /* 011 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[] rdd_mutableStateArray_1 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[4]; /* 012 */ /* 013 */ public GeneratedIteratorForCodegenStage1(Object[] references) { /* 014 */ this.references = references; /* 015 */ } /* 016 */ /* 017 */ public void init(int index, scala.collection.Iterator[] inputs) { /* 018 */ partitionIndex = index; /* 019 */ this.inputs = inputs; /* 020 */ rdd_input_0 = inputs[0]; /* 021 */ rdd_mutableStateArray_0[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(2, 32); /* 022 */ rdd_mutableStateArray_0[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(rdd_mutableStateArray_0[0], 4); /* 023 */ rdd_mutableStateArray_1[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[1], 4); /* 024 */ rdd_mutableStateArray_1[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[1], 8); /* 025 */ rdd_mutableStateArray_0[2] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(2, 32); /* 026 */ rdd_mutableStateArray_0[3] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(rdd_mutableStateArray_0[2], 4); /* 027 */ rdd_mutableStateArray_1[2] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[3], 4); /* 028 */ rdd_mutableStateArray_1[3] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[3], 8); /* 029 */ /* 030 */ } /* 031 */ /* 032 */ private void project_writeFields_0_1(InternalRow project_tmpInput_0) { /* 033 */ if ((project_tmpInput_0.isNullAt(3))) { /* 034 */ rdd_mutableStateArray_0[3].setNullAt(3); /* 035 */ } else { /* 036 */ // Remember the current cursor so that we can calculate how many bytes are /* 037 */ // written later. /* 038 */ final int project_previousCursor_2 = rdd_mutableStateArray_0[3].cursor(); /* 039 */ /* 040 */ final ArrayData project_tmpInput_2 = (project_tmpInput_0.getArray(3)); /* 041 */ if (project_tmpInput_2 instanceof UnsafeArrayData) { /* 042 */ rdd_mutableStateArray_0[3].write((UnsafeArrayData) project_tmpInput_2); /* 043 */ } else { /* 044 */ final int project_numElements_1 = project_tmpInput_2.numElements(); /* 045 */ rdd_mutableStateArray_1[3].initialize(project_numElements_1); /* 046 */ /* 047 */ for (int project_index_1 = 0; project_index_1 < project_numElements_1; project_index_1++) { /* 048 */ rdd_mutableStateArray_1[3].write(project_index_1, project_tmpInput_2.getDouble(project_index_1)); /* 049 */ } /* 050 */ } /* 051 */ /* 052 */ rdd_mutableStateArray_0[3].setOffsetAndSizeFromPreviousCursor(3, project_previousCursor_2); /* 053 */ } /* 054 */ /* 055 */ } /* 056 */ /* 057 */ private void project_writeFields_0_0(InternalRow project_tmpInput_0) { /* 058 */ rdd_mutableStateArray_0[3].write(0, (project_tmpInput_0.getByte(0))); /* 059 */ /* 060 */ if ((project_tmpInput_0.isNullAt(1))) { /* 061 */ rdd_mutableStateArray_0[3].setNullAt(1); /* 062 */ } else { /* 063 */ rdd_mutableStateArray_0[3].write(1, (project_tmpInput_0.getInt(1))); /* 064 */ } /* 065 */ /* 066 */ if ((project_tmpInput_0.isNullAt(2))) { /* 067 */ rdd_mutableStateArray_0[3].setNullAt(2); /* 068 */ } else { /* 069 */ // Remember the current cursor so that we can calculate how many bytes are /* 070 */ // written later. /* 071 */ final int project_previousCursor_1 = rdd_mutableStateArray_0[3].cursor(); /* 072 */ /* 073 */ final ArrayData project_tmpInput_1 = (project_tmpInput_0.getArray(2)); /* 074 */ if (project_tmpInput_1 instanceof UnsafeArrayData) { /* 075 */ rdd_mutableStateArray_0[3].write((UnsafeArrayData) project_tmpInput_1); /* 076 */ } else { /* 077 */ final int project_numElements_0 = project_tmpInput_1.numElements(); /* 078 */ rdd_mutableStateArray_1[2].initialize(project_numElements_0); /* 079 */ /* 080 */ for (int project_index_0 = 0; project_index_0 < project_numElements_0; project_index_0++) { /* 081 */ rdd_mutableStateArray_1[2].write(project_index_0, project_tmpInput_1.getInt(project_index_0)); /* 082 */ } /* 083 */ } /* 084 */ /* 085 */ rdd_mutableStateArray_0[3].setOffsetAndSizeFromPreviousCursor(2, project_previousCursor_1); /* 086 */ } /* 087 */ /* 088 */ } /* 089 */ /* 090 */ protected void processNext() throws java.io.IOException { /* 091 */ while ( rdd_input_0.hasNext()) { /* 092 */ InternalRow rdd_row_0 = (InternalRow) rdd_input_0.next(); /* 093 */ ((org.apache.spark.sql.execution.metric.SQLMetric) references[0] /* numOutputRows */).add(1); /* 094 */ boolean rdd_isNull_1 = rdd_row_0.isNullAt(1); /* 095 */ double rdd_value_1 = rdd_isNull_1 ? /* 096 */ -1.0 : (rdd_row_0.getDouble(1)); /* 097 */ boolean rdd_isNull_0 = rdd_row_0.isNullAt(0); /* 098 */ InternalRow rdd_value_0 = rdd_isNull_0 ? /* 099 */ null : (rdd_row_0.getStruct(0, 4)); /* 100 */ Object project_arg_0 = rdd_isNull_0 ? null : ((scala.Function1[]) references[1] /* converters */)[0].apply(rdd_value_0); /* 101 */ /* 102 */ InternalRow project_result_0 = null; /* 103 */ try { /* 104 */ project_result_0 = (InternalRow)((scala.Function1[]) references[1] /* converters */)[1].apply(((scala.Function1) references[3] /* udf */).apply(project_arg_0)); /* 105 */ } catch (Exception e) { /* 106 */ throw new org.apache.spark.SparkException(((java.lang.String) references[2] /* errMsg */), e); /* 107 */ } /* 108 */ /* 109 */ boolean project_isNull_1 = project_result_0 == null; /* 110 */ InternalRow project_value_1 = null; /* 111 */ if (!project_isNull_1) { /* 112 */ project_value_1 = project_result_0; /* 113 */ } /* 114 */ rdd_mutableStateArray_0[2].reset(); /* 115 */ /* 116 */ rdd_mutableStateArray_0[2].zeroOutNullBytes(); /* 117 */ /* 118 */ if (rdd_isNull_1) { /* 119 */ rdd_mutableStateArray_0[2].setNullAt(0); /* 120 */ } else { /* 121 */ rdd_mutableStateArray_0[2].write(0, rdd_value_1); /* 122 */ } /* 123 */ /* 124 */ if (project_isNull_1) { /* 125 */ rdd_mutableStateArray_0[2].setNullAt(1); /* 126 */ } else { /* 127 */ final InternalRow project_tmpInput_0 = project_value_1; /* 128 */ if (project_tmpInput_0 instanceof UnsafeRow) { /* 129 */ rdd_mutableStateArray_0[2].write(1, (UnsafeRow) project_tmpInput_0); /* 130 */ } else { /* 131 */ // Remember the current cursor so that we can calculate how many bytes are /* 132 */ // written later. /* 133 */ final int project_previousCursor_0 = rdd_mutableStateArray_0[2].cursor(); /* 134 */ /* 135 */ rdd_mutableStateArray_0[3].resetRowWriter(); /* 136 */ project_writeFields_0_0(project_tmpInput_0); /* 137 */ project_writeFields_0_1(project_tmpInput_0); /* 138 */ /* 139 */ rdd_mutableStateArray_0[2].setOffsetAndSizeFromPreviousCursor(1, project_previousCursor_0); /* 140 */ } /* 141 */ } /* 142 */ append((rdd_mutableStateArray_0[2].getRow())); /* 143 */ if (shouldStop()) return; /* 144 */ } /* 145 */ } /* 146 */ /* 147 */ } 20/01/23 18:54:01.423 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$groupByKey$2) is now cleaned +++ 20/01/23 18:54:01.437 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$groupByKey$3 20/01/23 18:54:01.442 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$groupByKey$3) is now cleaned +++ 20/01/23 18:54:01.443 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$groupByKey$4 20/01/23 18:54:01.451 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$groupByKey$4) is now cleaned +++ 20/01/23 18:54:01.457 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$partitionRatings$7 20/01/23 18:54:01.459 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$partitionRatings$7) is now cleaned +++ 20/01/23 18:54:01.471 Thread-4 INFO CodeGenerator: Code generated in 52.84374 ms 20/01/23 18:54:01.471 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$doExecute$4$adapted 20/01/23 18:54:01.473 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$doExecute$4$adapted) is now cleaned +++ 20/01/23 18:54:01.479 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$makeBlocks$1 20/01/23 18:54:01.481 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$rdd$1 20/01/23 18:54:01.482 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$makeBlocks$1) is now cleaned +++ 20/01/23 18:54:01.485 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$groupByKey$2 20/01/23 18:54:01.486 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$rdd$1) is now cleaned +++ 20/01/23 18:54:01.489 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$groupByKey$2) is now cleaned +++ 20/01/23 18:54:01.504 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$groupByKey$3 20/01/23 18:54:01.505 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$computeClusterStats$5 20/01/23 18:54:01.506 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$computeClusterStats$5) is now cleaned +++ 20/01/23 18:54:01.507 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$groupByKey$3) is now cleaned +++ 20/01/23 18:54:01.508 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$groupByKey$4 20/01/23 18:54:01.511 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$groupByKey$4) is now cleaned +++ 20/01/23 18:54:01.513 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$makeBlocks$5 20/01/23 18:54:01.515 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$makeBlocks$5) is now cleaned +++ 20/01/23 18:54:01.517 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$makeBlocks$7 20/01/23 18:54:01.520 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$makeBlocks$7) is now cleaned +++ 20/01/23 18:54:01.522 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$computeClusterStats$6 20/01/23 18:54:01.522 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$count$1$adapted 20/01/23 18:54:01.522 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$computeClusterStats$6) is now cleaned +++ 20/01/23 18:54:01.526 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$aggregateByKey$3 20/01/23 18:54:01.529 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$count$1$adapted) is now cleaned +++ 20/01/23 18:54:01.529 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$aggregateByKey$3) is now cleaned +++ 20/01/23 18:54:01.530 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$runJob$5 20/01/23 18:54:01.535 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$runJob$5) is now cleaned +++ 20/01/23 18:54:01.536 Thread-4 INFO SparkContext: Starting job: count at ALS.scala:944 20/01/23 18:54:01.537 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$computeClusterStats$6 20/01/23 18:54:01.538 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$computeClusterStats$6) is now cleaned +++ 20/01/23 18:54:01.539 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$computeClusterStats$7 20/01/23 18:54:01.539 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$computeClusterStats$7) is now cleaned +++ 20/01/23 18:54:01.543 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$collect$2 20/01/23 18:54:01.547 dag-scheduler-event-loop INFO DAGScheduler: Registering RDD 14 (mapPartitions at ALS.scala:1342) as input to shuffle 1 20/01/23 18:54:01.547 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$collect$2) is now cleaned +++ 20/01/23 18:54:01.553 dag-scheduler-event-loop INFO DAGScheduler: Registering RDD 17 (map at ALS.scala:1585) as input to shuffle 0 20/01/23 18:54:01.555 dag-scheduler-event-loop INFO DAGScheduler: Got job 1 (count at ALS.scala:944) with 10 output partitions 20/01/23 18:54:01.555 dag-scheduler-event-loop INFO DAGScheduler: Final stage: ResultStage 3 (count at ALS.scala:944) 20/01/23 18:54:01.555 dag-scheduler-event-loop INFO DAGScheduler: Parents of final stage: List(ShuffleMapStage 2) 20/01/23 18:54:01.556 Thread-4 DEBUG CountVectorizerModel: Input schema: {"type":"struct","fields":[{"name":"id","type":"long","nullable":true,"metadata":{}},{"name":"words","type":{"type":"array","elementType":"string","containsNull":true},"nullable":true,"metadata":{}},{"name":"expected","type":{"type":"udt","class":"org.apache.spark.ml.linalg.VectorUDT","pyClass":"pyspark.ml.linalg.VectorUDT","sqlType":{"type":"struct","fields":[{"name":"type","type":"byte","nullable":false,"metadata":{}},{"name":"size","type":"integer","nullable":true,"metadata":{}},{"name":"indices","type":{"type":"array","elementType":"integer","containsNull":false},"nullable":true,"metadata":{}},{"name":"values","type":{"type":"array","elementType":"double","containsNull":false},"nullable":true,"metadata":{}}]}},"nullable":true,"metadata":{}}]} 20/01/23 18:54:01.558 Thread-4 DEBUG CountVectorizerModel: Expected output schema: {"type":"struct","fields":[{"name":"id","type":"long","nullable":true,"metadata":{}},{"name":"words","type":{"type":"array","elementType":"string","containsNull":true},"nullable":true,"metadata":{}},{"name":"expected","type":{"type":"udt","class":"org.apache.spark.ml.linalg.VectorUDT","pyClass":"pyspark.ml.linalg.VectorUDT","sqlType":{"type":"struct","fields":[{"name":"type","type":"byte","nullable":false,"metadata":{}},{"name":"size","type":"integer","nullable":true,"metadata":{}},{"name":"indices","type":{"type":"array","elementType":"integer","containsNull":false},"nullable":true,"metadata":{}},{"name":"values","type":{"type":"array","elementType":"double","containsNull":false},"nullable":true,"metadata":{}}]}},"nullable":true,"metadata":{}},{"name":"CountVectorizerModel_f69b1202b289__output","type":{"type":"udt","class":"org.apache.spark.ml.linalg.VectorUDT","pyClass":"pyspark.ml.linalg.VectorUDT","sqlType":{"type":"struct","fields":[{"name":"type","type":"byte","nullable":false,"metadata":{}},{"name":"size","type":"integer","nullable":true,"metadata":{}},{"name":"indices","type":{"type":"array","elementType":"integer","containsNull":false},"nullable":true,"metadata":{}},{"name":"values","type":{"type":"array","elementType":"double","containsNull":false},"nullable":true,"metadata":{}}]}},"nullable":false,"metadata":{"ml_attr":{"attrs":{},"num_attrs":3}}}]} 20/01/23 18:54:01.560 Thread-4 INFO MemoryStore: Block broadcast_2 stored as values in memory (estimated size 256.0 B, free 366.3 MiB) 20/01/23 18:54:01.561 Thread-4 DEBUG BlockManager: Put block broadcast_2 locally took 1 ms 20/01/23 18:54:01.561 Thread-4 DEBUG BlockManager: Putting block broadcast_2 without replication took 1 ms 20/01/23 18:54:01.561 dag-scheduler-event-loop INFO DAGScheduler: Missing parents: List(ShuffleMapStage 2) 20/01/23 18:54:01.562 dag-scheduler-event-loop DEBUG DAGScheduler: submitStage(ResultStage 3 (name=count at ALS.scala:944;jobs=1)) 20/01/23 18:54:01.563 dag-scheduler-event-loop DEBUG DAGScheduler: missing: List(ShuffleMapStage 2) 20/01/23 18:54:01.565 Thread-4 INFO MemoryStore: Block broadcast_2_piece0 stored as bytes in memory (estimated size 264.0 B, free 366.3 MiB) 20/01/23 18:54:01.566 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added broadcast_2_piece0 in memory on 192.168.1.225:41233 (size: 264.0 B, free: 366.3 MiB) 20/01/23 18:54:01.567 Thread-4 DEBUG BlockManagerMaster: Updated info of block broadcast_2_piece0 20/01/23 18:54:01.567 Thread-4 DEBUG BlockManager: Told master about block broadcast_2_piece0 20/01/23 18:54:01.567 Thread-4 DEBUG BlockManager: Put block broadcast_2_piece0 locally took 2 ms 20/01/23 18:54:01.567 Thread-4 DEBUG BlockManager: Putting block broadcast_2_piece0 without replication took 2 ms 20/01/23 18:54:01.567 Thread-4 INFO SparkContext: Created broadcast 2 from broadcast at CountVectorizer.scala:306 20/01/23 18:54:01.572 dag-scheduler-event-loop DEBUG DAGScheduler: submitStage(ShuffleMapStage 2 (name=map at ALS.scala:1585;jobs=1)) 20/01/23 18:54:01.574 dag-scheduler-event-loop DEBUG DAGScheduler: missing: List(ShuffleMapStage 1) 20/01/23 18:54:01.574 dag-scheduler-event-loop DEBUG DAGScheduler: submitStage(ShuffleMapStage 1 (name=mapPartitions at ALS.scala:1342;jobs=1)) 20/01/23 18:54:01.575 dag-scheduler-event-loop DEBUG DAGScheduler: missing: List() 20/01/23 18:54:01.575 dag-scheduler-event-loop INFO DAGScheduler: Submitting ShuffleMapStage 1 (MapPartitionsRDD[14] at mapPartitions at ALS.scala:1342), which has no missing parents 20/01/23 18:54:01.575 dag-scheduler-event-loop DEBUG DAGScheduler: submitMissingTasks(ShuffleMapStage 1) 20/01/23 18:54:01.575 Thread-4 DEBUG Analyzer$ResolveReferences: Resolving 'words to words#1 20/01/23 18:54:01.584 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$runJob$5 20/01/23 18:54:01.587 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$runJob$5) is now cleaned +++ 20/01/23 18:54:01.588 Thread-4 INFO SparkContext: Starting job: collectAsMap at ClusteringEvaluator.scala:585 20/01/23 18:54:01.594 dag-scheduler-event-loop DEBUG SortShuffleManager: Can't use serialized shuffle for shuffle 0 because the serializer, org.apache.spark.serializer.JavaSerializer, does not support object relocation 20/01/23 18:54:01.599 dag-scheduler-event-loop INFO DAGScheduler: Registering RDD 11 (map at ClusteringEvaluator.scala:571) as input to shuffle 0 20/01/23 18:54:01.602 dag-scheduler-event-loop INFO DAGScheduler: Got job 1 (collectAsMap at ClusteringEvaluator.scala:585) with 4 output partitions 20/01/23 18:54:01.602 dag-scheduler-event-loop INFO DAGScheduler: Final stage: ResultStage 2 (collectAsMap at ClusteringEvaluator.scala:585) 20/01/23 18:54:01.602 dag-scheduler-event-loop INFO DAGScheduler: Parents of final stage: List(ShuffleMapStage 1) 20/01/23 18:54:01.603 dag-scheduler-event-loop INFO DAGScheduler: Missing parents: List(ShuffleMapStage 1) 20/01/23 18:54:01.604 dag-scheduler-event-loop DEBUG DAGScheduler: submitStage(ResultStage 2 (name=collectAsMap at ClusteringEvaluator.scala:585;jobs=1)) 20/01/23 18:54:01.609 dag-scheduler-event-loop DEBUG DAGScheduler: missing: List(ShuffleMapStage 1) 20/01/23 18:54:01.610 dag-scheduler-event-loop DEBUG DAGScheduler: submitStage(ShuffleMapStage 1 (name=map at ClusteringEvaluator.scala:571;jobs=1)) 20/01/23 18:54:01.610 dag-scheduler-event-loop DEBUG DAGScheduler: missing: List() 20/01/23 18:54:01.611 dag-scheduler-event-loop INFO DAGScheduler: Submitting ShuffleMapStage 1 (MapPartitionsRDD[11] at map at ClusteringEvaluator.scala:571), which has no missing parents 20/01/23 18:54:01.611 dag-scheduler-event-loop DEBUG DAGScheduler: submitMissingTasks(ShuffleMapStage 1) 20/01/23 18:54:01.623 Thread-4 DEBUG Analyzer$ResolveReferences: Resolving 'CountVectorizerModel_f69b1202b289__output to CountVectorizerModel_f69b1202b289__output#15 20/01/23 18:54:01.626 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_1 stored as values in memory (estimated size 27.3 KiB, free 366.2 MiB) 20/01/23 18:54:01.626 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_1 locally took 1 ms 20/01/23 18:54:01.626 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_1 without replication took 1 ms 20/01/23 18:54:01.628 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_1_piece0 stored as bytes in memory (estimated size 13.2 KiB, free 366.2 MiB) 20/01/23 18:54:01.629 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added broadcast_1_piece0 in memory on 192.168.1.225:35863 (size: 13.2 KiB, free: 366.3 MiB) 20/01/23 18:54:01.630 dag-scheduler-event-loop DEBUG BlockManagerMaster: Updated info of block broadcast_1_piece0 20/01/23 18:54:01.630 dag-scheduler-event-loop DEBUG BlockManager: Told master about block broadcast_1_piece0 20/01/23 18:54:01.630 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_1_piece0 locally took 2 ms 20/01/23 18:54:01.630 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_1_piece0 without replication took 2 ms 20/01/23 18:54:01.630 dag-scheduler-event-loop INFO SparkContext: Created broadcast 1 from broadcast at DAGScheduler.scala:1200 20/01/23 18:54:01.633 dag-scheduler-event-loop INFO DAGScheduler: Submitting 4 missing tasks from ShuffleMapStage 1 (MapPartitionsRDD[14] at mapPartitions at ALS.scala:1342) (first 15 tasks are for partitions Vector(0, 1, 2, 3)) 20/01/23 18:54:01.633 dag-scheduler-event-loop INFO TaskSchedulerImpl: Adding task set 1.0 with 4 tasks 20/01/23 18:54:01.634 dag-scheduler-event-loop DEBUG TaskSetManager: Epoch for TaskSet 1.0: 0 20/01/23 18:54:01.634 dag-scheduler-event-loop DEBUG TaskSetManager: Adding pending tasks took 0 ms 20/01/23 18:54:01.634 dag-scheduler-event-loop DEBUG TaskSetManager: Valid locality levels for TaskSet 1.0: NO_PREF, ANY 20/01/23 18:54:01.635 dispatcher-event-loop-1 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_1.0, runningTasks: 0 20/01/23 18:54:01.637 dispatcher-event-loop-1 INFO TaskSetManager: Starting task 0.0 in stage 1.0 (TID 1, 192.168.1.225, executor driver, partition 0, PROCESS_LOCAL, 7362 bytes) 20/01/23 18:54:01.637 dispatcher-event-loop-1 INFO TaskSetManager: Starting task 1.0 in stage 1.0 (TID 2, 192.168.1.225, executor driver, partition 1, PROCESS_LOCAL, 7402 bytes) 20/01/23 18:54:01.638 dispatcher-event-loop-1 INFO TaskSetManager: Starting task 2.0 in stage 1.0 (TID 3, 192.168.1.225, executor driver, partition 2, PROCESS_LOCAL, 7362 bytes) 20/01/23 18:54:01.638 dispatcher-event-loop-1 INFO TaskSetManager: Starting task 3.0 in stage 1.0 (TID 4, 192.168.1.225, executor driver, partition 3, PROCESS_LOCAL, 7402 bytes) 20/01/23 18:54:01.638 Executor task launch worker for task 1 INFO Executor: Running task 0.0 in stage 1.0 (TID 1) 20/01/23 18:54:01.639 Executor task launch worker for task 1 DEBUG ExecutorMetricsPoller: stageTCMP: (1, 0) -> 1 20/01/23 18:54:01.641 Executor task launch worker for task 1 DEBUG BlockManager: Getting local block broadcast_1 20/01/23 18:54:01.641 Executor task launch worker for task 1 DEBUG BlockManager: Level for block broadcast_1 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:01.661 Executor task launch worker for task 2 INFO Executor: Running task 1.0 in stage 1.0 (TID 2) 20/01/23 18:54:01.695 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_1 stored as values in memory (estimated size 25.1 KiB, free 366.3 MiB) 20/01/23 18:54:01.695 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_1 locally took 1 ms 20/01/23 18:54:01.696 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_1 without replication took 2 ms 20/01/23 18:54:01.697 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_1_piece0 stored as bytes in memory (estimated size 11.6 KiB, free 366.2 MiB) 20/01/23 18:54:01.698 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added broadcast_1_piece0 in memory on 192.168.1.225:45461 (size: 11.6 KiB, free: 366.3 MiB) 20/01/23 18:54:01.698 dag-scheduler-event-loop DEBUG BlockManagerMaster: Updated info of block broadcast_1_piece0 20/01/23 18:54:01.698 dag-scheduler-event-loop DEBUG BlockManager: Told master about block broadcast_1_piece0 20/01/23 18:54:01.699 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_1_piece0 locally took 2 ms 20/01/23 18:54:01.699 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_1_piece0 without replication took 2 ms 20/01/23 18:54:01.699 dag-scheduler-event-loop INFO SparkContext: Created broadcast 1 from broadcast at DAGScheduler.scala:1200 20/01/23 18:54:01.705 Executor task launch worker for task 3 INFO Executor: Running task 2.0 in stage 1.0 (TID 3) 20/01/23 18:54:01.706 Executor task launch worker for task 3 DEBUG ExecutorMetricsPoller: stageTCMP: (1, 0) -> 3 20/01/23 18:54:01.696 Executor task launch worker for task 2 DEBUG ExecutorMetricsPoller: stageTCMP: (1, 0) -> 2 20/01/23 18:54:01.713 dag-scheduler-event-loop INFO DAGScheduler: Submitting 4 missing tasks from ShuffleMapStage 1 (MapPartitionsRDD[11] at map at ClusteringEvaluator.scala:571) (first 15 tasks are for partitions Vector(0, 1, 2, 3)) 20/01/23 18:54:01.713 dag-scheduler-event-loop INFO TaskSchedulerImpl: Adding task set 1.0 with 4 tasks 20/01/23 18:54:01.713 dag-scheduler-event-loop DEBUG TaskSetManager: Epoch for TaskSet 1.0: 0 20/01/23 18:54:01.714 dag-scheduler-event-loop DEBUG TaskSetManager: Adding pending tasks took 0 ms 20/01/23 18:54:01.714 dag-scheduler-event-loop DEBUG TaskSetManager: Valid locality levels for TaskSet 1.0: NO_PREF, ANY 20/01/23 18:54:01.708 Executor task launch worker for task 4 INFO Executor: Running task 3.0 in stage 1.0 (TID 4) 20/01/23 18:54:01.714 dispatcher-event-loop-1 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_1.0, runningTasks: 0 20/01/23 18:54:01.715 Executor task launch worker for task 4 DEBUG ExecutorMetricsPoller: stageTCMP: (1, 0) -> 4 20/01/23 18:54:01.716 dispatcher-event-loop-1 INFO TaskSetManager: Starting task 0.0 in stage 1.0 (TID 1, 192.168.1.225, executor driver, partition 0, PROCESS_LOCAL, 7387 bytes) 20/01/23 18:54:01.717 dispatcher-event-loop-1 INFO TaskSetManager: Starting task 1.0 in stage 1.0 (TID 2, 192.168.1.225, executor driver, partition 1, PROCESS_LOCAL, 7452 bytes) 20/01/23 18:54:01.717 dispatcher-event-loop-1 INFO TaskSetManager: Starting task 2.0 in stage 1.0 (TID 3, 192.168.1.225, executor driver, partition 2, PROCESS_LOCAL, 7387 bytes) 20/01/23 18:54:01.717 dispatcher-event-loop-1 INFO TaskSetManager: Starting task 3.0 in stage 1.0 (TID 4, 192.168.1.225, executor driver, partition 3, PROCESS_LOCAL, 7452 bytes) 20/01/23 18:54:01.718 Executor task launch worker for task 1 INFO Executor: Running task 0.0 in stage 1.0 (TID 1) 20/01/23 18:54:01.719 Executor task launch worker for task 1 DEBUG ExecutorMetricsPoller: stageTCMP: (1, 0) -> 1 20/01/23 18:54:01.721 Executor task launch worker for task 1 DEBUG BlockManager: Getting local block broadcast_1 20/01/23 18:54:01.721 Executor task launch worker for task 1 DEBUG BlockManager: Level for block broadcast_1 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:01.742 Thread-4 DEBUG WholeStageCodegenExec: /* 001 */ public Object generate(Object[] references) { /* 002 */ return new GeneratedIteratorForCodegenStage1(references); /* 003 */ } /* 004 */ /* 005 */ // codegenStageId=1 /* 006 */ final class GeneratedIteratorForCodegenStage1 extends org.apache.spark.sql.execution.BufferedRowIterator { /* 007 */ private Object[] references; /* 008 */ private scala.collection.Iterator[] inputs; /* 009 */ private scala.collection.Iterator rdd_input_0; /* 010 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[] rdd_mutableStateArray_0 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[4]; /* 011 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[] rdd_mutableStateArray_1 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[5]; /* 012 */ /* 013 */ public GeneratedIteratorForCodegenStage1(Object[] references) { /* 014 */ this.references = references; /* 015 */ } /* 016 */ /* 017 */ public void init(int index, scala.collection.Iterator[] inputs) { /* 018 */ partitionIndex = index; /* 019 */ this.inputs = inputs; /* 020 */ wholestagecodegen_init_0_0(); /* 021 */ wholestagecodegen_init_0_1(); /* 022 */ /* 023 */ } /* 024 */ /* 025 */ private void project_writeFields_0_1(InternalRow project_tmpInput_0) { /* 026 */ if ((project_tmpInput_0.isNullAt(3))) { /* 027 */ rdd_mutableStateArray_0[3].setNullAt(3); /* 028 */ } else { /* 029 */ // Remember the current cursor so that we can calculate how many bytes are /* 030 */ // written later. /* 031 */ final int project_previousCursor_2 = rdd_mutableStateArray_0[3].cursor(); /* 032 */ /* 033 */ final ArrayData project_tmpInput_2 = (project_tmpInput_0.getArray(3)); /* 034 */ if (project_tmpInput_2 instanceof UnsafeArrayData) { /* 035 */ rdd_mutableStateArray_0[3].write((UnsafeArrayData) project_tmpInput_2); /* 036 */ } else { /* 037 */ final int project_numElements_1 = project_tmpInput_2.numElements(); /* 038 */ rdd_mutableStateArray_1[4].initialize(project_numElements_1); /* 039 */ /* 040 */ for (int project_index_1 = 0; project_index_1 < project_numElements_1; project_index_1++) { /* 041 */ rdd_mutableStateArray_1[4].write(project_index_1, project_tmpInput_2.getDouble(project_index_1)); /* 042 */ } /* 043 */ } /* 044 */ /* 045 */ rdd_mutableStateArray_0[3].setOffsetAndSizeFromPreviousCursor(3, project_previousCursor_2); /* 046 */ } /* 047 */ /* 048 */ } /* 049 */ /* 050 */ private void wholestagecodegen_init_0_1() { /* 051 */ rdd_mutableStateArray_1[4] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[3], 8); /* 052 */ /* 053 */ } /* 054 */ /* 055 */ private void project_writeFields_0_0(InternalRow project_tmpInput_0) { /* 056 */ rdd_mutableStateArray_0[3].write(0, (project_tmpInput_0.getByte(0))); /* 057 */ /* 058 */ if ((project_tmpInput_0.isNullAt(1))) { /* 059 */ rdd_mutableStateArray_0[3].setNullAt(1); /* 060 */ } else { /* 061 */ rdd_mutableStateArray_0[3].write(1, (project_tmpInput_0.getInt(1))); /* 062 */ } /* 063 */ /* 064 */ if ((project_tmpInput_0.isNullAt(2))) { /* 065 */ rdd_mutableStateArray_0[3].setNullAt(2); /* 066 */ } else { /* 067 */ // Remember the current cursor so that we can calculate how many bytes are /* 068 */ // written later. /* 069 */ final int project_previousCursor_1 = rdd_mutableStateArray_0[3].cursor(); /* 070 */ /* 071 */ final ArrayData project_tmpInput_1 = (project_tmpInput_0.getArray(2)); /* 072 */ if (project_tmpInput_1 instanceof UnsafeArrayData) { /* 073 */ rdd_mutableStateArray_0[3].write((UnsafeArrayData) project_tmpInput_1); /* 074 */ } else { /* 075 */ final int project_numElements_0 = project_tmpInput_1.numElements(); /* 076 */ rdd_mutableStateArray_1[3].initialize(project_numElements_0); /* 077 */ /* 078 */ for (int project_index_0 = 0; project_index_0 < project_numElements_0; project_index_0++) { /* 079 */ rdd_mutableStateArray_1[3].write(project_index_0, project_tmpInput_1.getInt(project_index_0)); /* 080 */ } /* 081 */ } /* 082 */ /* 083 */ rdd_mutableStateArray_0[3].setOffsetAndSizeFromPreviousCursor(2, project_previousCursor_1); /* 084 */ } /* 085 */ /* 086 */ } /* 087 */ /* 088 */ protected void processNext() throws java.io.IOException { /* 089 */ while ( rdd_input_0.hasNext()) { /* 090 */ InternalRow rdd_row_0 = (InternalRow) rdd_input_0.next(); /* 091 */ ((org.apache.spark.sql.execution.metric.SQLMetric) references[0] /* numOutputRows */).add(1); /* 092 */ boolean rdd_isNull_1 = rdd_row_0.isNullAt(1); /* 093 */ ArrayData rdd_value_1 = rdd_isNull_1 ? /* 094 */ null : (rdd_row_0.getArray(1)); /* 095 */ Object project_arg_0 = rdd_isNull_1 ? null : ((scala.Function1[]) references[1] /* converters */)[0].apply(rdd_value_1); /* 096 */ /* 097 */ InternalRow project_result_0 = null; /* 098 */ try { /* 099 */ project_result_0 = (InternalRow)((scala.Function1[]) references[1] /* converters */)[1].apply(((scala.Function1) references[3] /* udf */).apply(project_arg_0)); /* 100 */ } catch (Exception e) { /* 101 */ throw new org.apache.spark.SparkException(((java.lang.String) references[2] /* errMsg */), e); /* 102 */ } /* 103 */ /* 104 */ boolean project_isNull_0 = project_result_0 == null; /* 105 */ InternalRow project_value_0 = null; /* 106 */ if (!project_isNull_0) { /* 107 */ project_value_0 = project_result_0; /* 108 */ } /* 109 */ rdd_mutableStateArray_0[2].reset(); /* 110 */ /* 111 */ rdd_mutableStateArray_0[2].zeroOutNullBytes(); /* 112 */ /* 113 */ if (project_isNull_0) { /* 114 */ rdd_mutableStateArray_0[2].setNullAt(0); /* 115 */ } else { /* 116 */ final InternalRow project_tmpInput_0 = project_value_0; /* 117 */ if (project_tmpInput_0 instanceof UnsafeRow) { /* 118 */ rdd_mutableStateArray_0[2].write(0, (UnsafeRow) project_tmpInput_0); /* 119 */ } else { /* 120 */ // Remember the current cursor so that we can calculate how many bytes are /* 121 */ // written later. /* 122 */ final int project_previousCursor_0 = rdd_mutableStateArray_0[2].cursor(); /* 123 */ /* 124 */ rdd_mutableStateArray_0[3].resetRowWriter(); /* 125 */ project_writeFields_0_0(project_tmpInput_0); /* 126 */ project_writeFields_0_1(project_tmpInput_0); /* 127 */ /* 128 */ rdd_mutableStateArray_0[2].setOffsetAndSizeFromPreviousCursor(0, project_previousCursor_0); /* 129 */ } /* 130 */ } /* 131 */ append((rdd_mutableStateArray_0[2].getRow())); /* 132 */ if (shouldStop()) return; /* 133 */ } /* 134 */ } /* 135 */ /* 136 */ private void wholestagecodegen_init_0_0() { /* 137 */ rdd_input_0 = inputs[0]; /* 138 */ rdd_mutableStateArray_0[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(3, 64); /* 139 */ rdd_mutableStateArray_1[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[0], 8); /* 140 */ rdd_mutableStateArray_0[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(rdd_mutableStateArray_0[0], 4); /* 141 */ rdd_mutableStateArray_1[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[1], 4); /* 142 */ rdd_mutableStateArray_1[2] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[1], 8); /* 143 */ rdd_mutableStateArray_0[2] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(1, 32); /* 144 */ rdd_mutableStateArray_0[3] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(rdd_mutableStateArray_0[2], 4); /* 145 */ rdd_mutableStateArray_1[3] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[3], 4); /* 146 */ /* 147 */ } /* 148 */ /* 149 */ } 20/01/23 18:54:01.746 Thread-4 DEBUG CodeGenerator: /* 001 */ public Object generate(Object[] references) { /* 002 */ return new GeneratedIteratorForCodegenStage1(references); /* 003 */ } /* 004 */ /* 005 */ // codegenStageId=1 /* 006 */ final class GeneratedIteratorForCodegenStage1 extends org.apache.spark.sql.execution.BufferedRowIterator { /* 007 */ private Object[] references; /* 008 */ private scala.collection.Iterator[] inputs; /* 009 */ private scala.collection.Iterator rdd_input_0; /* 010 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[] rdd_mutableStateArray_0 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[4]; /* 011 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[] rdd_mutableStateArray_1 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[5]; /* 012 */ /* 013 */ public GeneratedIteratorForCodegenStage1(Object[] references) { /* 014 */ this.references = references; /* 015 */ } /* 016 */ /* 017 */ public void init(int index, scala.collection.Iterator[] inputs) { /* 018 */ partitionIndex = index; /* 019 */ this.inputs = inputs; /* 020 */ wholestagecodegen_init_0_0(); /* 021 */ wholestagecodegen_init_0_1(); /* 022 */ /* 023 */ } /* 024 */ /* 025 */ private void project_writeFields_0_1(InternalRow project_tmpInput_0) { /* 026 */ if ((project_tmpInput_0.isNullAt(3))) { /* 027 */ rdd_mutableStateArray_0[3].setNullAt(3); /* 028 */ } else { /* 029 */ // Remember the current cursor so that we can calculate how many bytes are /* 030 */ // written later. /* 031 */ final int project_previousCursor_2 = rdd_mutableStateArray_0[3].cursor(); /* 032 */ /* 033 */ final ArrayData project_tmpInput_2 = (project_tmpInput_0.getArray(3)); /* 034 */ if (project_tmpInput_2 instanceof UnsafeArrayData) { /* 035 */ rdd_mutableStateArray_0[3].write((UnsafeArrayData) project_tmpInput_2); /* 036 */ } else { /* 037 */ final int project_numElements_1 = project_tmpInput_2.numElements(); /* 038 */ rdd_mutableStateArray_1[4].initialize(project_numElements_1); /* 039 */ /* 040 */ for (int project_index_1 = 0; project_index_1 < project_numElements_1; project_index_1++) { /* 041 */ rdd_mutableStateArray_1[4].write(project_index_1, project_tmpInput_2.getDouble(project_index_1)); /* 042 */ } /* 043 */ } /* 044 */ /* 045 */ rdd_mutableStateArray_0[3].setOffsetAndSizeFromPreviousCursor(3, project_previousCursor_2); /* 046 */ } /* 047 */ /* 048 */ } /* 049 */ /* 050 */ private void wholestagecodegen_init_0_1() { /* 051 */ rdd_mutableStateArray_1[4] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[3], 8); /* 052 */ /* 053 */ } /* 054 */ /* 055 */ private void project_writeFields_0_0(InternalRow project_tmpInput_0) { /* 056 */ rdd_mutableStateArray_0[3].write(0, (project_tmpInput_0.getByte(0))); /* 057 */ /* 058 */ if ((project_tmpInput_0.isNullAt(1))) { /* 059 */ rdd_mutableStateArray_0[3].setNullAt(1); /* 060 */ } else { /* 061 */ rdd_mutableStateArray_0[3].write(1, (project_tmpInput_0.getInt(1))); /* 062 */ } /* 063 */ /* 064 */ if ((project_tmpInput_0.isNullAt(2))) { /* 065 */ rdd_mutableStateArray_0[3].setNullAt(2); /* 066 */ } else { /* 067 */ // Remember the current cursor so that we can calculate how many bytes are /* 068 */ // written later. /* 069 */ final int project_previousCursor_1 = rdd_mutableStateArray_0[3].cursor(); /* 070 */ /* 071 */ final ArrayData project_tmpInput_1 = (project_tmpInput_0.getArray(2)); /* 072 */ if (project_tmpInput_1 instanceof UnsafeArrayData) { /* 073 */ rdd_mutableStateArray_0[3].write((UnsafeArrayData) project_tmpInput_1); /* 074 */ } else { /* 075 */ final int project_numElements_0 = project_tmpInput_1.numElements(); /* 076 */ rdd_mutableStateArray_1[3].initialize(project_numElements_0); /* 077 */ /* 078 */ for (int project_index_0 = 0; project_index_0 < project_numElements_0; project_index_0++) { /* 079 */ rdd_mutableStateArray_1[3].write(project_index_0, project_tmpInput_1.getInt(project_index_0)); /* 080 */ } /* 081 */ } /* 082 */ /* 083 */ rdd_mutableStateArray_0[3].setOffsetAndSizeFromPreviousCursor(2, project_previousCursor_1); /* 084 */ } /* 085 */ /* 086 */ } /* 087 */ /* 088 */ protected void processNext() throws java.io.IOException { /* 089 */ while ( rdd_input_0.hasNext()) { /* 090 */ InternalRow rdd_row_0 = (InternalRow) rdd_input_0.next(); /* 091 */ ((org.apache.spark.sql.execution.metric.SQLMetric) references[0] /* numOutputRows */).add(1); /* 092 */ boolean rdd_isNull_1 = rdd_row_0.isNullAt(1); /* 093 */ ArrayData rdd_value_1 = rdd_isNull_1 ? /* 094 */ null : (rdd_row_0.getArray(1)); /* 095 */ Object project_arg_0 = rdd_isNull_1 ? null : ((scala.Function1[]) references[1] /* converters */)[0].apply(rdd_value_1); /* 096 */ /* 097 */ InternalRow project_result_0 = null; /* 098 */ try { /* 099 */ project_result_0 = (InternalRow)((scala.Function1[]) references[1] /* converters */)[1].apply(((scala.Function1) references[3] /* udf */).apply(project_arg_0)); /* 100 */ } catch (Exception e) { /* 101 */ throw new org.apache.spark.SparkException(((java.lang.String) references[2] /* errMsg */), e); /* 102 */ } /* 103 */ /* 104 */ boolean project_isNull_0 = project_result_0 == null; /* 105 */ InternalRow project_value_0 = null; /* 106 */ if (!project_isNull_0) { /* 107 */ project_value_0 = project_result_0; /* 108 */ } /* 109 */ rdd_mutableStateArray_0[2].reset(); /* 110 */ /* 111 */ rdd_mutableStateArray_0[2].zeroOutNullBytes(); /* 112 */ /* 113 */ if (project_isNull_0) { /* 114 */ rdd_mutableStateArray_0[2].setNullAt(0); /* 115 */ } else { /* 116 */ final InternalRow project_tmpInput_0 = project_value_0; /* 117 */ if (project_tmpInput_0 instanceof UnsafeRow) { /* 118 */ rdd_mutableStateArray_0[2].write(0, (UnsafeRow) project_tmpInput_0); /* 119 */ } else { /* 120 */ // Remember the current cursor so that we can calculate how many bytes are /* 121 */ // written later. /* 122 */ final int project_previousCursor_0 = rdd_mutableStateArray_0[2].cursor(); /* 123 */ /* 124 */ rdd_mutableStateArray_0[3].resetRowWriter(); /* 125 */ project_writeFields_0_0(project_tmpInput_0); /* 126 */ project_writeFields_0_1(project_tmpInput_0); /* 127 */ /* 128 */ rdd_mutableStateArray_0[2].setOffsetAndSizeFromPreviousCursor(0, project_previousCursor_0); /* 129 */ } /* 130 */ } /* 131 */ append((rdd_mutableStateArray_0[2].getRow())); /* 132 */ if (shouldStop()) return; /* 133 */ } /* 134 */ } /* 135 */ /* 136 */ private void wholestagecodegen_init_0_0() { /* 137 */ rdd_input_0 = inputs[0]; /* 138 */ rdd_mutableStateArray_0[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(3, 64); /* 139 */ rdd_mutableStateArray_1[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[0], 8); /* 140 */ rdd_mutableStateArray_0[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(rdd_mutableStateArray_0[0], 4); /* 141 */ rdd_mutableStateArray_1[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[1], 4); /* 142 */ rdd_mutableStateArray_1[2] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[1], 8); /* 143 */ rdd_mutableStateArray_0[2] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(1, 32); /* 144 */ rdd_mutableStateArray_0[3] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(rdd_mutableStateArray_0[2], 4); /* 145 */ rdd_mutableStateArray_1[3] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[3], 4); /* 146 */ /* 147 */ } /* 148 */ /* 149 */ } 20/01/23 18:54:01.752 Executor task launch worker for task 3 INFO Executor: Running task 2.0 in stage 1.0 (TID 3) 20/01/23 18:54:01.752 Executor task launch worker for task 2 INFO Executor: Running task 1.0 in stage 1.0 (TID 2) 20/01/23 18:54:01.752 Executor task launch worker for task 4 INFO Executor: Running task 3.0 in stage 1.0 (TID 4) 20/01/23 18:54:01.757 Executor task launch worker for task 2 DEBUG ExecutorMetricsPoller: stageTCMP: (1, 0) -> 2 20/01/23 18:54:01.757 Executor task launch worker for task 4 DEBUG ExecutorMetricsPoller: stageTCMP: (1, 0) -> 3 20/01/23 18:54:01.772 Executor task launch worker for task 3 DEBUG ExecutorMetricsPoller: stageTCMP: (1, 0) -> 4 20/01/23 18:54:01.820 Thread-4 INFO CodeGenerator: Code generated in 77.669554 ms 20/01/23 18:54:01.821 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$doExecute$4$adapted 20/01/23 18:54:01.822 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$doExecute$4$adapted) is now cleaned +++ 20/01/23 18:54:01.824 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$collect$2 20/01/23 18:54:01.827 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$collect$2) is now cleaned +++ 20/01/23 18:54:01.830 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$runJob$5 20/01/23 18:54:01.833 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$runJob$5) is now cleaned +++ 20/01/23 18:54:01.834 Thread-4 INFO SparkContext: Starting job: at /usr/lib/python3.6/runpy.py:85 20/01/23 18:54:01.836 dag-scheduler-event-loop INFO DAGScheduler: Got job 1 ( at /usr/lib/python3.6/runpy.py:85) with 4 output partitions 20/01/23 18:54:01.836 dag-scheduler-event-loop INFO DAGScheduler: Final stage: ResultStage 1 ( at /usr/lib/python3.6/runpy.py:85) 20/01/23 18:54:01.836 dag-scheduler-event-loop INFO DAGScheduler: Parents of final stage: List() 20/01/23 18:54:01.837 dag-scheduler-event-loop INFO DAGScheduler: Missing parents: List() 20/01/23 18:54:01.837 dag-scheduler-event-loop DEBUG DAGScheduler: submitStage(ResultStage 1 (name= at /usr/lib/python3.6/runpy.py:85;jobs=1)) 20/01/23 18:54:01.837 dag-scheduler-event-loop DEBUG DAGScheduler: missing: List() 20/01/23 18:54:01.838 dag-scheduler-event-loop INFO DAGScheduler: Submitting ResultStage 1 (MapPartitionsRDD[8] at at /usr/lib/python3.6/runpy.py:85), which has no missing parents 20/01/23 18:54:01.838 dag-scheduler-event-loop DEBUG DAGScheduler: submitMissingTasks(ResultStage 1) 20/01/23 18:54:01.847 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_3 stored as values in memory (estimated size 21.6 KiB, free 366.2 MiB) 20/01/23 18:54:01.847 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_3 locally took 2 ms 20/01/23 18:54:01.847 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_3 without replication took 2 ms 20/01/23 18:54:01.849 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_3_piece0 stored as bytes in memory (estimated size 9.9 KiB, free 366.2 MiB) 20/01/23 18:54:01.849 Executor task launch worker for task 1 DEBUG GenerateSafeProjection: code for createexternalrow(input[0, int, false], input[1, int, false], input[2, float, true], StructField(UDF(user),IntegerType,false), StructField(UDF(item),IntegerType,false), StructField(rating,FloatType,true)): /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificSafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificSafeProjection extends org.apache.spark.sql.catalyst.expressions.codegen.BaseProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private InternalRow mutableRow; /* 009 */ /* 010 */ /* 011 */ public SpecificSafeProjection(Object[] references) { /* 012 */ this.references = references; /* 013 */ mutableRow = (InternalRow) references[references.length - 1]; /* 014 */ /* 015 */ } /* 016 */ /* 017 */ public void initialize(int partitionIndex) { /* 018 */ /* 019 */ } /* 020 */ /* 021 */ public java.lang.Object apply(java.lang.Object _i) { /* 022 */ InternalRow i = (InternalRow) _i; /* 023 */ Object[] values_0 = new Object[3]; /* 024 */ /* 025 */ int value_1 = i.getInt(0); /* 026 */ if (false) { /* 027 */ values_0[0] = null; /* 028 */ } else { /* 029 */ values_0[0] = value_1; /* 030 */ } /* 031 */ /* 032 */ int value_2 = i.getInt(1); /* 033 */ if (false) { /* 034 */ values_0[1] = null; /* 035 */ } else { /* 036 */ values_0[1] = value_2; /* 037 */ } /* 038 */ /* 039 */ boolean isNull_3 = i.isNullAt(2); /* 040 */ float value_3 = isNull_3 ? /* 041 */ -1.0f : (i.getFloat(2)); /* 042 */ if (isNull_3) { /* 043 */ values_0[2] = null; /* 044 */ } else { /* 045 */ values_0[2] = value_3; /* 046 */ } /* 047 */ /* 048 */ final org.apache.spark.sql.Row value_0 = new org.apache.spark.sql.catalyst.expressions.GenericRowWithSchema(values_0, ((org.apache.spark.sql.types.StructType) references[0] /* schema */)); /* 049 */ if (false) { /* 050 */ mutableRow.setNullAt(0); /* 051 */ } else { /* 052 */ /* 053 */ mutableRow.update(0, value_0); /* 054 */ } /* 055 */ /* 056 */ return mutableRow; /* 057 */ } /* 058 */ /* 059 */ /* 060 */ } 20/01/23 18:54:01.850 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added broadcast_3_piece0 in memory on 192.168.1.225:41233 (size: 9.9 KiB, free: 366.3 MiB) 20/01/23 18:54:01.850 dag-scheduler-event-loop DEBUG BlockManagerMaster: Updated info of block broadcast_3_piece0 20/01/23 18:54:01.850 dag-scheduler-event-loop DEBUG BlockManager: Told master about block broadcast_3_piece0 20/01/23 18:54:01.850 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_3_piece0 locally took 2 ms 20/01/23 18:54:01.851 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_3_piece0 without replication took 2 ms 20/01/23 18:54:01.851 dag-scheduler-event-loop INFO SparkContext: Created broadcast 3 from broadcast at DAGScheduler.scala:1200 20/01/23 18:54:01.852 dag-scheduler-event-loop INFO DAGScheduler: Submitting 4 missing tasks from ResultStage 1 (MapPartitionsRDD[8] at at /usr/lib/python3.6/runpy.py:85) (first 15 tasks are for partitions Vector(0, 1, 2, 3)) 20/01/23 18:54:01.852 dag-scheduler-event-loop INFO TaskSchedulerImpl: Adding task set 1.0 with 4 tasks 20/01/23 18:54:01.852 dag-scheduler-event-loop DEBUG TaskSetManager: Epoch for TaskSet 1.0: 0 20/01/23 18:54:01.852 dag-scheduler-event-loop DEBUG TaskSetManager: Adding pending tasks took 0 ms 20/01/23 18:54:01.853 dag-scheduler-event-loop DEBUG TaskSetManager: Valid locality levels for TaskSet 1.0: NO_PREF, ANY 20/01/23 18:54:01.853 dispatcher-event-loop-3 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_1.0, runningTasks: 0 20/01/23 18:54:01.854 dispatcher-event-loop-3 INFO TaskSetManager: Starting task 0.0 in stage 1.0 (TID 4, 192.168.1.225, executor driver, partition 0, PROCESS_LOCAL, 7333 bytes) 20/01/23 18:54:01.854 dispatcher-event-loop-3 INFO TaskSetManager: Starting task 1.0 in stage 1.0 (TID 5, 192.168.1.225, executor driver, partition 1, PROCESS_LOCAL, 7421 bytes) 20/01/23 18:54:01.855 dispatcher-event-loop-3 INFO TaskSetManager: Starting task 2.0 in stage 1.0 (TID 6, 192.168.1.225, executor driver, partition 2, PROCESS_LOCAL, 7396 bytes) 20/01/23 18:54:01.855 dispatcher-event-loop-3 INFO TaskSetManager: Starting task 3.0 in stage 1.0 (TID 7, 192.168.1.225, executor driver, partition 3, PROCESS_LOCAL, 7385 bytes) 20/01/23 18:54:01.856 Executor task launch worker for task 4 INFO Executor: Running task 0.0 in stage 1.0 (TID 4) 20/01/23 18:54:01.856 Executor task launch worker for task 5 INFO Executor: Running task 1.0 in stage 1.0 (TID 5) 20/01/23 18:54:01.856 Executor task launch worker for task 6 INFO Executor: Running task 2.0 in stage 1.0 (TID 6) 20/01/23 18:54:01.856 Executor task launch worker for task 7 INFO Executor: Running task 3.0 in stage 1.0 (TID 7) 20/01/23 18:54:01.858 Executor task launch worker for task 7 DEBUG ExecutorMetricsPoller: stageTCMP: (1, 0) -> 1 20/01/23 18:54:01.859 Executor task launch worker for task 7 DEBUG BlockManager: Getting local block broadcast_3 20/01/23 18:54:01.859 Executor task launch worker for task 7 DEBUG BlockManager: Level for block broadcast_3 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:01.862 Executor task launch worker for task 4 DEBUG ExecutorMetricsPoller: stageTCMP: (1, 0) -> 2 20/01/23 18:54:01.864 Executor task launch worker for task 5 DEBUG ExecutorMetricsPoller: stageTCMP: (1, 0) -> 3 20/01/23 18:54:01.864 Executor task launch worker for task 6 DEBUG ExecutorMetricsPoller: stageTCMP: (1, 0) -> 4 20/01/23 18:54:01.873 Executor task launch worker for task 3 DEBUG GenerateSafeProjection: code for createexternalrow(input[0, int, false], input[1, int, false], input[2, float, true], StructField(UDF(user),IntegerType,false), StructField(UDF(item),IntegerType,false), StructField(rating,FloatType,true)): /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificSafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificSafeProjection extends org.apache.spark.sql.catalyst.expressions.codegen.BaseProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private InternalRow mutableRow; /* 009 */ /* 010 */ /* 011 */ public SpecificSafeProjection(Object[] references) { /* 012 */ this.references = references; /* 013 */ mutableRow = (InternalRow) references[references.length - 1]; /* 014 */ /* 015 */ } /* 016 */ /* 017 */ public void initialize(int partitionIndex) { /* 018 */ /* 019 */ } /* 020 */ /* 021 */ public java.lang.Object apply(java.lang.Object _i) { /* 022 */ InternalRow i = (InternalRow) _i; /* 023 */ Object[] values_0 = new Object[3]; /* 024 */ /* 025 */ int value_1 = i.getInt(0); /* 026 */ if (false) { /* 027 */ values_0[0] = null; /* 028 */ } else { /* 029 */ values_0[0] = value_1; /* 030 */ } /* 031 */ /* 032 */ int value_2 = i.getInt(1); /* 033 */ if (false) { /* 034 */ values_0[1] = null; /* 035 */ } else { /* 036 */ values_0[1] = value_2; /* 037 */ } /* 038 */ /* 039 */ boolean isNull_3 = i.isNullAt(2); /* 040 */ float value_3 = isNull_3 ? /* 041 */ -1.0f : (i.getFloat(2)); /* 042 */ if (isNull_3) { /* 043 */ values_0[2] = null; /* 044 */ } else { /* 045 */ values_0[2] = value_3; /* 046 */ } /* 047 */ /* 048 */ final org.apache.spark.sql.Row value_0 = new org.apache.spark.sql.catalyst.expressions.GenericRowWithSchema(values_0, ((org.apache.spark.sql.types.StructType) references[0] /* schema */)); /* 049 */ if (false) { /* 050 */ mutableRow.setNullAt(0); /* 051 */ } else { /* 052 */ /* 053 */ mutableRow.update(0, value_0); /* 054 */ } /* 055 */ /* 056 */ return mutableRow; /* 057 */ } /* 058 */ /* 059 */ /* 060 */ } 20/01/23 18:54:01.893 Executor task launch worker for task 4 DEBUG GenerateSafeProjection: code for createexternalrow(input[0, int, false], input[1, int, false], input[2, float, true], StructField(UDF(user),IntegerType,false), StructField(UDF(item),IntegerType,false), StructField(rating,FloatType,true)): /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificSafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificSafeProjection extends org.apache.spark.sql.catalyst.expressions.codegen.BaseProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private InternalRow mutableRow; /* 009 */ /* 010 */ /* 011 */ public SpecificSafeProjection(Object[] references) { /* 012 */ this.references = references; /* 013 */ mutableRow = (InternalRow) references[references.length - 1]; /* 014 */ /* 015 */ } /* 016 */ /* 017 */ public void initialize(int partitionIndex) { /* 018 */ /* 019 */ } /* 020 */ /* 021 */ public java.lang.Object apply(java.lang.Object _i) { /* 022 */ InternalRow i = (InternalRow) _i; /* 023 */ Object[] values_0 = new Object[3]; /* 024 */ /* 025 */ int value_1 = i.getInt(0); /* 026 */ if (false) { /* 027 */ values_0[0] = null; /* 028 */ } else { /* 029 */ values_0[0] = value_1; /* 030 */ } /* 031 */ /* 032 */ int value_2 = i.getInt(1); /* 033 */ if (false) { /* 034 */ values_0[1] = null; /* 035 */ } else { /* 036 */ values_0[1] = value_2; /* 037 */ } /* 038 */ /* 039 */ boolean isNull_3 = i.isNullAt(2); /* 040 */ float value_3 = isNull_3 ? /* 041 */ -1.0f : (i.getFloat(2)); /* 042 */ if (isNull_3) { /* 043 */ values_0[2] = null; /* 044 */ } else { /* 045 */ values_0[2] = value_3; /* 046 */ } /* 047 */ /* 048 */ final org.apache.spark.sql.Row value_0 = new org.apache.spark.sql.catalyst.expressions.GenericRowWithSchema(values_0, ((org.apache.spark.sql.types.StructType) references[0] /* schema */)); /* 049 */ if (false) { /* 050 */ mutableRow.setNullAt(0); /* 051 */ } else { /* 052 */ /* 053 */ mutableRow.update(0, value_0); /* 054 */ } /* 055 */ /* 056 */ return mutableRow; /* 057 */ } /* 058 */ /* 059 */ /* 060 */ } 20/01/23 18:54:01.933 Executor task launch worker for task 2 DEBUG GenerateSafeProjection: code for createexternalrow(input[0, int, false], input[1, int, false], input[2, float, true], StructField(UDF(user),IntegerType,false), StructField(UDF(item),IntegerType,false), StructField(rating,FloatType,true)): /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificSafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificSafeProjection extends org.apache.spark.sql.catalyst.expressions.codegen.BaseProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private InternalRow mutableRow; /* 009 */ /* 010 */ /* 011 */ public SpecificSafeProjection(Object[] references) { /* 012 */ this.references = references; /* 013 */ mutableRow = (InternalRow) references[references.length - 1]; /* 014 */ /* 015 */ } /* 016 */ /* 017 */ public void initialize(int partitionIndex) { /* 018 */ /* 019 */ } /* 020 */ /* 021 */ public java.lang.Object apply(java.lang.Object _i) { /* 022 */ InternalRow i = (InternalRow) _i; /* 023 */ Object[] values_0 = new Object[3]; /* 024 */ /* 025 */ int value_1 = i.getInt(0); /* 026 */ if (false) { /* 027 */ values_0[0] = null; /* 028 */ } else { /* 029 */ values_0[0] = value_1; /* 030 */ } /* 031 */ /* 032 */ int value_2 = i.getInt(1); /* 033 */ if (false) { /* 034 */ values_0[1] = null; /* 035 */ } else { /* 036 */ values_0[1] = value_2; /* 037 */ } /* 038 */ /* 039 */ boolean isNull_3 = i.isNullAt(2); /* 040 */ float value_3 = isNull_3 ? /* 041 */ -1.0f : (i.getFloat(2)); /* 042 */ if (isNull_3) { /* 043 */ values_0[2] = null; /* 044 */ } else { /* 045 */ values_0[2] = value_3; /* 046 */ } /* 047 */ /* 048 */ final org.apache.spark.sql.Row value_0 = new org.apache.spark.sql.catalyst.expressions.GenericRowWithSchema(values_0, ((org.apache.spark.sql.types.StructType) references[0] /* schema */)); /* 049 */ if (false) { /* 050 */ mutableRow.setNullAt(0); /* 051 */ } else { /* 052 */ /* 053 */ mutableRow.update(0, value_0); /* 054 */ } /* 055 */ /* 056 */ return mutableRow; /* 057 */ } /* 058 */ /* 059 */ /* 060 */ } 20/01/23 18:54:01.900 Executor task launch worker for task 1 INFO PythonRunner: Times: total = 101, boot = 26, init = 75, finish = 0 20/01/23 18:54:01.914 Executor task launch worker for task 3 INFO PythonRunner: Times: total = 114, boot = 43, init = 71, finish = 0 20/01/23 18:54:01.955 Executor task launch worker for task 6 DEBUG BlockManager: Getting local block broadcast_2 20/01/23 18:54:01.955 Executor task launch worker for task 6 DEBUG BlockManager: Level for block broadcast_2 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:01.956 Executor task launch worker for task 6 INFO PythonRunner: Times: total = 51, boot = -558, init = 609, finish = 0 20/01/23 18:54:01.958 Executor task launch worker for task 6 INFO Executor: Finished task 2.0 in stage 1.0 (TID 6). 2099 bytes result sent to driver 20/01/23 18:54:01.958 Executor task launch worker for task 6 DEBUG ExecutorMetricsPoller: stageTCMP: (1, 0) -> 3 20/01/23 18:54:01.958 Executor task launch worker for task 4 INFO PythonRunner: Times: total = 50, boot = -543, init = 593, finish = 0 20/01/23 18:54:01.960 Executor task launch worker for task 4 INFO Executor: Finished task 0.0 in stage 1.0 (TID 4). 2051 bytes result sent to driver 20/01/23 18:54:01.960 Executor task launch worker for task 4 DEBUG ExecutorMetricsPoller: stageTCMP: (1, 0) -> 2 20/01/23 18:54:01.962 dispatcher-event-loop-1 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_1.0, runningTasks: 3 20/01/23 18:54:01.962 dispatcher-event-loop-1 DEBUG TaskSetManager: No tasks for locality level NO_PREF, so moving to locality level ANY 20/01/23 18:54:01.963 dispatcher-event-loop-1 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_1.0, runningTasks: 2 20/01/23 18:54:01.964 task-result-getter-1 INFO TaskSetManager: Finished task 0.0 in stage 1.0 (TID 4) in 110 ms on 192.168.1.225 (executor driver) (1/4) 20/01/23 18:54:01.965 task-result-getter-0 INFO TaskSetManager: Finished task 2.0 in stage 1.0 (TID 6) in 111 ms on 192.168.1.225 (executor driver) (2/4) 20/01/23 18:54:01.970 Executor task launch worker for task 5 INFO PythonRunner: Times: total = 88, boot = -554, init = 642, finish = 0 20/01/23 18:54:01.971 Executor task launch worker for task 5 INFO Executor: Finished task 1.0 in stage 1.0 (TID 5). 2114 bytes result sent to driver 20/01/23 18:54:01.972 Executor task launch worker for task 5 DEBUG ExecutorMetricsPoller: stageTCMP: (1, 0) -> 1 20/01/23 18:54:01.972 dispatcher-event-loop-0 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_1.0, runningTasks: 1 20/01/23 18:54:01.973 task-result-getter-2 INFO TaskSetManager: Finished task 1.0 in stage 1.0 (TID 5) in 118 ms on 192.168.1.225 (executor driver) (3/4) 20/01/23 18:54:01.978 Executor task launch worker for task 7 INFO PythonRunner: Times: total = 97, boot = -548, init = 645, finish = 0 20/01/23 18:54:01.978 Executor task launch worker for task 4 INFO PythonRunner: Times: total = 178, boot = 82, init = 96, finish = 0 20/01/23 18:54:01.980 Executor task launch worker for task 2 INFO PythonRunner: Times: total = 172, boot = 81, init = 91, finish = 0 20/01/23 18:54:01.982 Executor task launch worker for task 7 INFO Executor: Finished task 3.0 in stage 1.0 (TID 7). 2115 bytes result sent to driver 20/01/23 18:54:01.982 Executor task launch worker for task 7 DEBUG ExecutorMetricsPoller: removing (1, 0) from stageTCMP 20/01/23 18:54:01.982 dispatcher-event-loop-3 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_1.0, runningTasks: 0 20/01/23 18:54:01.983 task-result-getter-3 INFO TaskSetManager: Finished task 3.0 in stage 1.0 (TID 7) in 128 ms on 192.168.1.225 (executor driver) (4/4) 20/01/23 18:54:01.983 task-result-getter-3 INFO TaskSchedulerImpl: Removed TaskSet 1.0, whose tasks have all completed, from pool 20/01/23 18:54:01.985 dag-scheduler-event-loop INFO DAGScheduler: ResultStage 1 ( at /usr/lib/python3.6/runpy.py:85) finished in 0.146 s 20/01/23 18:54:01.985 dag-scheduler-event-loop DEBUG DAGScheduler: After removal of stage 1, remaining stages = 0 20/01/23 18:54:01.985 dag-scheduler-event-loop INFO DAGScheduler: Job 1 is finished. Cancelling potential speculative or zombie tasks for this job 20/01/23 18:54:01.985 dag-scheduler-event-loop INFO TaskSchedulerImpl: Killing all running tasks in stage 1: Stage finished 20/01/23 18:54:01.985 Thread-4 INFO DAGScheduler: Job 1 finished: at /usr/lib/python3.6/runpy.py:85, took 0.150908 s 20/01/23 18:54:02.027 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$pythonToJava$1 20/01/23 18:54:02.028 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$pythonToJava$1) is now cleaned +++ 20/01/23 18:54:02.030 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$toJavaArray$1 20/01/23 18:54:02.030 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$toJavaArray$1) is now cleaned +++ 20/01/23 18:54:02.033 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$applySchemaToPythonRDD$1 20/01/23 18:54:02.034 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$applySchemaToPythonRDD$1) is now cleaned +++ 20/01/23 18:54:02.089 Executor task launch worker for task 3 DEBUG GenerateSafeProjection: code for createexternalrow(input[0, double, true], newInstance(class org.apache.spark.ml.linalg.VectorUDT).deserialize, StructField(prediction,DoubleType,true), StructField(normalizedFeatures,org.apache.spark.ml.linalg.VectorUDT@3bfc3ba7,true)): /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificSafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificSafeProjection extends org.apache.spark.sql.catalyst.expressions.codegen.BaseProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private InternalRow mutableRow; /* 009 */ private boolean resultIsNull_0; /* 010 */ private InternalRow[] mutableStateArray_0 = new InternalRow[1]; /* 011 */ /* 012 */ public SpecificSafeProjection(Object[] references) { /* 013 */ this.references = references; /* 014 */ mutableRow = (InternalRow) references[references.length - 1]; /* 015 */ /* 016 */ /* 017 */ } /* 018 */ /* 019 */ public void initialize(int partitionIndex) { /* 020 */ /* 021 */ } /* 022 */ /* 023 */ public java.lang.Object apply(java.lang.Object _i) { /* 024 */ InternalRow i = (InternalRow) _i; /* 025 */ org.apache.spark.sql.Row value_5 = CreateExternalRow_0(i); /* 026 */ if (false) { /* 027 */ mutableRow.setNullAt(0); /* 028 */ } else { /* 029 */ /* 030 */ mutableRow.update(0, value_5); /* 031 */ } /* 032 */ /* 033 */ return mutableRow; /* 034 */ } /* 035 */ /* 036 */ /* 037 */ private org.apache.spark.sql.Row CreateExternalRow_0(InternalRow i) { /* 038 */ Object[] values_0 = new Object[2]; /* 039 */ /* 040 */ boolean isNull_1 = i.isNullAt(0); /* 041 */ double value_1 = isNull_1 ? /* 042 */ -1.0 : (i.getDouble(0)); /* 043 */ if (isNull_1) { /* 044 */ values_0[0] = null; /* 045 */ } else { /* 046 */ values_0[0] = value_1; /* 047 */ } /* 048 */ /* 049 */ final org.apache.spark.ml.linalg.VectorUDT value_3 = false ? /* 050 */ null : new org.apache.spark.ml.linalg.VectorUDT(); /* 051 */ boolean isNull_2 = true; /* 052 */ org.apache.spark.ml.linalg.Vector value_2 = null; /* 053 */ if (!false) { /* 054 */ resultIsNull_0 = false; /* 055 */ if (!resultIsNull_0) { /* 056 */ boolean isNull_4 = i.isNullAt(1); /* 057 */ InternalRow value_4 = isNull_4 ? /* 058 */ null : (i.getStruct(1, 4)); /* 059 */ resultIsNull_0 = isNull_4; /* 060 */ mutableStateArray_0[0] = value_4; /* 061 */ } /* 062 */ /* 063 */ isNull_2 = resultIsNull_0; /* 064 */ if (!isNull_2) { /* 065 */ /* 066 */ Object funcResult_0 = null; /* 067 */ funcResult_0 = value_3.deserialize(mutableStateArray_0[0]); /* 068 */ /* 069 */ if (funcResult_0 != null) { /* 070 */ value_2 = (org.apache.spark.ml.linalg.Vector) funcResult_0; /* 071 */ } else { /* 072 */ isNull_2 = true; /* 073 */ } /* 074 */ /* 075 */ /* 076 */ } /* 077 */ } /* 078 */ if (isNull_2) { /* 079 */ values_0[1] = null; /* 080 */ } else { /* 081 */ values_0[1] = value_2; /* 082 */ } /* 083 */ /* 084 */ final org.apache.spark.sql.Row value_0 = new org.apache.spark.sql.catalyst.expressions.GenericRowWithSchema(values_0, ((org.apache.spark.sql.types.StructType) references[0] /* schema */)); /* 085 */ /* 086 */ return value_0; /* 087 */ } /* 088 */ /* 089 */ } 20/01/23 18:54:02.089 Executor task launch worker for task 4 DEBUG GenerateSafeProjection: code for createexternalrow(input[0, double, true], newInstance(class org.apache.spark.ml.linalg.VectorUDT).deserialize, StructField(prediction,DoubleType,true), StructField(normalizedFeatures,org.apache.spark.ml.linalg.VectorUDT@3bfc3ba7,true)): /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificSafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificSafeProjection extends org.apache.spark.sql.catalyst.expressions.codegen.BaseProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private InternalRow mutableRow; /* 009 */ private boolean resultIsNull_0; /* 010 */ private InternalRow[] mutableStateArray_0 = new InternalRow[1]; /* 011 */ /* 012 */ public SpecificSafeProjection(Object[] references) { /* 013 */ this.references = references; /* 014 */ mutableRow = (InternalRow) references[references.length - 1]; /* 015 */ /* 016 */ /* 017 */ } /* 018 */ /* 019 */ public void initialize(int partitionIndex) { /* 020 */ /* 021 */ } /* 022 */ /* 023 */ public java.lang.Object apply(java.lang.Object _i) { /* 024 */ InternalRow i = (InternalRow) _i; /* 025 */ org.apache.spark.sql.Row value_5 = CreateExternalRow_0(i); /* 026 */ if (false) { /* 027 */ mutableRow.setNullAt(0); /* 028 */ } else { /* 029 */ /* 030 */ mutableRow.update(0, value_5); /* 031 */ } /* 032 */ /* 033 */ return mutableRow; /* 034 */ } /* 035 */ /* 036 */ /* 037 */ private org.apache.spark.sql.Row CreateExternalRow_0(InternalRow i) { /* 038 */ Object[] values_0 = new Object[2]; /* 039 */ /* 040 */ boolean isNull_1 = i.isNullAt(0); /* 041 */ double value_1 = isNull_1 ? /* 042 */ -1.0 : (i.getDouble(0)); /* 043 */ if (isNull_1) { /* 044 */ values_0[0] = null; /* 045 */ } else { /* 046 */ values_0[0] = value_1; /* 047 */ } /* 048 */ /* 049 */ final org.apache.spark.ml.linalg.VectorUDT value_3 = false ? /* 050 */ null : new org.apache.spark.ml.linalg.VectorUDT(); /* 051 */ boolean isNull_2 = true; /* 052 */ org.apache.spark.ml.linalg.Vector value_2 = null; /* 053 */ if (!false) { /* 054 */ resultIsNull_0 = false; /* 055 */ if (!resultIsNull_0) { /* 056 */ boolean isNull_4 = i.isNullAt(1); /* 057 */ InternalRow value_4 = isNull_4 ? /* 058 */ null : (i.getStruct(1, 4)); /* 059 */ resultIsNull_0 = isNull_4; /* 060 */ mutableStateArray_0[0] = value_4; /* 061 */ } /* 062 */ /* 063 */ isNull_2 = resultIsNull_0; /* 064 */ if (!isNull_2) { /* 065 */ /* 066 */ Object funcResult_0 = null; /* 067 */ funcResult_0 = value_3.deserialize(mutableStateArray_0[0]); /* 068 */ /* 069 */ if (funcResult_0 != null) { /* 070 */ value_2 = (org.apache.spark.ml.linalg.Vector) funcResult_0; /* 071 */ } else { /* 072 */ isNull_2 = true; /* 073 */ } /* 074 */ /* 075 */ /* 076 */ } /* 077 */ } /* 078 */ if (isNull_2) { /* 079 */ values_0[1] = null; /* 080 */ } else { /* 081 */ values_0[1] = value_2; /* 082 */ } /* 083 */ /* 084 */ final org.apache.spark.sql.Row value_0 = new org.apache.spark.sql.catalyst.expressions.GenericRowWithSchema(values_0, ((org.apache.spark.sql.types.StructType) references[0] /* schema */)); /* 085 */ /* 086 */ return value_0; /* 087 */ } /* 088 */ /* 089 */ } 20/01/23 18:54:02.096 Thread-4 DEBUG CountVectorizer: Input schema: {"type":"struct","fields":[{"name":"id","type":"long","nullable":true,"metadata":{}},{"name":"words","type":{"type":"array","elementType":"string","containsNull":true},"nullable":true,"metadata":{}},{"name":"expected","type":{"type":"udt","class":"org.apache.spark.ml.linalg.VectorUDT","pyClass":"pyspark.ml.linalg.VectorUDT","sqlType":{"type":"struct","fields":[{"name":"type","type":"byte","nullable":false,"metadata":{}},{"name":"size","type":"integer","nullable":true,"metadata":{}},{"name":"indices","type":{"type":"array","elementType":"integer","containsNull":false},"nullable":true,"metadata":{}},{"name":"values","type":{"type":"array","elementType":"double","containsNull":false},"nullable":true,"metadata":{}}]}},"nullable":true,"metadata":{}}]} 20/01/23 18:54:02.097 Thread-4 DEBUG CountVectorizer: Expected output schema: {"type":"struct","fields":[{"name":"id","type":"long","nullable":true,"metadata":{}},{"name":"words","type":{"type":"array","elementType":"string","containsNull":true},"nullable":true,"metadata":{}},{"name":"expected","type":{"type":"udt","class":"org.apache.spark.ml.linalg.VectorUDT","pyClass":"pyspark.ml.linalg.VectorUDT","sqlType":{"type":"struct","fields":[{"name":"type","type":"byte","nullable":false,"metadata":{}},{"name":"size","type":"integer","nullable":true,"metadata":{}},{"name":"indices","type":{"type":"array","elementType":"integer","containsNull":false},"nullable":true,"metadata":{}},{"name":"values","type":{"type":"array","elementType":"double","containsNull":false},"nullable":true,"metadata":{}}]}},"nullable":true,"metadata":{}},{"name":"features","type":{"type":"udt","class":"org.apache.spark.ml.linalg.VectorUDT","pyClass":"pyspark.ml.linalg.VectorUDT","sqlType":{"type":"struct","fields":[{"name":"type","type":"byte","nullable":false,"metadata":{}},{"name":"size","type":"integer","nullable":true,"metadata":{}},{"name":"indices","type":{"type":"array","elementType":"integer","containsNull":false},"nullable":true,"metadata":{}},{"name":"values","type":{"type":"array","elementType":"double","containsNull":false},"nullable":true,"metadata":{}}]}},"nullable":false,"metadata":{}}]} 20/01/23 18:54:02.100 Thread-4 DEBUG Analyzer$ResolveReferences: Resolving 'words to words#22 20/01/23 18:54:02.100 Executor task launch worker for task 2 DEBUG GenerateSafeProjection: code for createexternalrow(input[0, double, true], newInstance(class org.apache.spark.ml.linalg.VectorUDT).deserialize, StructField(prediction,DoubleType,true), StructField(normalizedFeatures,org.apache.spark.ml.linalg.VectorUDT@3bfc3ba7,true)): /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificSafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificSafeProjection extends org.apache.spark.sql.catalyst.expressions.codegen.BaseProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private InternalRow mutableRow; /* 009 */ private boolean resultIsNull_0; /* 010 */ private InternalRow[] mutableStateArray_0 = new InternalRow[1]; /* 011 */ /* 012 */ public SpecificSafeProjection(Object[] references) { /* 013 */ this.references = references; /* 014 */ mutableRow = (InternalRow) references[references.length - 1]; /* 015 */ /* 016 */ /* 017 */ } /* 018 */ /* 019 */ public void initialize(int partitionIndex) { /* 020 */ /* 021 */ } /* 022 */ /* 023 */ public java.lang.Object apply(java.lang.Object _i) { /* 024 */ InternalRow i = (InternalRow) _i; /* 025 */ org.apache.spark.sql.Row value_5 = CreateExternalRow_0(i); /* 026 */ if (false) { /* 027 */ mutableRow.setNullAt(0); /* 028 */ } else { /* 029 */ /* 030 */ mutableRow.update(0, value_5); /* 031 */ } /* 032 */ /* 033 */ return mutableRow; /* 034 */ } /* 035 */ /* 036 */ /* 037 */ private org.apache.spark.sql.Row CreateExternalRow_0(InternalRow i) { /* 038 */ Object[] values_0 = new Object[2]; /* 039 */ /* 040 */ boolean isNull_1 = i.isNullAt(0); /* 041 */ double value_1 = isNull_1 ? /* 042 */ -1.0 : (i.getDouble(0)); /* 043 */ if (isNull_1) { /* 044 */ values_0[0] = null; /* 045 */ } else { /* 046 */ values_0[0] = value_1; /* 047 */ } /* 048 */ /* 049 */ final org.apache.spark.ml.linalg.VectorUDT value_3 = false ? /* 050 */ null : new org.apache.spark.ml.linalg.VectorUDT(); /* 051 */ boolean isNull_2 = true; /* 052 */ org.apache.spark.ml.linalg.Vector value_2 = null; /* 053 */ if (!false) { /* 054 */ resultIsNull_0 = false; /* 055 */ if (!resultIsNull_0) { /* 056 */ boolean isNull_4 = i.isNullAt(1); /* 057 */ InternalRow value_4 = isNull_4 ? /* 058 */ null : (i.getStruct(1, 4)); /* 059 */ resultIsNull_0 = isNull_4; /* 060 */ mutableStateArray_0[0] = value_4; /* 061 */ } /* 062 */ /* 063 */ isNull_2 = resultIsNull_0; /* 064 */ if (!isNull_2) { /* 065 */ /* 066 */ Object funcResult_0 = null; /* 067 */ funcResult_0 = value_3.deserialize(mutableStateArray_0[0]); /* 068 */ /* 069 */ if (funcResult_0 != null) { /* 070 */ value_2 = (org.apache.spark.ml.linalg.Vector) funcResult_0; /* 071 */ } else { /* 072 */ isNull_2 = true; /* 073 */ } /* 074 */ /* 075 */ /* 076 */ } /* 077 */ } /* 078 */ if (isNull_2) { /* 079 */ values_0[1] = null; /* 080 */ } else { /* 081 */ values_0[1] = value_2; /* 082 */ } /* 083 */ /* 084 */ final org.apache.spark.sql.Row value_0 = new org.apache.spark.sql.catalyst.expressions.GenericRowWithSchema(values_0, ((org.apache.spark.sql.types.StructType) references[0] /* schema */)); /* 085 */ /* 086 */ return value_0; /* 087 */ } /* 088 */ /* 089 */ } 20/01/23 18:54:02.102 Executor task launch worker for task 1 DEBUG GenerateSafeProjection: code for createexternalrow(input[0, double, true], newInstance(class org.apache.spark.ml.linalg.VectorUDT).deserialize, StructField(prediction,DoubleType,true), StructField(normalizedFeatures,org.apache.spark.ml.linalg.VectorUDT@3bfc3ba7,true)): /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificSafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificSafeProjection extends org.apache.spark.sql.catalyst.expressions.codegen.BaseProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private InternalRow mutableRow; /* 009 */ private boolean resultIsNull_0; /* 010 */ private InternalRow[] mutableStateArray_0 = new InternalRow[1]; /* 011 */ /* 012 */ public SpecificSafeProjection(Object[] references) { /* 013 */ this.references = references; /* 014 */ mutableRow = (InternalRow) references[references.length - 1]; /* 015 */ /* 016 */ /* 017 */ } /* 018 */ /* 019 */ public void initialize(int partitionIndex) { /* 020 */ /* 021 */ } /* 022 */ /* 023 */ public java.lang.Object apply(java.lang.Object _i) { /* 024 */ InternalRow i = (InternalRow) _i; /* 025 */ org.apache.spark.sql.Row value_5 = CreateExternalRow_0(i); /* 026 */ if (false) { /* 027 */ mutableRow.setNullAt(0); /* 028 */ } else { /* 029 */ /* 030 */ mutableRow.update(0, value_5); /* 031 */ } /* 032 */ /* 033 */ return mutableRow; /* 034 */ } /* 035 */ /* 036 */ /* 037 */ private org.apache.spark.sql.Row CreateExternalRow_0(InternalRow i) { /* 038 */ Object[] values_0 = new Object[2]; /* 039 */ /* 040 */ boolean isNull_1 = i.isNullAt(0); /* 041 */ double value_1 = isNull_1 ? /* 042 */ -1.0 : (i.getDouble(0)); /* 043 */ if (isNull_1) { /* 044 */ values_0[0] = null; /* 045 */ } else { /* 046 */ values_0[0] = value_1; /* 047 */ } /* 048 */ /* 049 */ final org.apache.spark.ml.linalg.VectorUDT value_3 = false ? /* 050 */ null : new org.apache.spark.ml.linalg.VectorUDT(); /* 051 */ boolean isNull_2 = true; /* 052 */ org.apache.spark.ml.linalg.Vector value_2 = null; /* 053 */ if (!false) { /* 054 */ resultIsNull_0 = false; /* 055 */ if (!resultIsNull_0) { /* 056 */ boolean isNull_4 = i.isNullAt(1); /* 057 */ InternalRow value_4 = isNull_4 ? /* 058 */ null : (i.getStruct(1, 4)); /* 059 */ resultIsNull_0 = isNull_4; /* 060 */ mutableStateArray_0[0] = value_4; /* 061 */ } /* 062 */ /* 063 */ isNull_2 = resultIsNull_0; /* 064 */ if (!isNull_2) { /* 065 */ /* 066 */ Object funcResult_0 = null; /* 067 */ funcResult_0 = value_3.deserialize(mutableStateArray_0[0]); /* 068 */ /* 069 */ if (funcResult_0 != null) { /* 070 */ value_2 = (org.apache.spark.ml.linalg.Vector) funcResult_0; /* 071 */ } else { /* 072 */ isNull_2 = true; /* 073 */ } /* 074 */ /* 075 */ /* 076 */ } /* 077 */ } /* 078 */ if (isNull_2) { /* 079 */ values_0[1] = null; /* 080 */ } else { /* 081 */ values_0[1] = value_2; /* 082 */ } /* 083 */ /* 084 */ final org.apache.spark.sql.Row value_0 = new org.apache.spark.sql.catalyst.expressions.GenericRowWithSchema(values_0, ((org.apache.spark.sql.types.StructType) references[0] /* schema */)); /* 085 */ /* 086 */ return value_0; /* 087 */ } /* 088 */ /* 089 */ } 20/01/23 18:54:02.091 Executor task launch worker for task 3 DEBUG CodeGenerator: /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificSafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificSafeProjection extends org.apache.spark.sql.catalyst.expressions.codegen.BaseProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private InternalRow mutableRow; /* 009 */ private boolean resultIsNull_0; /* 010 */ private InternalRow[] mutableStateArray_0 = new InternalRow[1]; /* 011 */ /* 012 */ public SpecificSafeProjection(Object[] references) { /* 013 */ this.references = references; /* 014 */ mutableRow = (InternalRow) references[references.length - 1]; /* 015 */ /* 016 */ /* 017 */ } /* 018 */ /* 019 */ public void initialize(int partitionIndex) { /* 020 */ /* 021 */ } /* 022 */ /* 023 */ public java.lang.Object apply(java.lang.Object _i) { /* 024 */ InternalRow i = (InternalRow) _i; /* 025 */ org.apache.spark.sql.Row value_5 = CreateExternalRow_0(i); /* 026 */ if (false) { /* 027 */ mutableRow.setNullAt(0); /* 028 */ } else { /* 029 */ /* 030 */ mutableRow.update(0, value_5); /* 031 */ } /* 032 */ /* 033 */ return mutableRow; /* 034 */ } /* 035 */ /* 036 */ /* 037 */ private org.apache.spark.sql.Row CreateExternalRow_0(InternalRow i) { /* 038 */ Object[] values_0 = new Object[2]; /* 039 */ /* 040 */ boolean isNull_1 = i.isNullAt(0); /* 041 */ double value_1 = isNull_1 ? /* 042 */ -1.0 : (i.getDouble(0)); /* 043 */ if (isNull_1) { /* 044 */ values_0[0] = null; /* 045 */ } else { /* 046 */ values_0[0] = value_1; /* 047 */ } /* 048 */ /* 049 */ final org.apache.spark.ml.linalg.VectorUDT value_3 = false ? /* 050 */ null : new org.apache.spark.ml.linalg.VectorUDT(); /* 051 */ boolean isNull_2 = true; /* 052 */ org.apache.spark.ml.linalg.Vector value_2 = null; /* 053 */ if (!false) { /* 054 */ resultIsNull_0 = false; /* 055 */ if (!resultIsNull_0) { /* 056 */ boolean isNull_4 = i.isNullAt(1); /* 057 */ InternalRow value_4 = isNull_4 ? /* 058 */ null : (i.getStruct(1, 4)); /* 059 */ resultIsNull_0 = isNull_4; /* 060 */ mutableStateArray_0[0] = value_4; /* 061 */ } /* 062 */ /* 063 */ isNull_2 = resultIsNull_0; /* 064 */ if (!isNull_2) { /* 065 */ /* 066 */ Object funcResult_0 = null; /* 067 */ funcResult_0 = value_3.deserialize(mutableStateArray_0[0]); /* 068 */ /* 069 */ if (funcResult_0 != null) { /* 070 */ value_2 = (org.apache.spark.ml.linalg.Vector) funcResult_0; /* 071 */ } else { /* 072 */ isNull_2 = true; /* 073 */ } /* 074 */ /* 075 */ /* 076 */ } /* 077 */ } /* 078 */ if (isNull_2) { /* 079 */ values_0[1] = null; /* 080 */ } else { /* 081 */ values_0[1] = value_2; /* 082 */ } /* 083 */ /* 084 */ final org.apache.spark.sql.Row value_0 = new org.apache.spark.sql.catalyst.expressions.GenericRowWithSchema(values_0, ((org.apache.spark.sql.types.StructType) references[0] /* schema */)); /* 085 */ /* 086 */ return value_0; /* 087 */ } /* 088 */ /* 089 */ } 20/01/23 18:54:02.130 Executor task launch worker for task 3 INFO CodeGenerator: Code generated in 40.792244 ms 20/01/23 18:54:02.139 main DEBUG PythonGatewayServer: Exiting due to broken pipe from Python driver 20/01/23 18:54:02.160 Thread-1 INFO ShutdownHookManager: Shutdown hook called 20/01/23 18:54:02.161 Thread-1 INFO ShutdownHookManager: Deleting directory /home/jenkins/workspace/spark-master-test-python-arm/python/target/22a04de0-c639-4801-aead-76fbc7eb088f/spark-9be496c1-624c-4a61-9fa3-b7ee419d20cf/pyspark-f79d824a-78d9-44c0-bffc-29c9757b3d14 20/01/23 18:54:02.166 Thread-1 INFO ShutdownHookManager: Deleting directory /home/jenkins/workspace/spark-master-test-python-arm/python/target/22a04de0-c639-4801-aead-76fbc7eb088f/spark-9be496c1-624c-4a61-9fa3-b7ee419d20cf 20/01/23 18:54:02.172 Executor task launch worker for task 1 WARN BLAS: Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS 20/01/23 18:54:02.173 Executor task launch worker for task 1 WARN BLAS: Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS 20/01/23 18:54:02.175 Thread-1 INFO ShutdownHookManager: Deleting directory /home/jenkins/workspace/spark-master-test-python-arm/python/target/22a04de0-c639-4801-aead-76fbc7eb088f/spark-96acca51-2715-48f2-b0e9-986ac3829297 20/01/23 18:54:02.216 Thread-4 DEBUG WholeStageCodegenExec: /* 001 */ public Object generate(Object[] references) { /* 002 */ return new GeneratedIteratorForCodegenStage1(references); /* 003 */ } /* 004 */ /* 005 */ // codegenStageId=1 /* 006 */ final class GeneratedIteratorForCodegenStage1 extends org.apache.spark.sql.execution.BufferedRowIterator { /* 007 */ private Object[] references; /* 008 */ private scala.collection.Iterator[] inputs; /* 009 */ private scala.collection.Iterator rdd_input_0; /* 010 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[] rdd_mutableStateArray_0 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[3]; /* 011 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[] rdd_mutableStateArray_1 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[4]; /* 012 */ /* 013 */ public GeneratedIteratorForCodegenStage1(Object[] references) { /* 014 */ this.references = references; /* 015 */ } /* 016 */ /* 017 */ public void init(int index, scala.collection.Iterator[] inputs) { /* 018 */ partitionIndex = index; /* 019 */ this.inputs = inputs; /* 020 */ rdd_input_0 = inputs[0]; /* 021 */ rdd_mutableStateArray_0[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(3, 64); /* 022 */ rdd_mutableStateArray_1[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[0], 8); /* 023 */ rdd_mutableStateArray_0[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(rdd_mutableStateArray_0[0], 4); /* 024 */ rdd_mutableStateArray_1[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[1], 4); /* 025 */ rdd_mutableStateArray_1[2] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[1], 8); /* 026 */ rdd_mutableStateArray_0[2] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(1, 32); /* 027 */ rdd_mutableStateArray_1[3] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[2], 8); /* 028 */ /* 029 */ } /* 030 */ /* 031 */ protected void processNext() throws java.io.IOException { /* 032 */ while ( rdd_input_0.hasNext()) { /* 033 */ InternalRow rdd_row_0 = (InternalRow) rdd_input_0.next(); /* 034 */ ((org.apache.spark.sql.execution.metric.SQLMetric) references[0] /* numOutputRows */).add(1); /* 035 */ boolean rdd_isNull_1 = rdd_row_0.isNullAt(1); /* 036 */ ArrayData rdd_value_1 = rdd_isNull_1 ? /* 037 */ null : (rdd_row_0.getArray(1)); /* 038 */ rdd_mutableStateArray_0[2].reset(); /* 039 */ /* 040 */ rdd_mutableStateArray_0[2].zeroOutNullBytes(); /* 041 */ /* 042 */ if (rdd_isNull_1) { /* 043 */ rdd_mutableStateArray_0[2].setNullAt(0); /* 044 */ } else { /* 045 */ // Remember the current cursor so that we can calculate how many bytes are /* 046 */ // written later. /* 047 */ final int project_previousCursor_0 = rdd_mutableStateArray_0[2].cursor(); /* 048 */ /* 049 */ final ArrayData project_tmpInput_0 = rdd_value_1; /* 050 */ if (project_tmpInput_0 instanceof UnsafeArrayData) { /* 051 */ rdd_mutableStateArray_0[2].write((UnsafeArrayData) project_tmpInput_0); /* 052 */ } else { /* 053 */ final int project_numElements_0 = project_tmpInput_0.numElements(); /* 054 */ rdd_mutableStateArray_1[3].initialize(project_numElements_0); /* 055 */ /* 056 */ for (int project_index_0 = 0; project_index_0 < project_numElements_0; project_index_0++) { /* 057 */ if (project_tmpInput_0.isNullAt(project_index_0)) { /* 058 */ rdd_mutableStateArray_1[3].setNull8Bytes(project_index_0); /* 059 */ } else { /* 060 */ rdd_mutableStateArray_1[3].write(project_index_0, project_tmpInput_0.getUTF8String(project_index_0)); /* 061 */ } /* 062 */ /* 063 */ } /* 064 */ } /* 065 */ /* 066 */ rdd_mutableStateArray_0[2].setOffsetAndSizeFromPreviousCursor(0, project_previousCursor_0); /* 067 */ } /* 068 */ append((rdd_mutableStateArray_0[2].getRow())); /* 069 */ if (shouldStop()) return; /* 070 */ } /* 071 */ } /* 072 */ /* 073 */ } 20/01/23 18:54:02.218 Thread-4 DEBUG CodeGenerator: /* 001 */ public Object generate(Object[] references) { /* 002 */ return new GeneratedIteratorForCodegenStage1(references); /* 003 */ } /* 004 */ /* 005 */ // codegenStageId=1 /* 006 */ final class GeneratedIteratorForCodegenStage1 extends org.apache.spark.sql.execution.BufferedRowIterator { /* 007 */ private Object[] references; /* 008 */ private scala.collection.Iterator[] inputs; /* 009 */ private scala.collection.Iterator rdd_input_0; /* 010 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[] rdd_mutableStateArray_0 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[3]; /* 011 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[] rdd_mutableStateArray_1 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[4]; /* 012 */ /* 013 */ public GeneratedIteratorForCodegenStage1(Object[] references) { /* 014 */ this.references = references; /* 015 */ } /* 016 */ /* 017 */ public void init(int index, scala.collection.Iterator[] inputs) { /* 018 */ partitionIndex = index; /* 019 */ this.inputs = inputs; /* 020 */ rdd_input_0 = inputs[0]; /* 021 */ rdd_mutableStateArray_0[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(3, 64); /* 022 */ rdd_mutableStateArray_1[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[0], 8); /* 023 */ rdd_mutableStateArray_0[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(rdd_mutableStateArray_0[0], 4); /* 024 */ rdd_mutableStateArray_1[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[1], 4); /* 025 */ rdd_mutableStateArray_1[2] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[1], 8); /* 026 */ rdd_mutableStateArray_0[2] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(1, 32); /* 027 */ rdd_mutableStateArray_1[3] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[2], 8); /* 028 */ /* 029 */ } /* 030 */ /* 031 */ protected void processNext() throws java.io.IOException { /* 032 */ while ( rdd_input_0.hasNext()) { /* 033 */ InternalRow rdd_row_0 = (InternalRow) rdd_input_0.next(); /* 034 */ ((org.apache.spark.sql.execution.metric.SQLMetric) references[0] /* numOutputRows */).add(1); /* 035 */ boolean rdd_isNull_1 = rdd_row_0.isNullAt(1); /* 036 */ ArrayData rdd_value_1 = rdd_isNull_1 ? /* 037 */ null : (rdd_row_0.getArray(1)); /* 038 */ rdd_mutableStateArray_0[2].reset(); /* 039 */ /* 040 */ rdd_mutableStateArray_0[2].zeroOutNullBytes(); /* 041 */ /* 042 */ if (rdd_isNull_1) { /* 043 */ rdd_mutableStateArray_0[2].setNullAt(0); /* 044 */ } else { /* 045 */ // Remember the current cursor so that we can calculate how many bytes are /* 046 */ // written later. /* 047 */ final int project_previousCursor_0 = rdd_mutableStateArray_0[2].cursor(); /* 048 */ /* 049 */ final ArrayData project_tmpInput_0 = rdd_value_1; /* 050 */ if (project_tmpInput_0 instanceof UnsafeArrayData) { /* 051 */ rdd_mutableStateArray_0[2].write((UnsafeArrayData) project_tmpInput_0); /* 052 */ } else { /* 053 */ final int project_numElements_0 = project_tmpInput_0.numElements(); /* 054 */ rdd_mutableStateArray_1[3].initialize(project_numElements_0); /* 055 */ /* 056 */ for (int project_index_0 = 0; project_index_0 < project_numElements_0; project_index_0++) { /* 057 */ if (project_tmpInput_0.isNullAt(project_index_0)) { /* 058 */ rdd_mutableStateArray_1[3].setNull8Bytes(project_index_0); /* 059 */ } else { /* 060 */ rdd_mutableStateArray_1[3].write(project_index_0, project_tmpInput_0.getUTF8String(project_index_0)); /* 061 */ } /* 062 */ /* 063 */ } /* 064 */ } /* 065 */ /* 066 */ rdd_mutableStateArray_0[2].setOffsetAndSizeFromPreviousCursor(0, project_previousCursor_0); /* 067 */ } /* 068 */ append((rdd_mutableStateArray_0[2].getRow())); /* 069 */ if (shouldStop()) return; /* 070 */ } /* 071 */ } /* 072 */ /* 073 */ } 20/01/23 18:54:02.240 Executor task launch worker for task 1 INFO Executor: Finished task 0.0 in stage 1.0 (TID 1). 2039 bytes result sent to driver 20/01/23 18:54:02.240 Executor task launch worker for task 3 INFO Executor: Finished task 2.0 in stage 1.0 (TID 3). 2039 bytes result sent to driver 20/01/23 18:54:02.240 Executor task launch worker for task 2 INFO Executor: Finished task 1.0 in stage 1.0 (TID 2). 2082 bytes result sent to driver 20/01/23 18:54:02.241 Executor task launch worker for task 1 DEBUG ExecutorMetricsPoller: stageTCMP: (1, 0) -> 3 20/01/23 18:54:02.241 Executor task launch worker for task 2 DEBUG ExecutorMetricsPoller: stageTCMP: (1, 0) -> 2 20/01/23 18:54:02.241 Executor task launch worker for task 3 DEBUG ExecutorMetricsPoller: stageTCMP: (1, 0) -> 1 20/01/23 18:54:02.251 dispatcher-event-loop-3 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_1.0, runningTasks: 3 20/01/23 18:54:02.251 dispatcher-event-loop-3 DEBUG TaskSetManager: No tasks for locality level NO_PREF, so moving to locality level ANY 20/01/23 18:54:02.252 Executor task launch worker for task 4 INFO Executor: Finished task 3.0 in stage 1.0 (TID 4). 2039 bytes result sent to driver 20/01/23 18:54:02.252 Executor task launch worker for task 4 DEBUG ExecutorMetricsPoller: removing (1, 0) from stageTCMP 20/01/23 18:54:02.255 Thread-4 INFO CodeGenerator: Code generated in 38.305089 ms 20/01/23 18:54:02.255 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$doExecute$4$adapted 20/01/23 18:54:02.256 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$doExecute$4$adapted) is now cleaned +++ 20/01/23 18:54:02.261 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$rdd$1 20/01/23 18:54:02.261 dispatcher-event-loop-3 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_1.0, runningTasks: 2 20/01/23 18:54:02.264 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$rdd$1) is now cleaned +++ 20/01/23 18:54:02.269 dispatcher-event-loop-3 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_1.0, runningTasks: 1 20/01/23 18:54:02.269 dispatcher-event-loop-3 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_1.0, runningTasks: 0 20/01/23 18:54:02.272 task-result-getter-3 INFO TaskSetManager: Finished task 1.0 in stage 1.0 (TID 2) in 635 ms on 192.168.1.225 (executor driver) (1/4) 20/01/23 18:54:02.273 task-result-getter-1 INFO TaskSetManager: Finished task 0.0 in stage 1.0 (TID 1) in 638 ms on 192.168.1.225 (executor driver) (2/4) 20/01/23 18:54:02.273 task-result-getter-0 INFO TaskSetManager: Finished task 3.0 in stage 1.0 (TID 4) in 635 ms on 192.168.1.225 (executor driver) (3/4) 20/01/23 18:54:02.274 dag-scheduler-event-loop DEBUG DAGScheduler: ShuffleMapTask finished on driver 20/01/23 18:54:02.278 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$fit$2 20/01/23 18:54:02.279 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$fit$2) is now cleaned +++ 20/01/23 18:54:02.279 task-result-getter-2 INFO TaskSetManager: Finished task 2.0 in stage 1.0 (TID 3) in 642 ms on 192.168.1.225 (executor driver) (4/4) 20/01/23 18:54:02.280 task-result-getter-2 INFO TaskSchedulerImpl: Removed TaskSet 1.0, whose tasks have all completed, from pool 20/01/23 18:54:02.283 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$fit$4 20/01/23 18:54:02.284 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$fit$4) is now cleaned +++ 20/01/23 18:54:02.286 dag-scheduler-event-loop DEBUG DAGScheduler: ShuffleMapTask finished on driver 20/01/23 18:54:02.288 dag-scheduler-event-loop DEBUG DAGScheduler: ShuffleMapTask finished on driver 20/01/23 18:54:02.291 dag-scheduler-event-loop DEBUG DAGScheduler: ShuffleMapTask finished on driver 20/01/23 18:54:02.291 dag-scheduler-event-loop INFO DAGScheduler: ShuffleMapStage 1 (mapPartitions at ALS.scala:1342) finished in 0.703 s 20/01/23 18:54:02.292 dag-scheduler-event-loop INFO DAGScheduler: looking for newly runnable stages 20/01/23 18:54:02.293 dag-scheduler-event-loop INFO DAGScheduler: running: Set() 20/01/23 18:54:02.293 dag-scheduler-event-loop INFO DAGScheduler: waiting: Set(ShuffleMapStage 2, ResultStage 3) 20/01/23 18:54:02.294 dag-scheduler-event-loop INFO DAGScheduler: failed: Set() 20/01/23 18:54:02.294 dag-scheduler-event-loop DEBUG MapOutputTrackerMaster: Increasing epoch to 1 20/01/23 18:54:02.297 dag-scheduler-event-loop DEBUG DAGScheduler: submitStage(ShuffleMapStage 2 (name=map at ALS.scala:1585;jobs=1)) 20/01/23 18:54:02.298 dag-scheduler-event-loop DEBUG DAGScheduler: missing: List() 20/01/23 18:54:02.298 dag-scheduler-event-loop INFO DAGScheduler: Submitting ShuffleMapStage 2 (MapPartitionsRDD[17] at map at ALS.scala:1585), which has no missing parents 20/01/23 18:54:02.298 dag-scheduler-event-loop DEBUG DAGScheduler: submitMissingTasks(ShuffleMapStage 2) 20/01/23 18:54:02.301 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$reduceByKey$2 20/01/23 18:54:02.305 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$reduceByKey$2) is now cleaned +++ 20/01/23 18:54:02.312 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$fit$9 20/01/23 18:54:02.313 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$fit$9) is now cleaned +++ 20/01/23 18:54:02.314 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$fit$9 20/01/23 18:54:02.314 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$fit$9) is now cleaned +++ 20/01/23 18:54:02.316 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$fit$11 20/01/23 18:54:02.317 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$fit$11) is now cleaned +++ 20/01/23 18:54:02.323 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$count$1$adapted 20/01/23 18:54:02.325 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$count$1$adapted) is now cleaned +++ 20/01/23 18:54:02.326 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$runJob$5 20/01/23 18:54:02.329 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$runJob$5) is now cleaned +++ 20/01/23 18:54:02.329 Thread-4 INFO SparkContext: Starting job: count at CountVectorizer.scala:233 20/01/23 18:54:02.334 dag-scheduler-event-loop DEBUG SortShuffleManager: Can't use serialized shuffle for shuffle 0 because the serializer, org.apache.spark.serializer.JavaSerializer, does not support object relocation 20/01/23 18:54:02.351 dag-scheduler-event-loop INFO DAGScheduler: Registering RDD 19 (flatMap at CountVectorizer.scala:212) as input to shuffle 0 20/01/23 18:54:02.388 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(7) 20/01/23 18:54:02.389 Executor task launch worker for task 4 INFO PythonRunner: Times: total = 68, boot = 16, init = 52, finish = 0 20/01/23 18:54:02.389 Executor task launch worker for task 2 INFO PythonRunner: Times: total = 75, boot = 22, init = 53, finish = 0 20/01/23 18:54:02.389 Executor task launch worker for task 1 INFO PythonRunner: Times: total = 98, boot = 37, init = 61, finish = 0 20/01/23 18:54:02.390 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 7 20/01/23 18:54:02.391 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 7 20/01/23 18:54:02.391 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(13) 20/01/23 18:54:02.392 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 13 20/01/23 18:54:02.392 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 13 20/01/23 18:54:02.392 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(35) 20/01/23 18:54:02.392 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 35 20/01/23 18:54:02.392 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 35 20/01/23 18:54:02.392 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(31) 20/01/23 18:54:02.392 Executor task launch worker for task 3 INFO PythonRunner: Times: total = 76, boot = 28, init = 48, finish = 0 20/01/23 18:54:02.392 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 31 20/01/23 18:54:02.392 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 31 20/01/23 18:54:02.392 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(8) 20/01/23 18:54:02.392 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 8 20/01/23 18:54:02.393 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 8 20/01/23 18:54:02.393 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(15) 20/01/23 18:54:02.393 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 15 20/01/23 18:54:02.393 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 15 20/01/23 18:54:02.393 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(21) 20/01/23 18:54:02.393 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 21 20/01/23 18:54:02.393 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 21 20/01/23 18:54:02.393 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(1) 20/01/23 18:54:02.393 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 1 20/01/23 18:54:02.393 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 1 20/01/23 18:54:02.393 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(28) 20/01/23 18:54:02.393 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 28 20/01/23 18:54:02.393 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 28 20/01/23 18:54:02.393 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(32) 20/01/23 18:54:02.393 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 32 20/01/23 18:54:02.393 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 32 20/01/23 18:54:02.393 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(23) 20/01/23 18:54:02.393 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 23 20/01/23 18:54:02.393 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 23 20/01/23 18:54:02.393 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(14) 20/01/23 18:54:02.393 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 14 20/01/23 18:54:02.393 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 14 20/01/23 18:54:02.394 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(24) 20/01/23 18:54:02.394 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 24 20/01/23 18:54:02.394 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 24 20/01/23 18:54:02.394 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(10) 20/01/23 18:54:02.394 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 10 20/01/23 18:54:02.394 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 10 20/01/23 18:54:02.394 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(22) 20/01/23 18:54:02.394 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 22 20/01/23 18:54:02.394 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 22 20/01/23 18:54:02.394 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(9) 20/01/23 18:54:02.394 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 9 20/01/23 18:54:02.394 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 9 20/01/23 18:54:02.394 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(3) 20/01/23 18:54:02.394 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 3 20/01/23 18:54:02.394 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 3 20/01/23 18:54:02.394 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(19) 20/01/23 18:54:02.394 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 19 20/01/23 18:54:02.394 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 19 20/01/23 18:54:02.394 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(4) 20/01/23 18:54:02.394 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 4 20/01/23 18:54:02.394 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 4 20/01/23 18:54:02.394 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(17) 20/01/23 18:54:02.394 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 17 20/01/23 18:54:02.395 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 17 20/01/23 18:54:02.395 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(29) 20/01/23 18:54:02.395 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 29 20/01/23 18:54:02.395 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 29 20/01/23 18:54:02.395 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(6) 20/01/23 18:54:02.395 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 6 20/01/23 18:54:02.395 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 6 20/01/23 18:54:02.395 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(16) 20/01/23 18:54:02.395 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 16 20/01/23 18:54:02.395 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 16 20/01/23 18:54:02.395 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(34) 20/01/23 18:54:02.395 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 34 20/01/23 18:54:02.395 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 34 20/01/23 18:54:02.395 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanBroadcast(0) 20/01/23 18:54:02.396 Spark Context Cleaner DEBUG ContextCleaner: Cleaning broadcast 0 20/01/23 18:54:02.396 Spark Context Cleaner DEBUG TorrentBroadcast: Unpersisting TorrentBroadcast 0 20/01/23 18:54:02.400 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_2 stored as values in memory (estimated size 28.8 KiB, free 366.2 MiB) 20/01/23 18:54:02.400 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_2 locally took 1 ms 20/01/23 18:54:02.400 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_2 without replication took 1 ms 20/01/23 18:54:02.401 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_2_piece0 stored as bytes in memory (estimated size 13.9 KiB, free 366.2 MiB) 20/01/23 18:54:02.402 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added broadcast_2_piece0 in memory on 192.168.1.225:35863 (size: 13.9 KiB, free: 366.3 MiB) 20/01/23 18:54:02.403 dag-scheduler-event-loop DEBUG BlockManagerMaster: Updated info of block broadcast_2_piece0 20/01/23 18:54:02.403 dag-scheduler-event-loop DEBUG BlockManager: Told master about block broadcast_2_piece0 20/01/23 18:54:02.403 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_2_piece0 locally took 1 ms 20/01/23 18:54:02.403 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_2_piece0 without replication took 2 ms 20/01/23 18:54:02.403 dag-scheduler-event-loop INFO SparkContext: Created broadcast 2 from broadcast at DAGScheduler.scala:1200 20/01/23 18:54:02.404 dag-scheduler-event-loop INFO DAGScheduler: Submitting 4 missing tasks from ShuffleMapStage 2 (MapPartitionsRDD[17] at map at ALS.scala:1585) (first 15 tasks are for partitions Vector(0, 1, 2, 3)) 20/01/23 18:54:02.404 dag-scheduler-event-loop INFO TaskSchedulerImpl: Adding task set 2.0 with 4 tasks 20/01/23 18:54:02.405 dag-scheduler-event-loop DEBUG TaskSetManager: Epoch for TaskSet 2.0: 1 20/01/23 18:54:02.406 dag-scheduler-event-loop DEBUG TaskSetManager: Adding pending tasks took 1 ms 20/01/23 18:54:02.407 dag-scheduler-event-loop DEBUG TaskSetManager: Valid locality levels for TaskSet 2.0: NODE_LOCAL, ANY 20/01/23 18:54:02.408 dispatcher-event-loop-1 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_2.0, runningTasks: 0 20/01/23 18:54:02.410 dispatcher-event-loop-1 INFO TaskSetManager: Starting task 0.0 in stage 2.0 (TID 5, 192.168.1.225, executor driver, partition 0, NODE_LOCAL, 7132 bytes) 20/01/23 18:54:02.411 dispatcher-event-loop-1 INFO TaskSetManager: Starting task 1.0 in stage 2.0 (TID 6, 192.168.1.225, executor driver, partition 1, NODE_LOCAL, 7132 bytes) 20/01/23 18:54:02.411 dispatcher-event-loop-1 INFO TaskSetManager: Starting task 2.0 in stage 2.0 (TID 7, 192.168.1.225, executor driver, partition 2, NODE_LOCAL, 7132 bytes) 20/01/23 18:54:02.412 dispatcher-event-loop-1 INFO TaskSetManager: Starting task 3.0 in stage 2.0 (TID 8, 192.168.1.225, executor driver, partition 3, NODE_LOCAL, 7132 bytes) 20/01/23 18:54:02.412 Executor task launch worker for task 5 INFO Executor: Running task 0.0 in stage 2.0 (TID 5) 20/01/23 18:54:02.412 Executor task launch worker for task 6 INFO Executor: Running task 1.0 in stage 2.0 (TID 6) 20/01/23 18:54:02.412 Executor task launch worker for task 7 INFO Executor: Running task 2.0 in stage 2.0 (TID 7) 20/01/23 18:54:02.412 Executor task launch worker for task 8 INFO Executor: Running task 3.0 in stage 2.0 (TID 8) 20/01/23 18:54:02.413 Executor task launch worker for task 7 DEBUG ExecutorMetricsPoller: stageTCMP: (2, 0) -> 1 20/01/23 18:54:02.413 Executor task launch worker for task 8 DEBUG ExecutorMetricsPoller: stageTCMP: (2, 0) -> 2 20/01/23 18:54:02.414 Executor task launch worker for task 7 DEBUG BlockManager: Getting local block broadcast_2 20/01/23 18:54:02.414 Executor task launch worker for task 7 DEBUG BlockManager: Level for block broadcast_2 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:02.416 Executor task launch worker for task 6 DEBUG ExecutorMetricsPoller: stageTCMP: (2, 0) -> 3 20/01/23 18:54:02.421 Executor task launch worker for task 5 DEBUG ExecutorMetricsPoller: stageTCMP: (2, 0) -> 4 20/01/23 18:54:02.429 Executor task launch worker for task 8 DEBUG BlockManager: Getting local block rdd_16_3 20/01/23 18:54:02.430 Executor task launch worker for task 6 DEBUG BlockManager: Getting local block rdd_16_1 20/01/23 18:54:02.430 Executor task launch worker for task 6 DEBUG BlockManager: Block rdd_16_1 was not found 20/01/23 18:54:02.430 Executor task launch worker for task 8 DEBUG BlockManager: Block rdd_16_3 was not found 20/01/23 18:54:02.430 block-manager-slave-async-thread-pool-0 DEBUG BlockManagerSlaveEndpoint: removing broadcast 0 20/01/23 18:54:02.431 Executor task launch worker for task 1 DEBUG TaskMemoryManager: Task 1 release 0.0 B from org.apache.spark.util.collection.ExternalSorter@4d45db93 20/01/23 18:54:02.431 Executor task launch worker for task 6 DEBUG BlockManager: Getting remote block rdd_16_1 20/01/23 18:54:02.431 block-manager-slave-async-thread-pool-0 DEBUG BlockManager: Removing broadcast 0 20/01/23 18:54:02.433 Executor task launch worker for task 7 DEBUG BlockManager: Getting local block rdd_16_2 20/01/23 18:54:02.433 Executor task launch worker for task 6 DEBUG BlockManager: Block rdd_16_1 is unknown by block manager master 20/01/23 18:54:02.431 Executor task launch worker for task 3 DEBUG TaskMemoryManager: Task 3 release 0.0 B from org.apache.spark.util.collection.ExternalSorter@50bc363c 20/01/23 18:54:02.434 Executor task launch worker for task 7 DEBUG BlockManager: Block rdd_16_2 was not found 20/01/23 18:54:02.437 Executor task launch worker for task 7 DEBUG BlockManager: Getting remote block rdd_16_2 20/01/23 18:54:02.437 Executor task launch worker for task 8 DEBUG BlockManager: Getting remote block rdd_16_3 20/01/23 18:54:02.435 Executor task launch worker for task 6 DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 1, partitions 1-2 20/01/23 18:54:02.438 Executor task launch worker for task 8 DEBUG BlockManager: Block rdd_16_3 is unknown by block manager master 20/01/23 18:54:02.431 Executor task launch worker for task 4 DEBUG TaskMemoryManager: Task 4 release 0.0 B from org.apache.spark.util.collection.ExternalSorter@528868e4 20/01/23 18:54:02.432 Executor task launch worker for task 2 DEBUG TaskMemoryManager: Task 2 release 0.0 B from org.apache.spark.util.collection.ExternalSorter@38934bec 20/01/23 18:54:02.438 Executor task launch worker for task 8 DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 1, partitions 3-4 20/01/23 18:54:02.438 Executor task launch worker for task 7 DEBUG BlockManager: Block rdd_16_2 is unknown by block manager master 20/01/23 18:54:02.439 Executor task launch worker for task 7 DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 1, partitions 2-3 20/01/23 18:54:02.441 Executor task launch worker for task 5 DEBUG BlockManager: Getting local block rdd_16_0 20/01/23 18:54:02.441 Executor task launch worker for task 5 DEBUG BlockManager: Block rdd_16_0 was not found 20/01/23 18:54:02.441 Executor task launch worker for task 5 DEBUG BlockManager: Getting remote block rdd_16_0 20/01/23 18:54:02.441 Executor task launch worker for task 5 DEBUG BlockManager: Block rdd_16_0 is unknown by block manager master 20/01/23 18:54:02.441 Executor task launch worker for task 5 DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 1, partitions 0-1 20/01/23 18:54:02.441 Executor task launch worker for task 1 INFO Executor: Finished task 0.0 in stage 1.0 (TID 1). 2168 bytes result sent to driver 20/01/23 18:54:02.442 Executor task launch worker for task 1 DEBUG ExecutorMetricsPoller: stageTCMP: (1, 0) -> 3 20/01/23 18:54:02.443 Executor task launch worker for task 4 INFO Executor: Finished task 3.0 in stage 1.0 (TID 4). 2125 bytes result sent to driver 20/01/23 18:54:02.443 Executor task launch worker for task 4 DEBUG ExecutorMetricsPoller: stageTCMP: (1, 0) -> 2 20/01/23 18:54:02.443 block-manager-slave-async-thread-pool-0 DEBUG BlockManager: Removing block broadcast_0_piece0 20/01/23 18:54:02.444 Executor task launch worker for task 3 INFO Executor: Finished task 2.0 in stage 1.0 (TID 3). 2125 bytes result sent to driver 20/01/23 18:54:02.444 Executor task launch worker for task 3 DEBUG ExecutorMetricsPoller: stageTCMP: (1, 0) -> 1 20/01/23 18:54:02.441 Executor task launch worker for task 2 INFO Executor: Finished task 1.0 in stage 1.0 (TID 2). 2125 bytes result sent to driver 20/01/23 18:54:02.444 Executor task launch worker for task 2 DEBUG ExecutorMetricsPoller: removing (1, 0) from stageTCMP 20/01/23 18:54:02.458 dispatcher-event-loop-1 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_1.0, runningTasks: 3 20/01/23 18:54:02.458 dispatcher-event-loop-1 DEBUG TaskSetManager: No tasks for locality level NO_PREF, so moving to locality level ANY 20/01/23 18:54:02.458 block-manager-slave-async-thread-pool-0 DEBUG MemoryStore: Block broadcast_0_piece0 of size 7123 dropped from memory (free 384040576) 20/01/23 18:54:02.461 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Removed broadcast_0_piece0 on 192.168.1.225:45461 in memory (size: 7.0 KiB, free: 366.3 MiB) 20/01/23 18:54:02.475 dispatcher-event-loop-1 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_1.0, runningTasks: 2 20/01/23 18:54:02.478 dispatcher-event-loop-1 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_1.0, runningTasks: 1 20/01/23 18:54:02.479 task-result-getter-1 INFO TaskSetManager: Finished task 0.0 in stage 1.0 (TID 1) in 764 ms on 192.168.1.225 (executor driver) (1/4) 20/01/23 18:54:02.479 block-manager-slave-async-thread-pool-0 DEBUG BlockManagerMaster: Updated info of block broadcast_0_piece0 20/01/23 18:54:02.479 block-manager-slave-async-thread-pool-0 DEBUG BlockManager: Told master about block broadcast_0_piece0 20/01/23 18:54:02.479 dispatcher-event-loop-1 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_1.0, runningTasks: 0 20/01/23 18:54:02.480 task-result-getter-0 INFO TaskSetManager: Finished task 1.0 in stage 1.0 (TID 2) in 764 ms on 192.168.1.225 (executor driver) (2/4) 20/01/23 18:54:02.480 block-manager-slave-async-thread-pool-0 DEBUG BlockManager: Removing block broadcast_0 20/01/23 18:54:02.481 block-manager-slave-async-thread-pool-0 DEBUG MemoryStore: Block broadcast_0 of size 15280 dropped from memory (free 384055856) 20/01/23 18:54:02.482 dag-scheduler-event-loop DEBUG DAGScheduler: ShuffleMapTask finished on driver 20/01/23 18:54:02.483 task-result-getter-3 INFO TaskSetManager: Finished task 2.0 in stage 1.0 (TID 3) in 766 ms on 192.168.1.225 (executor driver) (3/4) 20/01/23 18:54:02.484 dag-scheduler-event-loop DEBUG DAGScheduler: ShuffleMapTask finished on driver 20/01/23 18:54:02.485 dag-scheduler-event-loop DEBUG DAGScheduler: ShuffleMapTask finished on driver 20/01/23 18:54:02.486 task-result-getter-2 INFO TaskSetManager: Finished task 3.0 in stage 1.0 (TID 4) in 769 ms on 192.168.1.225 (executor driver) (4/4) 20/01/23 18:54:02.487 task-result-getter-2 INFO TaskSchedulerImpl: Removed TaskSet 1.0, whose tasks have all completed, from pool 20/01/23 18:54:02.488 dag-scheduler-event-loop DEBUG DAGScheduler: ShuffleMapTask finished on driver 20/01/23 18:54:02.488 dag-scheduler-event-loop INFO DAGScheduler: ShuffleMapStage 1 (map at ClusteringEvaluator.scala:571) finished in 0.871 s 20/01/23 18:54:02.489 dag-scheduler-event-loop INFO DAGScheduler: looking for newly runnable stages 20/01/23 18:54:02.490 dag-scheduler-event-loop INFO DAGScheduler: running: Set() 20/01/23 18:54:02.490 dag-scheduler-event-loop INFO DAGScheduler: waiting: Set(ResultStage 2) 20/01/23 18:54:02.491 dag-scheduler-event-loop INFO DAGScheduler: failed: Set() 20/01/23 18:54:02.491 dag-scheduler-event-loop DEBUG MapOutputTrackerMaster: Increasing epoch to 1 20/01/23 18:54:02.494 dag-scheduler-event-loop DEBUG DAGScheduler: submitStage(ResultStage 2 (name=collectAsMap at ClusteringEvaluator.scala:585;jobs=1)) 20/01/23 18:54:02.495 dag-scheduler-event-loop DEBUG DAGScheduler: missing: List() 20/01/23 18:54:02.495 dag-scheduler-event-loop INFO DAGScheduler: Submitting ResultStage 2 (ShuffledRDD[12] at aggregateByKey at ClusteringEvaluator.scala:572), which has no missing parents 20/01/23 18:54:02.495 dag-scheduler-event-loop DEBUG DAGScheduler: submitMissingTasks(ResultStage 2) 20/01/23 18:54:02.498 block-manager-slave-async-thread-pool-2 DEBUG BlockManagerSlaveEndpoint: Done removing broadcast 0, response is 0 20/01/23 18:54:02.510 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_2 stored as values in memory (estimated size 4.6 KiB, free 366.3 MiB) 20/01/23 18:54:02.511 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_2 locally took 1 ms 20/01/23 18:54:02.511 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_2 without replication took 1 ms 20/01/23 18:54:02.519 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_2_piece0 stored as bytes in memory (estimated size 2.6 KiB, free 366.3 MiB) 20/01/23 18:54:02.520 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added broadcast_2_piece0 in memory on 192.168.1.225:45461 (size: 2.6 KiB, free: 366.3 MiB) 20/01/23 18:54:02.523 dag-scheduler-event-loop DEBUG BlockManagerMaster: Updated info of block broadcast_2_piece0 20/01/23 18:54:02.523 dag-scheduler-event-loop DEBUG BlockManager: Told master about block broadcast_2_piece0 20/01/23 18:54:02.523 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_2_piece0 locally took 4 ms 20/01/23 18:54:02.523 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_2_piece0 without replication took 5 ms 20/01/23 18:54:02.524 dag-scheduler-event-loop INFO SparkContext: Created broadcast 2 from broadcast at DAGScheduler.scala:1200 20/01/23 18:54:02.525 dag-scheduler-event-loop INFO DAGScheduler: Submitting 4 missing tasks from ResultStage 2 (ShuffledRDD[12] at aggregateByKey at ClusteringEvaluator.scala:572) (first 15 tasks are for partitions Vector(0, 1, 2, 3)) 20/01/23 18:54:02.525 dag-scheduler-event-loop INFO TaskSchedulerImpl: Adding task set 2.0 with 4 tasks 20/01/23 18:54:02.525 dag-scheduler-event-loop DEBUG TaskSetManager: Epoch for TaskSet 2.0: 1 20/01/23 18:54:02.526 block-manager-slave-async-thread-pool-2 DEBUG BlockManagerSlaveEndpoint: Sent response: 0 to 192.168.1.225:33699 20/01/23 18:54:02.530 dag-scheduler-event-loop DEBUG TaskSetManager: Adding pending tasks took 4 ms 20/01/23 18:54:02.531 dag-scheduler-event-loop DEBUG TaskSetManager: Valid locality levels for TaskSet 2.0: NODE_LOCAL, NO_PREF, ANY 20/01/23 18:54:02.534 dispatcher-event-loop-3 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_2.0, runningTasks: 0 20/01/23 18:54:02.536 dispatcher-event-loop-3 INFO TaskSetManager: Starting task 0.0 in stage 2.0 (TID 5, 192.168.1.225, executor driver, partition 0, NODE_LOCAL, 7143 bytes) 20/01/23 18:54:02.536 dispatcher-event-loop-3 DEBUG TaskSetManager: No tasks for locality level NODE_LOCAL, so moving to locality level NO_PREF 20/01/23 18:54:02.537 dispatcher-event-loop-3 DEBUG TaskSetManager: Moving to ANY after waiting for 0ms 20/01/23 18:54:02.537 dispatcher-event-loop-3 INFO TaskSetManager: Starting task 1.0 in stage 2.0 (TID 6, 192.168.1.225, executor driver, partition 1, PROCESS_LOCAL, 7143 bytes) 20/01/23 18:54:02.537 dispatcher-event-loop-3 INFO TaskSetManager: Starting task 2.0 in stage 2.0 (TID 7, 192.168.1.225, executor driver, partition 2, PROCESS_LOCAL, 7143 bytes) 20/01/23 18:54:02.540 dispatcher-event-loop-3 INFO TaskSetManager: Starting task 3.0 in stage 2.0 (TID 8, 192.168.1.225, executor driver, partition 3, PROCESS_LOCAL, 7143 bytes) 20/01/23 18:54:02.541 Executor task launch worker for task 5 INFO Executor: Running task 0.0 in stage 2.0 (TID 5) 20/01/23 18:54:02.542 Executor task launch worker for task 5 DEBUG ExecutorMetricsPoller: stageTCMP: (2, 0) -> 1 20/01/23 18:54:02.544 Executor task launch worker for task 5 DEBUG BlockManager: Getting local block broadcast_2 20/01/23 18:54:02.544 Executor task launch worker for task 5 DEBUG BlockManager: Level for block broadcast_2 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:02.550 Executor task launch worker for task 6 INFO Executor: Running task 1.0 in stage 2.0 (TID 6) 20/01/23 18:54:02.551 Executor task launch worker for task 6 DEBUG ExecutorMetricsPoller: stageTCMP: (2, 0) -> 2 20/01/23 18:54:02.547 Executor task launch worker for task 5 DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 0, partitions 0-1 20/01/23 18:54:02.554 Executor task launch worker for task 6 DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 0, partitions 1-2 20/01/23 18:54:02.554 Spark Context Cleaner DEBUG ContextCleaner: Cleaned broadcast 0 20/01/23 18:54:02.554 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(37) 20/01/23 18:54:02.554 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 37 20/01/23 18:54:02.554 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 37 20/01/23 18:54:02.555 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(30) 20/01/23 18:54:02.555 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 30 20/01/23 18:54:02.555 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 30 20/01/23 18:54:02.555 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(5) 20/01/23 18:54:02.555 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 5 20/01/23 18:54:02.555 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 5 20/01/23 18:54:02.555 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(18) 20/01/23 18:54:02.555 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 18 20/01/23 18:54:02.555 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 18 20/01/23 18:54:02.555 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(36) 20/01/23 18:54:02.555 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 36 20/01/23 18:54:02.555 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 36 20/01/23 18:54:02.555 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(12) 20/01/23 18:54:02.555 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 12 20/01/23 18:54:02.555 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 12 20/01/23 18:54:02.555 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(26) 20/01/23 18:54:02.555 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 26 20/01/23 18:54:02.555 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 26 20/01/23 18:54:02.555 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(20) 20/01/23 18:54:02.555 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 20 20/01/23 18:54:02.555 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 20 20/01/23 18:54:02.555 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(11) 20/01/23 18:54:02.556 Executor task launch worker for task 7 INFO Executor: Running task 2.0 in stage 2.0 (TID 7) 20/01/23 18:54:02.556 Executor task launch worker for task 8 INFO Executor: Running task 3.0 in stage 2.0 (TID 8) 20/01/23 18:54:02.565 Executor task launch worker for task 8 DEBUG ExecutorMetricsPoller: stageTCMP: (2, 0) -> 3 20/01/23 18:54:02.568 Executor task launch worker for task 8 DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 0, partitions 3-4 20/01/23 18:54:02.568 Executor task launch worker for task 7 DEBUG ExecutorMetricsPoller: stageTCMP: (2, 0) -> 4 20/01/23 18:54:02.571 Executor task launch worker for task 7 DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 0, partitions 2-3 20/01/23 18:54:02.573 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 11 20/01/23 18:54:02.573 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 11 20/01/23 18:54:02.573 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(25) 20/01/23 18:54:02.573 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 25 20/01/23 18:54:02.573 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 25 20/01/23 18:54:02.573 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(2) 20/01/23 18:54:02.573 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 2 20/01/23 18:54:02.573 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 2 20/01/23 18:54:02.574 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(27) 20/01/23 18:54:02.574 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 27 20/01/23 18:54:02.574 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 27 20/01/23 18:54:02.574 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(33) 20/01/23 18:54:02.574 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 33 20/01/23 18:54:02.574 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 33 20/01/23 18:54:02.574 Executor task launch worker for task 7 DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 20/01/23 18:54:02.574 Executor task launch worker for task 5 DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 20/01/23 18:54:02.578 Executor task launch worker for task 7 INFO ShuffleBlockFetcherIterator: Getting 0 (0.0 B) non-empty blocks including 0 (0.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 20/01/23 18:54:02.581 Executor task launch worker for task 7 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 8 ms 20/01/23 18:54:02.581 Executor task launch worker for task 8 DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 20/01/23 18:54:02.581 Executor task launch worker for task 7 DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: 20/01/23 18:54:02.581 Executor task launch worker for task 8 INFO ShuffleBlockFetcherIterator: Getting 0 (0.0 B) non-empty blocks including 0 (0.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 20/01/23 18:54:02.581 Executor task launch worker for task 8 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 20/01/23 18:54:02.581 Executor task launch worker for task 8 DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: 20/01/23 18:54:02.582 Executor task launch worker for task 6 DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 20/01/23 18:54:02.582 Executor task launch worker for task 8 DEBUG ShuffleBlockFetcherIterator: Got local blocks in 1 ms 20/01/23 18:54:02.582 Executor task launch worker for task 7 DEBUG ShuffleBlockFetcherIterator: Got local blocks in 10 ms 20/01/23 18:54:02.586 Executor task launch worker for task 5 INFO ShuffleBlockFetcherIterator: Getting 4 (1370.0 B) non-empty blocks including 4 (1370.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 20/01/23 18:54:02.586 Executor task launch worker for task 5 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 12 ms 20/01/23 18:54:02.586 Executor task launch worker for task 5 DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_0_1_0,0), (shuffle_0_2_0,1), (shuffle_0_3_0,2), (shuffle_0_4_0,3) 20/01/23 18:54:02.586 Executor task launch worker for task 6 INFO ShuffleBlockFetcherIterator: Getting 0 (0.0 B) non-empty blocks including 0 (0.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 20/01/23 18:54:02.588 Executor task launch worker for task 6 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 5 ms 20/01/23 18:54:02.588 Executor task launch worker for task 6 DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: 20/01/23 18:54:02.588 Executor task launch worker for task 6 DEBUG ShuffleBlockFetcherIterator: Got local blocks in 6 ms 20/01/23 18:54:02.606 Executor task launch worker for task 5 DEBUG ShuffleBlockFetcherIterator: Got local blocks in 32 ms 20/01/23 18:54:02.617 Executor task launch worker for task 6 DEBUG TaskMemoryManager: Task 6 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@7327e90c 20/01/23 18:54:02.617 Executor task launch worker for task 8 DEBUG TaskMemoryManager: Task 8 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@5607304d 20/01/23 18:54:02.618 Executor task launch worker for task 6 INFO Executor: Finished task 1.0 in stage 2.0 (TID 6). 1263 bytes result sent to driver 20/01/23 18:54:02.618 Executor task launch worker for task 6 DEBUG ExecutorMetricsPoller: stageTCMP: (2, 0) -> 3 20/01/23 18:54:02.617 Executor task launch worker for task 7 DEBUG TaskMemoryManager: Task 7 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@761d3dac 20/01/23 18:54:02.619 dispatcher-event-loop-2 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_2.0, runningTasks: 3 20/01/23 18:54:02.620 task-result-getter-1 INFO TaskSetManager: Finished task 1.0 in stage 2.0 (TID 6) in 83 ms on 192.168.1.225 (executor driver) (1/4) 20/01/23 18:54:02.620 Executor task launch worker for task 7 INFO Executor: Finished task 2.0 in stage 2.0 (TID 7). 1263 bytes result sent to driver 20/01/23 18:54:02.620 Executor task launch worker for task 7 DEBUG ExecutorMetricsPoller: stageTCMP: (2, 0) -> 2 20/01/23 18:54:02.620 dispatcher-event-loop-3 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_2.0, runningTasks: 2 20/01/23 18:54:02.621 task-result-getter-0 INFO TaskSetManager: Finished task 2.0 in stage 2.0 (TID 7) in 84 ms on 192.168.1.225 (executor driver) (2/4) 20/01/23 18:54:02.619 Executor task launch worker for task 8 INFO Executor: Finished task 3.0 in stage 2.0 (TID 8). 1263 bytes result sent to driver 20/01/23 18:54:02.623 dispatcher-event-loop-1 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_2.0, runningTasks: 1 20/01/23 18:54:02.623 Executor task launch worker for task 8 DEBUG ExecutorMetricsPoller: stageTCMP: (2, 0) -> 1 20/01/23 18:54:02.624 task-result-getter-3 INFO TaskSetManager: Finished task 3.0 in stage 2.0 (TID 8) in 86 ms on 192.168.1.225 (executor driver) (3/4) 20/01/23 18:54:02.635 Executor task launch worker for task 5 DEBUG TaskMemoryManager: Task 5 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@5dbd4d90 20/01/23 18:54:02.636 Executor task launch worker for task 5 INFO Executor: Finished task 0.0 in stage 2.0 (TID 5). 1711 bytes result sent to driver 20/01/23 18:54:02.636 Executor task launch worker for task 5 DEBUG ExecutorMetricsPoller: removing (2, 0) from stageTCMP 20/01/23 18:54:02.637 dispatcher-event-loop-0 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_2.0, runningTasks: 0 20/01/23 18:54:02.638 task-result-getter-2 INFO TaskSetManager: Finished task 0.0 in stage 2.0 (TID 5) in 103 ms on 192.168.1.225 (executor driver) (4/4) 20/01/23 18:54:02.638 task-result-getter-2 INFO TaskSchedulerImpl: Removed TaskSet 2.0, whose tasks have all completed, from pool 20/01/23 18:54:02.639 dag-scheduler-event-loop INFO DAGScheduler: ResultStage 2 (collectAsMap at ClusteringEvaluator.scala:585) finished in 0.130 s 20/01/23 18:54:02.639 dag-scheduler-event-loop DEBUG DAGScheduler: After removal of stage 2, remaining stages = 1 20/01/23 18:54:02.639 dag-scheduler-event-loop DEBUG DAGScheduler: After removal of stage 1, remaining stages = 0 20/01/23 18:54:02.639 dag-scheduler-event-loop INFO DAGScheduler: Job 1 is finished. Cancelling potential speculative or zombie tasks for this job 20/01/23 18:54:02.639 dag-scheduler-event-loop INFO TaskSchedulerImpl: Killing all running tasks in stage 2: Stage finished 20/01/23 18:54:02.641 Thread-4 INFO DAGScheduler: Job 1 finished: collectAsMap at ClusteringEvaluator.scala:585, took 1.052550 s 20/01/23 18:54:02.646 Thread-4 INFO MemoryStore: Block broadcast_3 stored as values in memory (estimated size 400.0 B, free 366.3 MiB) 20/01/23 18:54:02.647 Thread-4 DEBUG BlockManager: Put block broadcast_3 locally took 3 ms 20/01/23 18:54:02.647 Thread-4 DEBUG BlockManager: Putting block broadcast_3 without replication took 3 ms 20/01/23 18:54:02.654 Thread-4 INFO MemoryStore: Block broadcast_3_piece0 stored as bytes in memory (estimated size 484.0 B, free 366.3 MiB) 20/01/23 18:54:02.654 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added broadcast_3_piece0 in memory on 192.168.1.225:45461 (size: 484.0 B, free: 366.3 MiB) 20/01/23 18:54:02.655 dag-scheduler-event-loop INFO DAGScheduler: Got job 2 (count at CountVectorizer.scala:233) with 4 output partitions 20/01/23 18:54:02.655 dag-scheduler-event-loop INFO DAGScheduler: Final stage: ResultStage 3 (count at CountVectorizer.scala:233) 20/01/23 18:54:02.655 dag-scheduler-event-loop INFO DAGScheduler: Parents of final stage: List(ShuffleMapStage 2) 20/01/23 18:54:02.655 Thread-4 DEBUG BlockManagerMaster: Updated info of block broadcast_3_piece0 20/01/23 18:54:02.655 Thread-4 DEBUG BlockManager: Told master about block broadcast_3_piece0 20/01/23 18:54:02.655 Thread-4 DEBUG BlockManager: Put block broadcast_3_piece0 locally took 1 ms 20/01/23 18:54:02.655 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(9) 20/01/23 18:54:02.655 Thread-4 DEBUG BlockManager: Putting block broadcast_3_piece0 without replication took 1 ms 20/01/23 18:54:02.656 Thread-4 INFO SparkContext: Created broadcast 3 from broadcast at ClusteringEvaluator.scala:644 20/01/23 18:54:02.656 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 9 20/01/23 18:54:02.657 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 9 20/01/23 18:54:02.657 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(17) 20/01/23 18:54:02.657 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 17 20/01/23 18:54:02.657 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 17 20/01/23 18:54:02.657 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(19) 20/01/23 18:54:02.657 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 19 20/01/23 18:54:02.657 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 19 20/01/23 18:54:02.657 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(23) 20/01/23 18:54:02.658 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 23 20/01/23 18:54:02.658 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 23 20/01/23 18:54:02.658 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(4) 20/01/23 18:54:02.658 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 4 20/01/23 18:54:02.658 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 4 20/01/23 18:54:02.658 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(7) 20/01/23 18:54:02.658 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 7 20/01/23 18:54:02.658 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 7 20/01/23 18:54:02.658 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanBroadcast(3) 20/01/23 18:54:02.658 Spark Context Cleaner DEBUG ContextCleaner: Cleaning broadcast 3 20/01/23 18:54:02.659 Spark Context Cleaner DEBUG TorrentBroadcast: Unpersisting TorrentBroadcast 3 20/01/23 18:54:02.661 dag-scheduler-event-loop INFO DAGScheduler: Missing parents: List(ShuffleMapStage 2) 20/01/23 18:54:02.664 dag-scheduler-event-loop DEBUG DAGScheduler: submitStage(ResultStage 3 (name=count at CountVectorizer.scala:233;jobs=2)) 20/01/23 18:54:02.664 dag-scheduler-event-loop DEBUG DAGScheduler: missing: List(ShuffleMapStage 2) 20/01/23 18:54:02.665 dag-scheduler-event-loop DEBUG DAGScheduler: submitStage(ShuffleMapStage 2 (name=flatMap at CountVectorizer.scala:212;jobs=2)) 20/01/23 18:54:02.666 dag-scheduler-event-loop DEBUG DAGScheduler: missing: List() 20/01/23 18:54:02.666 dag-scheduler-event-loop INFO DAGScheduler: Submitting ShuffleMapStage 2 (MapPartitionsRDD[19] at flatMap at CountVectorizer.scala:212), which has no missing parents 20/01/23 18:54:02.666 dag-scheduler-event-loop DEBUG DAGScheduler: submitMissingTasks(ShuffleMapStage 2) 20/01/23 18:54:02.730 block-manager-slave-async-thread-pool-0 DEBUG BlockManagerSlaveEndpoint: removing broadcast 3 20/01/23 18:54:02.731 block-manager-slave-async-thread-pool-0 DEBUG BlockManager: Removing broadcast 3 20/01/23 18:54:02.735 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(10) 20/01/23 18:54:02.736 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 10 20/01/23 18:54:02.737 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 10 20/01/23 18:54:02.738 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(12) 20/01/23 18:54:02.738 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 12 20/01/23 18:54:02.738 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 12 20/01/23 18:54:02.738 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(14) 20/01/23 18:54:02.738 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 14 20/01/23 18:54:02.738 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 14 20/01/23 18:54:02.738 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(27) 20/01/23 18:54:02.738 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 27 20/01/23 18:54:02.738 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 27 20/01/23 18:54:02.738 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(29) 20/01/23 18:54:02.738 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 29 20/01/23 18:54:02.738 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 29 20/01/23 18:54:02.738 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(23) 20/01/23 18:54:02.738 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 23 20/01/23 18:54:02.738 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 23 20/01/23 18:54:02.738 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanBroadcast(1) 20/01/23 18:54:02.739 block-manager-slave-async-thread-pool-0 DEBUG BlockManager: Removing block broadcast_3_piece0 20/01/23 18:54:02.739 Spark Context Cleaner DEBUG ContextCleaner: Cleaning broadcast 1 20/01/23 18:54:02.740 Spark Context Cleaner DEBUG TorrentBroadcast: Unpersisting TorrentBroadcast 1 20/01/23 18:54:02.743 block-manager-slave-async-thread-pool-0 DEBUG MemoryStore: Block broadcast_3_piece0 of size 10144 dropped from memory (free 384034034) 20/01/23 18:54:02.745 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Removed broadcast_3_piece0 on 192.168.1.225:41233 in memory (size: 9.9 KiB, free: 366.3 MiB) 20/01/23 18:54:02.745 Executor task launch worker for task 7 DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 20/01/23 18:54:02.745 Executor task launch worker for task 6 DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 20/01/23 18:54:02.745 Executor task launch worker for task 8 DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 20/01/23 18:54:02.745 Executor task launch worker for task 5 DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 20/01/23 18:54:02.755 Executor task launch worker for task 5 INFO ShuffleBlockFetcherIterator: Getting 1 (334.0 B) non-empty blocks including 1 (334.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 20/01/23 18:54:02.756 Executor task launch worker for task 6 INFO ShuffleBlockFetcherIterator: Getting 1 (334.0 B) non-empty blocks including 1 (334.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 20/01/23 18:54:02.757 block-manager-slave-async-thread-pool-0 DEBUG BlockManagerMaster: Updated info of block broadcast_3_piece0 20/01/23 18:54:02.757 block-manager-slave-async-thread-pool-0 DEBUG BlockManager: Told master about block broadcast_3_piece0 20/01/23 18:54:02.757 Executor task launch worker for task 6 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 12 ms 20/01/23 18:54:02.758 block-manager-slave-async-thread-pool-0 DEBUG BlockManager: Removing block broadcast_3 20/01/23 18:54:02.758 Executor task launch worker for task 6 DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_1_1_1,0) 20/01/23 18:54:02.758 block-manager-slave-async-thread-pool-0 DEBUG MemoryStore: Block broadcast_3 of size 22096 dropped from memory (free 384056130) 20/01/23 18:54:02.758 Executor task launch worker for task 8 INFO ShuffleBlockFetcherIterator: Getting 2 (668.0 B) non-empty blocks including 2 (668.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 20/01/23 18:54:02.758 Executor task launch worker for task 8 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 16 ms 20/01/23 18:54:02.759 Executor task launch worker for task 8 DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_1_2_3,1), (shuffle_1_4_3,3) 20/01/23 18:54:02.758 Executor task launch worker for task 7 INFO ShuffleBlockFetcherIterator: Getting 2 (668.0 B) non-empty blocks including 2 (668.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 20/01/23 18:54:02.761 Executor task launch worker for task 7 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 17 ms 20/01/23 18:54:02.762 Executor task launch worker for task 7 DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_1_2_2,1), (shuffle_1_3_2,2) 20/01/23 18:54:02.760 Executor task launch worker for task 5 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 15 ms 20/01/23 18:54:02.762 Executor task launch worker for task 5 DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_1_4_0,3) 20/01/23 18:54:02.763 block-manager-slave-async-thread-pool-0 DEBUG BlockManagerSlaveEndpoint: removing broadcast 1 20/01/23 18:54:02.763 Executor task launch worker for task 5 DEBUG ShuffleBlockFetcherIterator: Got local blocks in 19 ms 20/01/23 18:54:02.764 Executor task launch worker for task 6 DEBUG ShuffleBlockFetcherIterator: Got local blocks in 19 ms 20/01/23 18:54:02.764 Executor task launch worker for task 8 DEBUG ShuffleBlockFetcherIterator: Got local blocks in 21 ms 20/01/23 18:54:02.765 Executor task launch worker for task 7 DEBUG ShuffleBlockFetcherIterator: Got local blocks in 21 ms 20/01/23 18:54:02.766 block-manager-slave-async-thread-pool-0 DEBUG BlockManager: Removing broadcast 1 20/01/23 18:54:02.768 block-manager-slave-async-thread-pool-0 DEBUG BlockManager: Removing block broadcast_1 20/01/23 18:54:02.769 block-manager-slave-async-thread-pool-0 DEBUG MemoryStore: Block broadcast_1 of size 27984 dropped from memory (free 383998981) 20/01/23 18:54:02.770 block-manager-slave-async-thread-pool-0 DEBUG BlockManager: Removing block broadcast_1_piece0 20/01/23 18:54:02.774 block-manager-slave-async-thread-pool-0 DEBUG MemoryStore: Block broadcast_1_piece0 of size 13553 dropped from memory (free 384012534) 20/01/23 18:54:02.777 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Removed broadcast_1_piece0 on 192.168.1.225:35863 in memory (size: 13.2 KiB, free: 366.3 MiB) 20/01/23 18:54:02.777 block-manager-slave-async-thread-pool-0 DEBUG BlockManagerMaster: Updated info of block broadcast_1_piece0 20/01/23 18:54:02.778 block-manager-slave-async-thread-pool-0 DEBUG BlockManager: Told master about block broadcast_1_piece0 20/01/23 18:54:02.787 block-manager-slave-async-thread-pool-2 DEBUG BlockManagerSlaveEndpoint: Done removing broadcast 1, response is 0 20/01/23 18:54:02.787 block-manager-slave-async-thread-pool-2 DEBUG BlockManagerSlaveEndpoint: Done removing broadcast 3, response is 0 20/01/23 18:54:02.790 block-manager-slave-async-thread-pool-2 DEBUG BlockManagerSlaveEndpoint: Sent response: 0 to 192.168.1.225:42003 20/01/23 18:54:02.790 block-manager-slave-async-thread-pool-2 DEBUG BlockManagerSlaveEndpoint: Sent response: 0 to 192.168.1.225:45091 20/01/23 18:54:02.791 Spark Context Cleaner DEBUG ContextCleaner: Cleaned broadcast 3 20/01/23 18:54:02.800 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(25) 20/01/23 18:54:02.800 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 25 20/01/23 18:54:02.801 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 25 20/01/23 18:54:02.801 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(8) 20/01/23 18:54:02.801 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 8 20/01/23 18:54:02.801 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 8 20/01/23 18:54:02.801 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(51) 20/01/23 18:54:02.801 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 51 20/01/23 18:54:02.801 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 51 20/01/23 18:54:02.801 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(21) 20/01/23 18:54:02.801 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 21 20/01/23 18:54:02.801 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 21 20/01/23 18:54:02.801 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(12) 20/01/23 18:54:02.801 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 12 20/01/23 18:54:02.801 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 12 20/01/23 18:54:02.801 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(40) 20/01/23 18:54:02.801 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 40 20/01/23 18:54:02.801 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 40 20/01/23 18:54:02.801 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(27) 20/01/23 18:54:02.801 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 27 20/01/23 18:54:02.801 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 27 20/01/23 18:54:02.801 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(52) 20/01/23 18:54:02.801 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 52 20/01/23 18:54:02.801 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 52 20/01/23 18:54:02.801 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(20) 20/01/23 18:54:02.801 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 20 20/01/23 18:54:02.802 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 20 20/01/23 18:54:02.802 Spark Context Cleaner DEBUG ContextCleaner: Cleaned broadcast 1 20/01/23 18:54:02.803 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(11) 20/01/23 18:54:02.803 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 11 20/01/23 18:54:02.803 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 11 20/01/23 18:54:02.803 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(7) 20/01/23 18:54:02.803 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 7 20/01/23 18:54:02.803 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 7 20/01/23 18:54:02.803 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(22) 20/01/23 18:54:02.803 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 22 20/01/23 18:54:02.803 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 22 20/01/23 18:54:02.803 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(13) 20/01/23 18:54:02.803 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 13 20/01/23 18:54:02.803 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 13 20/01/23 18:54:02.803 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(8) 20/01/23 18:54:02.803 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 8 20/01/23 18:54:02.803 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 8 20/01/23 18:54:02.803 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(16) 20/01/23 18:54:02.803 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 16 20/01/23 18:54:02.803 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 16 20/01/23 18:54:02.803 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(21) 20/01/23 18:54:02.803 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 21 20/01/23 18:54:02.803 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 21 20/01/23 18:54:02.803 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(6) 20/01/23 18:54:02.803 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 6 20/01/23 18:54:02.803 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 6 20/01/23 18:54:02.804 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(18) 20/01/23 18:54:02.804 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 18 20/01/23 18:54:02.804 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 18 20/01/23 18:54:02.804 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(20) 20/01/23 18:54:02.804 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 20 20/01/23 18:54:02.804 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 20 20/01/23 18:54:02.804 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(24) 20/01/23 18:54:02.804 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 24 20/01/23 18:54:02.804 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 24 20/01/23 18:54:02.804 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(19) 20/01/23 18:54:02.804 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 19 20/01/23 18:54:02.804 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 19 20/01/23 18:54:02.804 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(15) 20/01/23 18:54:02.804 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 15 20/01/23 18:54:02.804 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 15 20/01/23 18:54:02.804 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(17) 20/01/23 18:54:02.804 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 17 20/01/23 18:54:02.804 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 17 20/01/23 18:54:02.804 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(9) 20/01/23 18:54:02.804 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 9 20/01/23 18:54:02.804 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 9 20/01/23 18:54:02.804 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(5) 20/01/23 18:54:02.804 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 5 20/01/23 18:54:02.804 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 5 20/01/23 18:54:02.804 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(28) 20/01/23 18:54:02.804 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 28 20/01/23 18:54:02.804 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 28 20/01/23 18:54:02.804 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanBroadcast(0) 20/01/23 18:54:02.805 Spark Context Cleaner DEBUG ContextCleaner: Cleaning broadcast 0 20/01/23 18:54:02.805 Spark Context Cleaner DEBUG TorrentBroadcast: Unpersisting TorrentBroadcast 0 20/01/23 18:54:02.802 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(6) 20/01/23 18:54:02.805 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 6 20/01/23 18:54:02.805 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 6 20/01/23 18:54:02.805 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(54) 20/01/23 18:54:02.805 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 54 20/01/23 18:54:02.805 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 54 20/01/23 18:54:02.805 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(30) 20/01/23 18:54:02.805 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 30 20/01/23 18:54:02.805 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 30 20/01/23 18:54:02.805 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(22) 20/01/23 18:54:02.806 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 22 20/01/23 18:54:02.806 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 22 20/01/23 18:54:02.806 block-manager-slave-async-thread-pool-3 DEBUG BlockManagerSlaveEndpoint: removing broadcast 0 20/01/23 18:54:02.806 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(49) 20/01/23 18:54:02.806 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 49 20/01/23 18:54:02.806 block-manager-slave-async-thread-pool-3 DEBUG BlockManager: Removing broadcast 0 20/01/23 18:54:02.806 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 49 20/01/23 18:54:02.806 block-manager-slave-async-thread-pool-3 DEBUG BlockManager: Removing block broadcast_0_piece0 20/01/23 18:54:02.806 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(48) 20/01/23 18:54:02.806 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 48 20/01/23 18:54:02.806 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 48 20/01/23 18:54:02.806 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(50) 20/01/23 18:54:02.806 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 50 20/01/23 18:54:02.806 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 50 20/01/23 18:54:02.806 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(14) 20/01/23 18:54:02.806 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 14 20/01/23 18:54:02.806 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 14 20/01/23 18:54:02.806 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(36) 20/01/23 18:54:02.806 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 36 20/01/23 18:54:02.806 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 36 20/01/23 18:54:02.806 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(32) 20/01/23 18:54:02.806 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 32 20/01/23 18:54:02.806 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 32 20/01/23 18:54:02.806 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanBroadcast(1) 20/01/23 18:54:02.806 Spark Context Cleaner DEBUG ContextCleaner: Cleaning broadcast 1 20/01/23 18:54:02.806 Spark Context Cleaner DEBUG TorrentBroadcast: Unpersisting TorrentBroadcast 1 20/01/23 18:54:02.807 block-manager-slave-async-thread-pool-3 DEBUG MemoryStore: Block broadcast_0_piece0 of size 12199 dropped from memory (free 384024733) 20/01/23 18:54:02.808 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Removed broadcast_0_piece0 on 192.168.1.225:35863 in memory (size: 11.9 KiB, free: 366.3 MiB) 20/01/23 18:54:02.808 block-manager-slave-async-thread-pool-3 DEBUG BlockManagerMaster: Updated info of block broadcast_0_piece0 20/01/23 18:54:02.808 block-manager-slave-async-thread-pool-3 DEBUG BlockManager: Told master about block broadcast_0_piece0 20/01/23 18:54:02.808 block-manager-slave-async-thread-pool-3 DEBUG BlockManager: Removing block broadcast_0 20/01/23 18:54:02.809 block-manager-slave-async-thread-pool-3 DEBUG MemoryStore: Block broadcast_0 of size 24880 dropped from memory (free 384049613) 20/01/23 18:54:02.809 block-manager-slave-async-thread-pool-3 DEBUG BlockManagerSlaveEndpoint: removing broadcast 1 20/01/23 18:54:02.810 block-manager-slave-async-thread-pool-3 DEBUG BlockManager: Removing broadcast 1 20/01/23 18:54:02.810 block-manager-slave-async-thread-pool-3 DEBUG BlockManager: Removing block broadcast_1 20/01/23 18:54:02.810 block-manager-slave-async-thread-pool-3 DEBUG MemoryStore: Block broadcast_1 of size 25448 dropped from memory (free 384081578) 20/01/23 18:54:02.810 block-manager-slave-async-thread-pool-3 DEBUG BlockManager: Removing block broadcast_1_piece0 20/01/23 18:54:02.811 block-manager-slave-async-thread-pool-3 DEBUG MemoryStore: Block broadcast_1_piece0 of size 10770 dropped from memory (free 384092348) 20/01/23 18:54:02.812 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Removed broadcast_1_piece0 on 192.168.1.225:41233 in memory (size: 10.5 KiB, free: 366.3 MiB) 20/01/23 18:54:02.812 block-manager-slave-async-thread-pool-3 DEBUG BlockManagerMaster: Updated info of block broadcast_1_piece0 20/01/23 18:54:02.812 block-manager-slave-async-thread-pool-3 DEBUG BlockManager: Told master about block broadcast_1_piece0 20/01/23 18:54:02.813 block-manager-slave-async-thread-pool-5 DEBUG BlockManagerSlaveEndpoint: Done removing broadcast 0, response is 0 20/01/23 18:54:02.813 block-manager-slave-async-thread-pool-5 DEBUG BlockManagerSlaveEndpoint: Done removing broadcast 1, response is 0 20/01/23 18:54:02.814 block-manager-slave-async-thread-pool-5 DEBUG BlockManagerSlaveEndpoint: Sent response: 0 to 192.168.1.225:42003 20/01/23 18:54:02.814 Executor task launch worker for task 6 DEBUG TaskMemoryManager: Task 6 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@4d51d0fd 20/01/23 18:54:02.815 Executor task launch worker for task 6 INFO MemoryStore: Block rdd_16_1 stored as values in memory (estimated size 176.0 B, free 366.3 MiB) 20/01/23 18:54:02.815 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_4 stored as values in memory (estimated size 21.5 KiB, free 366.3 MiB) 20/01/23 18:54:02.816 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_4 locally took 2 ms 20/01/23 18:54:02.816 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_16_1 in memory on 192.168.1.225:35863 (size: 176.0 B, free: 366.3 MiB) 20/01/23 18:54:02.816 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_4 without replication took 2 ms 20/01/23 18:54:02.816 Spark Context Cleaner DEBUG ContextCleaner: Cleaned broadcast 1 20/01/23 18:54:02.816 Executor task launch worker for task 6 DEBUG BlockManagerMaster: Updated info of block rdd_16_1 20/01/23 18:54:02.816 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(47) 20/01/23 18:54:02.816 Executor task launch worker for task 6 DEBUG BlockManager: Told master about block rdd_16_1 20/01/23 18:54:02.816 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 47 20/01/23 18:54:02.816 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 47 20/01/23 18:54:02.816 Executor task launch worker for task 6 DEBUG BlockManager: Put block rdd_16_1 locally took 382 ms 20/01/23 18:54:02.816 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(16) 20/01/23 18:54:02.816 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 16 20/01/23 18:54:02.816 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 16 20/01/23 18:54:02.816 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(13) 20/01/23 18:54:02.816 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 13 20/01/23 18:54:02.816 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 13 20/01/23 18:54:02.816 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(35) 20/01/23 18:54:02.816 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 35 20/01/23 18:54:02.816 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 35 20/01/23 18:54:02.816 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(53) 20/01/23 18:54:02.817 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 53 20/01/23 18:54:02.817 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 53 20/01/23 18:54:02.817 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(34) 20/01/23 18:54:02.817 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 34 20/01/23 18:54:02.817 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 34 20/01/23 18:54:02.817 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(42) 20/01/23 18:54:02.817 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 42 20/01/23 18:54:02.817 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 42 20/01/23 18:54:02.817 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(39) 20/01/23 18:54:02.817 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 39 20/01/23 18:54:02.817 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 39 20/01/23 18:54:02.817 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(15) 20/01/23 18:54:02.817 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 15 20/01/23 18:54:02.817 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 15 20/01/23 18:54:02.817 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(46) 20/01/23 18:54:02.817 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 46 20/01/23 18:54:02.817 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 46 20/01/23 18:54:02.817 Executor task launch worker for task 6 DEBUG BlockManager: Putting block rdd_16_1 without replication took 383 ms 20/01/23 18:54:02.817 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(10) 20/01/23 18:54:02.817 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 10 20/01/23 18:54:02.817 Executor task launch worker for task 6 DEBUG BlockManager: Getting local block rdd_16_1 20/01/23 18:54:02.817 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 10 20/01/23 18:54:02.817 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(37) 20/01/23 18:54:02.817 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 37 20/01/23 18:54:02.817 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 37 20/01/23 18:54:02.819 Executor task launch worker for task 6 DEBUG BlockManager: Level for block rdd_16_1 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:02.819 Thread-4 DEBUG Analyzer$ResolveReferences: Resolving 'normalizedFeatures to normalizedFeatures#7 20/01/23 18:54:02.819 Thread-4 DEBUG Analyzer$ResolveReferences: Resolving 'prediction to prediction#1 20/01/23 18:54:02.817 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(43) 20/01/23 18:54:02.818 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_4_piece0 stored as bytes in memory (estimated size 10.5 KiB, free 366.3 MiB) 20/01/23 18:54:02.820 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 43 20/01/23 18:54:02.820 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 43 20/01/23 18:54:02.820 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(18) 20/01/23 18:54:02.820 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 18 20/01/23 18:54:02.820 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 18 20/01/23 18:54:02.820 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(24) 20/01/23 18:54:02.821 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 24 20/01/23 18:54:02.821 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 24 20/01/23 18:54:02.821 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(31) 20/01/23 18:54:02.821 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 31 20/01/23 18:54:02.821 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 31 20/01/23 18:54:02.821 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(44) 20/01/23 18:54:02.821 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 44 20/01/23 18:54:02.815 Executor task launch worker for task 8 DEBUG TaskMemoryManager: Task 8 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@54b235f6 20/01/23 18:54:02.816 Executor task launch worker for task 5 DEBUG TaskMemoryManager: Task 5 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@52b277bf 20/01/23 18:54:02.821 block-manager-slave-async-thread-pool-5 DEBUG BlockManagerSlaveEndpoint: Sent response: 0 to 192.168.1.225:45091 20/01/23 18:54:02.815 Executor task launch worker for task 7 DEBUG TaskMemoryManager: Task 7 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@1fde04c1 20/01/23 18:54:02.822 Executor task launch worker for task 5 INFO MemoryStore: Block rdd_16_0 stored as values in memory (estimated size 176.0 B, free 366.3 MiB) 20/01/23 18:54:02.822 Executor task launch worker for task 8 INFO MemoryStore: Block rdd_16_3 stored as values in memory (estimated size 328.0 B, free 366.3 MiB) 20/01/23 18:54:02.822 Executor task launch worker for task 7 INFO MemoryStore: Block rdd_16_2 stored as values in memory (estimated size 328.0 B, free 366.3 MiB) 20/01/23 18:54:02.822 Spark Context Cleaner DEBUG ContextCleaner: Cleaned broadcast 0 20/01/23 18:54:02.822 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_16_3 in memory on 192.168.1.225:35863 (size: 328.0 B, free: 366.3 MiB) 20/01/23 18:54:02.823 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(25) 20/01/23 18:54:02.823 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 25 20/01/23 18:54:02.823 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 25 20/01/23 18:54:02.823 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(26) 20/01/23 18:54:02.821 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 44 20/01/23 18:54:02.823 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 26 20/01/23 18:54:02.823 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 26 20/01/23 18:54:02.823 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(38) 20/01/23 18:54:02.823 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 38 20/01/23 18:54:02.823 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 38 20/01/23 18:54:02.823 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(3) 20/01/23 18:54:02.823 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 3 20/01/23 18:54:02.821 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added broadcast_4_piece0 in memory on 192.168.1.225:41233 (size: 10.5 KiB, free: 366.3 MiB) 20/01/23 18:54:02.823 dag-scheduler-event-loop DEBUG BlockManagerMaster: Updated info of block broadcast_4_piece0 20/01/23 18:54:02.824 dag-scheduler-event-loop DEBUG BlockManager: Told master about block broadcast_4_piece0 20/01/23 18:54:02.824 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_4_piece0 locally took 6 ms 20/01/23 18:54:02.824 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_4_piece0 without replication took 6 ms 20/01/23 18:54:02.824 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 3 20/01/23 18:54:02.824 dag-scheduler-event-loop INFO SparkContext: Created broadcast 4 from broadcast at DAGScheduler.scala:1200 20/01/23 18:54:02.824 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(33) 20/01/23 18:54:02.824 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 33 20/01/23 18:54:02.825 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 33 20/01/23 18:54:02.825 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(26) 20/01/23 18:54:02.825 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 26 20/01/23 18:54:02.825 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 26 20/01/23 18:54:02.825 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(5) 20/01/23 18:54:02.825 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 5 20/01/23 18:54:02.825 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 5 20/01/23 18:54:02.825 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(45) 20/01/23 18:54:02.825 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 45 20/01/23 18:54:02.825 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 45 20/01/23 18:54:02.825 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(41) 20/01/23 18:54:02.825 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 41 20/01/23 18:54:02.825 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 41 20/01/23 18:54:02.825 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(11) 20/01/23 18:54:02.825 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 11 20/01/23 18:54:02.825 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 11 20/01/23 18:54:02.827 dag-scheduler-event-loop INFO DAGScheduler: Submitting 4 missing tasks from ShuffleMapStage 2 (MapPartitionsRDD[19] at flatMap at CountVectorizer.scala:212) (first 15 tasks are for partitions Vector(0, 1, 2, 3)) 20/01/23 18:54:02.827 dag-scheduler-event-loop INFO TaskSchedulerImpl: Adding task set 2.0 with 4 tasks 20/01/23 18:54:02.828 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_16_0 in memory on 192.168.1.225:35863 (size: 176.0 B, free: 366.3 MiB) 20/01/23 18:54:02.828 dag-scheduler-event-loop DEBUG TaskSetManager: Epoch for TaskSet 2.0: 0 20/01/23 18:54:02.828 dag-scheduler-event-loop DEBUG TaskSetManager: Adding pending tasks took 0 ms 20/01/23 18:54:02.828 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_16_2 in memory on 192.168.1.225:35863 (size: 328.0 B, free: 366.3 MiB) 20/01/23 18:54:02.828 dag-scheduler-event-loop DEBUG TaskSetManager: Valid locality levels for TaskSet 2.0: NO_PREF, ANY 20/01/23 18:54:02.828 Executor task launch worker for task 7 DEBUG BlockManagerMaster: Updated info of block rdd_16_2 20/01/23 18:54:02.828 Executor task launch worker for task 5 DEBUG BlockManagerMaster: Updated info of block rdd_16_0 20/01/23 18:54:02.828 Executor task launch worker for task 7 DEBUG BlockManager: Told master about block rdd_16_2 20/01/23 18:54:02.829 Executor task launch worker for task 7 DEBUG BlockManager: Put block rdd_16_2 locally took 390 ms 20/01/23 18:54:02.829 dispatcher-event-loop-1 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_2.0, runningTasks: 0 20/01/23 18:54:02.829 Executor task launch worker for task 7 DEBUG BlockManager: Putting block rdd_16_2 without replication took 390 ms 20/01/23 18:54:02.829 Executor task launch worker for task 7 DEBUG BlockManager: Getting local block rdd_16_2 20/01/23 18:54:02.829 Executor task launch worker for task 7 DEBUG BlockManager: Level for block rdd_16_2 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:02.830 Executor task launch worker for task 8 DEBUG BlockManagerMaster: Updated info of block rdd_16_3 20/01/23 18:54:02.830 dispatcher-event-loop-1 INFO TaskSetManager: Starting task 0.0 in stage 2.0 (TID 8, 192.168.1.225, executor driver, partition 0, PROCESS_LOCAL, 7421 bytes) 20/01/23 18:54:02.830 Executor task launch worker for task 8 DEBUG BlockManager: Told master about block rdd_16_3 20/01/23 18:54:02.830 Executor task launch worker for task 8 DEBUG BlockManager: Put block rdd_16_3 locally took 392 ms 20/01/23 18:54:02.831 Executor task launch worker for task 8 DEBUG BlockManager: Putting block rdd_16_3 without replication took 392 ms 20/01/23 18:54:02.831 Executor task launch worker for task 8 DEBUG BlockManager: Getting local block rdd_16_3 20/01/23 18:54:02.831 dispatcher-event-loop-1 INFO TaskSetManager: Starting task 1.0 in stage 2.0 (TID 9, 192.168.1.225, executor driver, partition 1, PROCESS_LOCAL, 7385 bytes) 20/01/23 18:54:02.831 Executor task launch worker for task 8 DEBUG BlockManager: Level for block rdd_16_3 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:02.831 dispatcher-event-loop-1 INFO TaskSetManager: Starting task 2.0 in stage 2.0 (TID 10, 192.168.1.225, executor driver, partition 2, PROCESS_LOCAL, 7400 bytes) 20/01/23 18:54:02.831 dispatcher-event-loop-1 INFO TaskSetManager: Starting task 3.0 in stage 2.0 (TID 11, 192.168.1.225, executor driver, partition 3, PROCESS_LOCAL, 7382 bytes) 20/01/23 18:54:02.832 Executor task launch worker for task 8 INFO Executor: Running task 0.0 in stage 2.0 (TID 8) 20/01/23 18:54:02.832 Executor task launch worker for task 9 INFO Executor: Running task 1.0 in stage 2.0 (TID 9) 20/01/23 18:54:02.832 Executor task launch worker for task 11 INFO Executor: Running task 3.0 in stage 2.0 (TID 11) 20/01/23 18:54:02.832 Executor task launch worker for task 10 INFO Executor: Running task 2.0 in stage 2.0 (TID 10) 20/01/23 18:54:02.833 Executor task launch worker for task 5 DEBUG BlockManager: Told master about block rdd_16_0 20/01/23 18:54:02.833 Executor task launch worker for task 5 DEBUG BlockManager: Put block rdd_16_0 locally took 391 ms 20/01/23 18:54:02.833 Executor task launch worker for task 10 DEBUG ExecutorMetricsPoller: stageTCMP: (2, 0) -> 1 20/01/23 18:54:02.833 Executor task launch worker for task 5 DEBUG BlockManager: Putting block rdd_16_0 without replication took 391 ms 20/01/23 18:54:02.833 Executor task launch worker for task 5 DEBUG BlockManager: Getting local block rdd_16_0 20/01/23 18:54:02.833 Executor task launch worker for task 5 DEBUG BlockManager: Level for block rdd_16_0 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:02.834 Executor task launch worker for task 10 DEBUG BlockManager: Getting local block broadcast_4 20/01/23 18:54:02.834 Executor task launch worker for task 10 DEBUG BlockManager: Level for block broadcast_4 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:02.838 Executor task launch worker for task 11 DEBUG ExecutorMetricsPoller: stageTCMP: (2, 0) -> 2 20/01/23 18:54:02.837 Executor task launch worker for task 8 DEBUG ALS: Converting to local indices took 0.005355723 seconds. 20/01/23 18:54:02.838 Executor task launch worker for task 5 DEBUG ALS: Converting to local indices took 0.002923168 seconds. 20/01/23 18:54:02.837 Executor task launch worker for task 7 DEBUG ALS: Converting to local indices took 0.00653927 seconds. 20/01/23 18:54:02.838 Executor task launch worker for task 9 DEBUG ExecutorMetricsPoller: stageTCMP: (2, 0) -> 3 20/01/23 18:54:02.838 Executor task launch worker for task 8 DEBUG ExecutorMetricsPoller: stageTCMP: (2, 0) -> 4 20/01/23 18:54:02.839 Executor task launch worker for task 6 DEBUG ALS: Converting to local indices took 0.016428392 seconds. 20/01/23 18:54:02.841 Executor task launch worker for task 7 DEBUG ALS: Converting to local indices took 0.00173014 seconds. 20/01/23 18:54:02.843 Executor task launch worker for task 6 INFO Executor: Finished task 1.0 in stage 2.0 (TID 6). 2432 bytes result sent to driver 20/01/23 18:54:02.843 dispatcher-event-loop-2 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_2.0, runningTasks: 3 20/01/23 18:54:02.844 dispatcher-event-loop-2 DEBUG TaskSetManager: No tasks for locality level NODE_LOCAL, so moving to locality level ANY 20/01/23 18:54:02.844 Executor task launch worker for task 7 INFO Executor: Finished task 2.0 in stage 2.0 (TID 7). 2432 bytes result sent to driver 20/01/23 18:54:02.844 Executor task launch worker for task 6 DEBUG ExecutorMetricsPoller: stageTCMP: (2, 0) -> 3 20/01/23 18:54:02.844 task-result-getter-3 INFO TaskSetManager: Finished task 1.0 in stage 2.0 (TID 6) in 433 ms on 192.168.1.225 (executor driver) (1/4) 20/01/23 18:54:02.845 task-result-getter-1 INFO TaskSetManager: Finished task 2.0 in stage 2.0 (TID 7) in 434 ms on 192.168.1.225 (executor driver) (2/4) 20/01/23 18:54:02.843 Executor task launch worker for task 8 DEBUG ALS: Converting to local indices took 0.001796582 seconds. 20/01/23 18:54:02.845 dispatcher-event-loop-0 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_2.0, runningTasks: 2 20/01/23 18:54:02.846 dag-scheduler-event-loop DEBUG DAGScheduler: ShuffleMapTask finished on driver 20/01/23 18:54:02.846 Executor task launch worker for task 7 DEBUG ExecutorMetricsPoller: stageTCMP: (2, 0) -> 2 20/01/23 18:54:02.847 dag-scheduler-event-loop DEBUG DAGScheduler: ShuffleMapTask finished on driver 20/01/23 18:54:02.851 Executor task launch worker for task 5 INFO Executor: Finished task 0.0 in stage 2.0 (TID 5). 2432 bytes result sent to driver 20/01/23 18:54:02.851 Executor task launch worker for task 5 DEBUG ExecutorMetricsPoller: stageTCMP: (2, 0) -> 1 20/01/23 18:54:02.852 Executor task launch worker for task 8 INFO Executor: Finished task 3.0 in stage 2.0 (TID 8). 2432 bytes result sent to driver 20/01/23 18:54:02.852 Executor task launch worker for task 8 DEBUG ExecutorMetricsPoller: removing (2, 0) from stageTCMP 20/01/23 18:54:02.852 dispatcher-event-loop-3 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_2.0, runningTasks: 1 20/01/23 18:54:02.853 dispatcher-event-loop-3 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_2.0, runningTasks: 0 20/01/23 18:54:02.854 task-result-getter-0 INFO TaskSetManager: Finished task 0.0 in stage 2.0 (TID 5) in 446 ms on 192.168.1.225 (executor driver) (3/4) 20/01/23 18:54:02.854 task-result-getter-2 INFO TaskSetManager: Finished task 3.0 in stage 2.0 (TID 8) in 443 ms on 192.168.1.225 (executor driver) (4/4) 20/01/23 18:54:02.855 task-result-getter-2 INFO TaskSchedulerImpl: Removed TaskSet 2.0, whose tasks have all completed, from pool 20/01/23 18:54:02.855 dag-scheduler-event-loop DEBUG DAGScheduler: ShuffleMapTask finished on driver 20/01/23 18:54:02.856 dag-scheduler-event-loop DEBUG DAGScheduler: ShuffleMapTask finished on driver 20/01/23 18:54:02.857 dag-scheduler-event-loop INFO DAGScheduler: ShuffleMapStage 2 (map at ALS.scala:1585) finished in 0.494 s 20/01/23 18:54:02.857 dag-scheduler-event-loop INFO DAGScheduler: looking for newly runnable stages 20/01/23 18:54:02.857 dag-scheduler-event-loop INFO DAGScheduler: running: Set() 20/01/23 18:54:02.857 dag-scheduler-event-loop INFO DAGScheduler: waiting: Set(ResultStage 3) 20/01/23 18:54:02.857 dag-scheduler-event-loop INFO DAGScheduler: failed: Set() 20/01/23 18:54:02.857 dag-scheduler-event-loop DEBUG MapOutputTrackerMaster: Increasing epoch to 2 20/01/23 18:54:02.858 dag-scheduler-event-loop DEBUG DAGScheduler: submitStage(ResultStage 3 (name=count at ALS.scala:944;jobs=1)) 20/01/23 18:54:02.862 dag-scheduler-event-loop DEBUG DAGScheduler: missing: List() 20/01/23 18:54:02.862 dag-scheduler-event-loop INFO DAGScheduler: Submitting ResultStage 3 (userOutBlocks MapPartitionsRDD[20] at mapValues at ALS.scala:1622), which has no missing parents 20/01/23 18:54:02.863 dag-scheduler-event-loop DEBUG DAGScheduler: submitMissingTasks(ResultStage 3) 20/01/23 18:54:02.872 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_3 stored as values in memory (estimated size 30.1 KiB, free 366.2 MiB) 20/01/23 18:54:02.873 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_3 locally took 1 ms 20/01/23 18:54:02.873 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_3 without replication took 1 ms 20/01/23 18:54:02.876 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_3_piece0 stored as bytes in memory (estimated size 14.2 KiB, free 366.2 MiB) 20/01/23 18:54:02.876 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added broadcast_3_piece0 in memory on 192.168.1.225:35863 (size: 14.2 KiB, free: 366.3 MiB) 20/01/23 18:54:02.877 dag-scheduler-event-loop DEBUG BlockManagerMaster: Updated info of block broadcast_3_piece0 20/01/23 18:54:02.877 dag-scheduler-event-loop DEBUG BlockManager: Told master about block broadcast_3_piece0 20/01/23 18:54:02.877 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_3_piece0 locally took 3 ms 20/01/23 18:54:02.877 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_3_piece0 without replication took 3 ms 20/01/23 18:54:02.878 dag-scheduler-event-loop INFO SparkContext: Created broadcast 3 from broadcast at DAGScheduler.scala:1200 20/01/23 18:54:02.879 dag-scheduler-event-loop INFO DAGScheduler: Submitting 10 missing tasks from ResultStage 3 (userOutBlocks MapPartitionsRDD[20] at mapValues at ALS.scala:1622) (first 15 tasks are for partitions Vector(0, 1, 2, 3, 4, 5, 6, 7, 8, 9)) 20/01/23 18:54:02.879 dag-scheduler-event-loop INFO TaskSchedulerImpl: Adding task set 3.0 with 10 tasks 20/01/23 18:54:02.879 dag-scheduler-event-loop DEBUG TaskSetManager: Epoch for TaskSet 3.0: 2 20/01/23 18:54:02.879 dag-scheduler-event-loop DEBUG TaskSetManager: Adding pending tasks took 0 ms 20/01/23 18:54:02.880 dag-scheduler-event-loop DEBUG TaskSetManager: Valid locality levels for TaskSet 3.0: NODE_LOCAL, NO_PREF, ANY 20/01/23 18:54:02.880 dispatcher-event-loop-2 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_3.0, runningTasks: 0 20/01/23 18:54:02.881 dispatcher-event-loop-2 INFO TaskSetManager: Starting task 0.0 in stage 3.0 (TID 9, 192.168.1.225, executor driver, partition 0, NODE_LOCAL, 7143 bytes) 20/01/23 18:54:02.881 dispatcher-event-loop-2 INFO TaskSetManager: Starting task 1.0 in stage 3.0 (TID 10, 192.168.1.225, executor driver, partition 1, NODE_LOCAL, 7143 bytes) 20/01/23 18:54:02.882 dispatcher-event-loop-2 INFO TaskSetManager: Starting task 2.0 in stage 3.0 (TID 11, 192.168.1.225, executor driver, partition 2, NODE_LOCAL, 7143 bytes) 20/01/23 18:54:02.882 dispatcher-event-loop-2 DEBUG TaskSetManager: No tasks for locality level NODE_LOCAL, so moving to locality level NO_PREF 20/01/23 18:54:02.883 dispatcher-event-loop-2 DEBUG TaskSetManager: Moving to ANY after waiting for 0ms 20/01/23 18:54:02.883 dispatcher-event-loop-2 INFO TaskSetManager: Starting task 3.0 in stage 3.0 (TID 12, 192.168.1.225, executor driver, partition 3, PROCESS_LOCAL, 7143 bytes) 20/01/23 18:54:02.883 Executor task launch worker for task 11 INFO Executor: Running task 2.0 in stage 3.0 (TID 11) 20/01/23 18:54:02.883 Executor task launch worker for task 9 INFO Executor: Running task 0.0 in stage 3.0 (TID 9) 20/01/23 18:54:02.883 Executor task launch worker for task 10 INFO Executor: Running task 1.0 in stage 3.0 (TID 10) 20/01/23 18:54:02.883 Executor task launch worker for task 12 INFO Executor: Running task 3.0 in stage 3.0 (TID 12) 20/01/23 18:54:02.884 Executor task launch worker for task 9 DEBUG ExecutorMetricsPoller: stageTCMP: (3, 0) -> 3 20/01/23 18:54:02.885 Executor task launch worker for task 12 DEBUG ExecutorMetricsPoller: stageTCMP: (3, 0) -> 4 20/01/23 18:54:02.884 Executor task launch worker for task 10 DEBUG ExecutorMetricsPoller: stageTCMP: (3, 0) -> 1 20/01/23 18:54:02.885 Executor task launch worker for task 9 DEBUG BlockManager: Getting local block broadcast_3 20/01/23 18:54:02.885 Executor task launch worker for task 9 DEBUG BlockManager: Level for block broadcast_3 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:02.884 Executor task launch worker for task 11 DEBUG ExecutorMetricsPoller: stageTCMP: (3, 0) -> 2 20/01/23 18:54:02.893 Executor task launch worker for task 9 DEBUG BlockManager: Getting local block rdd_20_0 20/01/23 18:54:02.893 Executor task launch worker for task 9 DEBUG BlockManager: Block rdd_20_0 was not found 20/01/23 18:54:02.893 Executor task launch worker for task 9 DEBUG BlockManager: Getting remote block rdd_20_0 20/01/23 18:54:02.896 Executor task launch worker for task 12 DEBUG BlockManager: Getting local block rdd_20_3 20/01/23 18:54:02.896 Executor task launch worker for task 12 DEBUG BlockManager: Block rdd_20_3 was not found 20/01/23 18:54:02.896 Executor task launch worker for task 12 DEBUG BlockManager: Getting remote block rdd_20_3 20/01/23 18:54:02.896 Executor task launch worker for task 12 DEBUG BlockManager: Block rdd_20_3 is unknown by block manager master 20/01/23 18:54:02.896 Executor task launch worker for task 12 DEBUG BlockManager: Getting local block rdd_19_3 20/01/23 18:54:02.896 Executor task launch worker for task 12 DEBUG BlockManager: Block rdd_19_3 was not found 20/01/23 18:54:02.896 Executor task launch worker for task 12 DEBUG BlockManager: Getting remote block rdd_19_3 20/01/23 18:54:02.899 Executor task launch worker for task 10 DEBUG BlockManager: Getting local block rdd_20_1 20/01/23 18:54:02.899 Executor task launch worker for task 10 DEBUG BlockManager: Block rdd_20_1 was not found 20/01/23 18:54:02.899 Executor task launch worker for task 10 DEBUG BlockManager: Getting remote block rdd_20_1 20/01/23 18:54:02.900 Executor task launch worker for task 12 DEBUG BlockManager: Block rdd_19_3 is unknown by block manager master 20/01/23 18:54:02.900 Executor task launch worker for task 12 DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 0, partitions 3-4 20/01/23 18:54:02.900 Executor task launch worker for task 12 DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 20/01/23 18:54:02.901 Executor task launch worker for task 12 INFO ShuffleBlockFetcherIterator: Getting 0 (0.0 B) non-empty blocks including 0 (0.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 20/01/23 18:54:02.901 Executor task launch worker for task 12 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 20/01/23 18:54:02.901 Executor task launch worker for task 12 DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: 20/01/23 18:54:02.901 Executor task launch worker for task 12 DEBUG ShuffleBlockFetcherIterator: Got local blocks in 0 ms 20/01/23 18:54:02.902 Executor task launch worker for task 12 DEBUG TaskMemoryManager: Task 12 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@59c0e8e7 20/01/23 18:54:02.902 Executor task launch worker for task 12 INFO MemoryStore: Block rdd_19_3 stored as values in memory (estimated size 16.0 B, free 366.2 MiB) 20/01/23 18:54:02.902 Executor task launch worker for task 10 DEBUG BlockManager: Block rdd_20_1 is unknown by block manager master 20/01/23 18:54:02.903 Executor task launch worker for task 11 DEBUG BlockManager: Getting local block rdd_20_2 20/01/23 18:54:02.903 Executor task launch worker for task 11 DEBUG BlockManager: Block rdd_20_2 was not found 20/01/23 18:54:02.903 Executor task launch worker for task 11 DEBUG BlockManager: Getting remote block rdd_20_2 20/01/23 18:54:02.903 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_19_3 in memory on 192.168.1.225:35863 (size: 16.0 B, free: 366.3 MiB) 20/01/23 18:54:02.903 Executor task launch worker for task 10 DEBUG BlockManager: Getting local block rdd_19_1 20/01/23 18:54:02.903 Executor task launch worker for task 10 DEBUG BlockManager: Block rdd_19_1 was not found 20/01/23 18:54:02.903 Executor task launch worker for task 10 DEBUG BlockManager: Getting remote block rdd_19_1 20/01/23 18:54:02.903 Executor task launch worker for task 10 DEBUG BlockManager: Block rdd_19_1 is unknown by block manager master 20/01/23 18:54:02.904 Executor task launch worker for task 10 DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 0, partitions 1-2 20/01/23 18:54:02.904 Executor task launch worker for task 10 DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 20/01/23 18:54:02.904 Executor task launch worker for task 11 DEBUG BlockManager: Block rdd_20_2 is unknown by block manager master 20/01/23 18:54:02.905 Executor task launch worker for task 10 INFO ShuffleBlockFetcherIterator: Getting 2 (502.0 B) non-empty blocks including 2 (502.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 20/01/23 18:54:02.905 Executor task launch worker for task 11 DEBUG BlockManager: Getting local block rdd_19_2 20/01/23 18:54:02.905 Executor task launch worker for task 11 DEBUG BlockManager: Block rdd_19_2 was not found 20/01/23 18:54:02.905 Executor task launch worker for task 11 DEBUG BlockManager: Getting remote block rdd_19_2 20/01/23 18:54:02.905 Executor task launch worker for task 12 DEBUG BlockManagerMaster: Updated info of block rdd_19_3 20/01/23 18:54:02.905 Executor task launch worker for task 12 DEBUG BlockManager: Told master about block rdd_19_3 20/01/23 18:54:02.905 Executor task launch worker for task 12 DEBUG BlockManager: Put block rdd_19_3 locally took 5 ms 20/01/23 18:54:02.905 Executor task launch worker for task 11 DEBUG BlockManager: Block rdd_19_2 is unknown by block manager master 20/01/23 18:54:02.905 Executor task launch worker for task 10 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 20/01/23 18:54:02.905 Executor task launch worker for task 10 DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_0_7_1,2), (shuffle_0_8_1,3) 20/01/23 18:54:02.906 Executor task launch worker for task 10 DEBUG ShuffleBlockFetcherIterator: Got local blocks in 1 ms 20/01/23 18:54:02.905 Executor task launch worker for task 12 DEBUG BlockManager: Putting block rdd_19_3 without replication took 5 ms 20/01/23 18:54:02.905 Executor task launch worker for task 11 DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 0, partitions 2-3 20/01/23 18:54:02.909 Executor task launch worker for task 11 DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 20/01/23 18:54:02.909 Executor task launch worker for task 11 INFO ShuffleBlockFetcherIterator: Getting 2 (527.0 B) non-empty blocks including 2 (527.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 20/01/23 18:54:02.910 Executor task launch worker for task 11 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 20/01/23 18:54:02.910 Executor task launch worker for task 11 DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_0_5_2,0), (shuffle_0_8_2,3) 20/01/23 18:54:02.910 Executor task launch worker for task 11 DEBUG ShuffleBlockFetcherIterator: Got local blocks in 1 ms 20/01/23 18:54:02.908 Executor task launch worker for task 12 DEBUG BlockManager: Getting local block rdd_19_3 20/01/23 18:54:02.911 Executor task launch worker for task 12 DEBUG BlockManager: Level for block rdd_19_3 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:02.911 Executor task launch worker for task 12 INFO MemoryStore: Block rdd_20_3 stored as values in memory (estimated size 16.0 B, free 366.2 MiB) 20/01/23 18:54:02.912 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_20_3 in memory on 192.168.1.225:35863 (size: 16.0 B, free: 366.3 MiB) 20/01/23 18:54:02.913 Executor task launch worker for task 12 DEBUG BlockManagerMaster: Updated info of block rdd_20_3 20/01/23 18:54:02.913 Executor task launch worker for task 12 DEBUG BlockManager: Told master about block rdd_20_3 20/01/23 18:54:02.913 Executor task launch worker for task 12 DEBUG BlockManager: Put block rdd_20_3 locally took 16 ms 20/01/23 18:54:02.913 Executor task launch worker for task 12 DEBUG BlockManager: Putting block rdd_20_3 without replication took 16 ms 20/01/23 18:54:02.913 Executor task launch worker for task 12 DEBUG BlockManager: Getting local block rdd_20_3 20/01/23 18:54:02.914 Executor task launch worker for task 12 DEBUG BlockManager: Level for block rdd_20_3 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:02.915 Executor task launch worker for task 12 INFO Executor: Finished task 3.0 in stage 3.0 (TID 12). 2222 bytes result sent to driver 20/01/23 18:54:02.915 Executor task launch worker for task 12 DEBUG ExecutorMetricsPoller: stageTCMP: (3, 0) -> 3 20/01/23 18:54:02.916 dispatcher-event-loop-1 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_3.0, runningTasks: 3 20/01/23 18:54:02.916 dispatcher-event-loop-1 INFO TaskSetManager: Starting task 4.0 in stage 3.0 (TID 13, 192.168.1.225, executor driver, partition 4, PROCESS_LOCAL, 7143 bytes) 20/01/23 18:54:02.917 Executor task launch worker for task 10 DEBUG ALS: Start sorting an uncompressed in-block of size 2. (sortId = 1730134278) 20/01/23 18:54:02.917 task-result-getter-3 INFO TaskSetManager: Finished task 3.0 in stage 3.0 (TID 12) in 34 ms on 192.168.1.225 (executor driver) (1/10) 20/01/23 18:54:02.920 Executor task launch worker for task 9 DEBUG BlockManager: Block rdd_20_0 is unknown by block manager master 20/01/23 18:54:02.920 Executor task launch worker for task 9 DEBUG BlockManager: Getting local block rdd_19_0 20/01/23 18:54:02.920 Executor task launch worker for task 9 DEBUG BlockManager: Block rdd_19_0 was not found 20/01/23 18:54:02.920 Executor task launch worker for task 9 DEBUG BlockManager: Getting remote block rdd_19_0 20/01/23 18:54:02.917 Executor task launch worker for task 13 INFO Executor: Running task 4.0 in stage 3.0 (TID 13) 20/01/23 18:54:02.921 Executor task launch worker for task 9 DEBUG BlockManager: Block rdd_19_0 is unknown by block manager master 20/01/23 18:54:02.921 Executor task launch worker for task 9 DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 0, partitions 0-1 20/01/23 18:54:02.921 Executor task launch worker for task 13 DEBUG ExecutorMetricsPoller: stageTCMP: (3, 0) -> 4 20/01/23 18:54:02.923 Executor task launch worker for task 9 DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 20/01/23 18:54:02.923 Executor task launch worker for task 10 DEBUG ALS: Sorting took 0.002939238 seconds. (sortId = 1730134278) 20/01/23 18:54:02.929 Executor task launch worker for task 11 DEBUG ALS: Start sorting an uncompressed in-block of size 2. (sortId = 1055188235) 20/01/23 18:54:02.929 Executor task launch worker for task 11 DEBUG ALS: Sorting took 3.555E-5 seconds. (sortId = 1055188235) 20/01/23 18:54:02.929 Executor task launch worker for task 9 INFO ShuffleBlockFetcherIterator: Getting 2 (502.0 B) non-empty blocks including 2 (502.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 20/01/23 18:54:02.929 Executor task launch worker for task 9 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 6 ms 20/01/23 18:54:02.929 Executor task launch worker for task 9 DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_0_6_0,1), (shuffle_0_7_0,2) 20/01/23 18:54:02.930 Executor task launch worker for task 9 DEBUG ShuffleBlockFetcherIterator: Got local blocks in 7 ms 20/01/23 18:54:02.932 Executor task launch worker for task 9 DEBUG ALS: Start sorting an uncompressed in-block of size 2. (sortId = 2137339598) 20/01/23 18:54:02.933 Executor task launch worker for task 9 DEBUG ALS: Sorting took 2.896E-5 seconds. (sortId = 2137339598) 20/01/23 18:54:02.938 Executor task launch worker for task 13 DEBUG BlockManager: Getting local block rdd_20_4 20/01/23 18:54:02.938 Executor task launch worker for task 13 DEBUG BlockManager: Block rdd_20_4 was not found 20/01/23 18:54:02.938 Executor task launch worker for task 13 DEBUG BlockManager: Getting remote block rdd_20_4 20/01/23 18:54:02.939 Executor task launch worker for task 13 DEBUG BlockManager: Block rdd_20_4 is unknown by block manager master 20/01/23 18:54:02.939 Executor task launch worker for task 13 DEBUG BlockManager: Getting local block rdd_19_4 20/01/23 18:54:02.939 Executor task launch worker for task 13 DEBUG BlockManager: Block rdd_19_4 was not found 20/01/23 18:54:02.939 Executor task launch worker for task 13 DEBUG BlockManager: Getting remote block rdd_19_4 20/01/23 18:54:02.940 Executor task launch worker for task 10 DEBUG TaskMemoryManager: Task 10 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@3634f9a5 20/01/23 18:54:02.940 Executor task launch worker for task 9 DEBUG TaskMemoryManager: Task 9 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@18c91eb2 20/01/23 18:54:02.940 Executor task launch worker for task 10 INFO MemoryStore: Block rdd_19_1 stored as values in memory (estimated size 192.0 B, free 366.2 MiB) 20/01/23 18:54:02.941 Executor task launch worker for task 9 INFO MemoryStore: Block rdd_19_0 stored as values in memory (estimated size 192.0 B, free 366.2 MiB) 20/01/23 18:54:02.940 Executor task launch worker for task 11 DEBUG TaskMemoryManager: Task 11 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@5c9ae381 20/01/23 18:54:02.941 Executor task launch worker for task 11 INFO MemoryStore: Block rdd_19_2 stored as values in memory (estimated size 192.0 B, free 366.2 MiB) 20/01/23 18:54:02.943 Executor task launch worker for task 13 DEBUG BlockManager: Block rdd_19_4 is unknown by block manager master 20/01/23 18:54:02.943 Executor task launch worker for task 13 DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 0, partitions 4-5 20/01/23 18:54:02.943 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_19_2 in memory on 192.168.1.225:35863 (size: 192.0 B, free: 366.3 MiB) 20/01/23 18:54:02.943 Executor task launch worker for task 13 DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 20/01/23 18:54:02.944 Executor task launch worker for task 13 INFO ShuffleBlockFetcherIterator: Getting 0 (0.0 B) non-empty blocks including 0 (0.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 20/01/23 18:54:02.944 Executor task launch worker for task 13 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 20/01/23 18:54:02.944 Executor task launch worker for task 11 DEBUG BlockManagerMaster: Updated info of block rdd_19_2 20/01/23 18:54:02.944 Executor task launch worker for task 11 DEBUG BlockManager: Told master about block rdd_19_2 20/01/23 18:54:02.944 Executor task launch worker for task 13 DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: 20/01/23 18:54:02.944 Executor task launch worker for task 11 DEBUG BlockManager: Put block rdd_19_2 locally took 38 ms 20/01/23 18:54:02.944 Executor task launch worker for task 11 DEBUG BlockManager: Putting block rdd_19_2 without replication took 38 ms 20/01/23 18:54:02.944 Executor task launch worker for task 11 DEBUG BlockManager: Getting local block rdd_19_2 20/01/23 18:54:02.944 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_19_0 in memory on 192.168.1.225:35863 (size: 192.0 B, free: 366.3 MiB) 20/01/23 18:54:02.944 Executor task launch worker for task 11 DEBUG BlockManager: Level for block rdd_19_2 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:02.945 Executor task launch worker for task 9 DEBUG BlockManagerMaster: Updated info of block rdd_19_0 20/01/23 18:54:02.945 Executor task launch worker for task 9 DEBUG BlockManager: Told master about block rdd_19_0 20/01/23 18:54:02.946 Executor task launch worker for task 9 DEBUG BlockManager: Put block rdd_19_0 locally took 24 ms 20/01/23 18:54:02.946 Executor task launch worker for task 9 DEBUG BlockManager: Putting block rdd_19_0 without replication took 24 ms 20/01/23 18:54:02.946 Executor task launch worker for task 9 DEBUG BlockManager: Getting local block rdd_19_0 20/01/23 18:54:02.946 Executor task launch worker for task 9 DEBUG BlockManager: Level for block rdd_19_0 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:02.946 Executor task launch worker for task 11 INFO MemoryStore: Block rdd_20_2 stored as values in memory (estimated size 296.0 B, free 366.2 MiB) 20/01/23 18:54:02.947 Executor task launch worker for task 13 DEBUG ShuffleBlockFetcherIterator: Got local blocks in 3 ms 20/01/23 18:54:02.947 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_20_2 in memory on 192.168.1.225:35863 (size: 296.0 B, free: 366.3 MiB) 20/01/23 18:54:02.947 Executor task launch worker for task 11 DEBUG BlockManagerMaster: Updated info of block rdd_20_2 20/01/23 18:54:02.947 Executor task launch worker for task 11 DEBUG BlockManager: Told master about block rdd_20_2 20/01/23 18:54:02.947 Executor task launch worker for task 11 DEBUG BlockManager: Put block rdd_20_2 locally took 42 ms 20/01/23 18:54:02.947 Executor task launch worker for task 11 DEBUG BlockManager: Putting block rdd_20_2 without replication took 42 ms 20/01/23 18:54:02.947 Executor task launch worker for task 11 DEBUG BlockManager: Getting local block rdd_20_2 20/01/23 18:54:02.947 Executor task launch worker for task 11 DEBUG BlockManager: Level for block rdd_20_2 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:02.948 Executor task launch worker for task 9 INFO MemoryStore: Block rdd_20_0 stored as values in memory (estimated size 296.0 B, free 366.2 MiB) 20/01/23 18:54:02.948 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_20_0 in memory on 192.168.1.225:35863 (size: 296.0 B, free: 366.3 MiB) 20/01/23 18:54:02.949 Executor task launch worker for task 11 INFO Executor: Finished task 2.0 in stage 3.0 (TID 11). 2222 bytes result sent to driver 20/01/23 18:54:02.949 Executor task launch worker for task 11 DEBUG ExecutorMetricsPoller: stageTCMP: (3, 0) -> 3 20/01/23 18:54:02.949 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_19_1 in memory on 192.168.1.225:35863 (size: 192.0 B, free: 366.3 MiB) 20/01/23 18:54:02.949 Executor task launch worker for task 10 DEBUG BlockManagerMaster: Updated info of block rdd_19_1 20/01/23 18:54:02.949 Executor task launch worker for task 10 DEBUG BlockManager: Told master about block rdd_19_1 20/01/23 18:54:02.950 Executor task launch worker for task 10 DEBUG BlockManager: Put block rdd_19_1 locally took 45 ms 20/01/23 18:54:02.950 Executor task launch worker for task 10 DEBUG BlockManager: Putting block rdd_19_1 without replication took 46 ms 20/01/23 18:54:02.950 Executor task launch worker for task 10 DEBUG BlockManager: Getting local block rdd_19_1 20/01/23 18:54:02.950 Executor task launch worker for task 9 DEBUG BlockManagerMaster: Updated info of block rdd_20_0 20/01/23 18:54:02.950 Executor task launch worker for task 9 DEBUG BlockManager: Told master about block rdd_20_0 20/01/23 18:54:02.949 dispatcher-event-loop-0 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_3.0, runningTasks: 3 20/01/23 18:54:02.950 Executor task launch worker for task 9 DEBUG BlockManager: Put block rdd_20_0 locally took 29 ms 20/01/23 18:54:02.950 Executor task launch worker for task 10 DEBUG BlockManager: Level for block rdd_19_1 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:02.950 Executor task launch worker for task 9 DEBUG BlockManager: Putting block rdd_20_0 without replication took 29 ms 20/01/23 18:54:02.950 Executor task launch worker for task 9 DEBUG BlockManager: Getting local block rdd_20_0 20/01/23 18:54:02.950 Executor task launch worker for task 9 DEBUG BlockManager: Level for block rdd_20_0 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:02.950 dispatcher-event-loop-0 INFO TaskSetManager: Starting task 5.0 in stage 3.0 (TID 14, 192.168.1.225, executor driver, partition 5, PROCESS_LOCAL, 7143 bytes) 20/01/23 18:54:02.951 task-result-getter-1 INFO TaskSetManager: Finished task 2.0 in stage 3.0 (TID 11) in 70 ms on 192.168.1.225 (executor driver) (2/10) 20/01/23 18:54:02.950 Executor task launch worker for task 13 DEBUG TaskMemoryManager: Task 13 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@66bee35d 20/01/23 18:54:02.951 Executor task launch worker for task 14 INFO Executor: Running task 5.0 in stage 3.0 (TID 14) 20/01/23 18:54:02.951 Executor task launch worker for task 10 INFO MemoryStore: Block rdd_20_1 stored as values in memory (estimated size 296.0 B, free 366.2 MiB) 20/01/23 18:54:02.951 Executor task launch worker for task 13 INFO MemoryStore: Block rdd_19_4 stored as values in memory (estimated size 16.0 B, free 366.2 MiB) 20/01/23 18:54:02.952 Executor task launch worker for task 14 DEBUG ExecutorMetricsPoller: stageTCMP: (3, 0) -> 4 20/01/23 18:54:02.953 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_20_1 in memory on 192.168.1.225:35863 (size: 296.0 B, free: 366.3 MiB) 20/01/23 18:54:02.953 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_19_4 in memory on 192.168.1.225:35863 (size: 16.0 B, free: 366.3 MiB) 20/01/23 18:54:02.953 Executor task launch worker for task 10 DEBUG BlockManagerMaster: Updated info of block rdd_20_1 20/01/23 18:54:02.953 Executor task launch worker for task 10 DEBUG BlockManager: Told master about block rdd_20_1 20/01/23 18:54:02.953 Executor task launch worker for task 10 DEBUG BlockManager: Put block rdd_20_1 locally took 50 ms 20/01/23 18:54:02.954 Executor task launch worker for task 10 DEBUG BlockManager: Putting block rdd_20_1 without replication took 51 ms 20/01/23 18:54:02.954 Executor task launch worker for task 10 DEBUG BlockManager: Getting local block rdd_20_1 20/01/23 18:54:02.954 Executor task launch worker for task 10 DEBUG BlockManager: Level for block rdd_20_1 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:02.954 Executor task launch worker for task 13 DEBUG BlockManagerMaster: Updated info of block rdd_19_4 20/01/23 18:54:02.954 Executor task launch worker for task 13 DEBUG BlockManager: Told master about block rdd_19_4 20/01/23 18:54:02.954 Executor task launch worker for task 13 DEBUG BlockManager: Put block rdd_19_4 locally took 11 ms 20/01/23 18:54:02.954 Executor task launch worker for task 13 DEBUG BlockManager: Putting block rdd_19_4 without replication took 11 ms 20/01/23 18:54:02.954 Executor task launch worker for task 13 DEBUG BlockManager: Getting local block rdd_19_4 20/01/23 18:54:02.955 Executor task launch worker for task 13 DEBUG BlockManager: Level for block rdd_19_4 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:02.955 Executor task launch worker for task 10 INFO Executor: Finished task 1.0 in stage 3.0 (TID 10). 2222 bytes result sent to driver 20/01/23 18:54:02.955 Executor task launch worker for task 13 INFO MemoryStore: Block rdd_20_4 stored as values in memory (estimated size 16.0 B, free 366.2 MiB) 20/01/23 18:54:02.956 dispatcher-event-loop-1 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_3.0, runningTasks: 3 20/01/23 18:54:02.954 Executor task launch worker for task 9 INFO Executor: Finished task 0.0 in stage 3.0 (TID 9). 2222 bytes result sent to driver 20/01/23 18:54:02.956 dispatcher-event-loop-1 INFO TaskSetManager: Starting task 6.0 in stage 3.0 (TID 15, 192.168.1.225, executor driver, partition 6, PROCESS_LOCAL, 7143 bytes) 20/01/23 18:54:02.957 task-result-getter-0 INFO TaskSetManager: Finished task 1.0 in stage 3.0 (TID 10) in 76 ms on 192.168.1.225 (executor driver) (3/10) 20/01/23 18:54:02.957 Executor task launch worker for task 10 DEBUG ExecutorMetricsPoller: stageTCMP: (3, 0) -> 3 20/01/23 18:54:02.957 Executor task launch worker for task 9 DEBUG ExecutorMetricsPoller: stageTCMP: (3, 0) -> 2 20/01/23 18:54:02.960 Executor task launch worker for task 14 DEBUG BlockManager: Getting local block rdd_20_5 20/01/23 18:54:02.960 Executor task launch worker for task 14 DEBUG BlockManager: Block rdd_20_5 was not found 20/01/23 18:54:02.960 Executor task launch worker for task 14 DEBUG BlockManager: Getting remote block rdd_20_5 20/01/23 18:54:02.962 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_20_4 in memory on 192.168.1.225:35863 (size: 16.0 B, free: 366.3 MiB) 20/01/23 18:54:02.962 Executor task launch worker for task 13 DEBUG BlockManagerMaster: Updated info of block rdd_20_4 20/01/23 18:54:02.963 Executor task launch worker for task 13 DEBUG BlockManager: Told master about block rdd_20_4 20/01/23 18:54:02.963 Executor task launch worker for task 13 DEBUG BlockManager: Put block rdd_20_4 locally took 23 ms 20/01/23 18:54:02.963 Executor task launch worker for task 13 DEBUG BlockManager: Putting block rdd_20_4 without replication took 23 ms 20/01/23 18:54:02.963 Executor task launch worker for task 13 DEBUG BlockManager: Getting local block rdd_20_4 20/01/23 18:54:02.963 Executor task launch worker for task 13 DEBUG BlockManager: Level for block rdd_20_4 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:02.963 Executor task launch worker for task 14 DEBUG BlockManager: Block rdd_20_5 is unknown by block manager master 20/01/23 18:54:02.963 Executor task launch worker for task 14 DEBUG BlockManager: Getting local block rdd_19_5 20/01/23 18:54:02.963 Executor task launch worker for task 14 DEBUG BlockManager: Block rdd_19_5 was not found 20/01/23 18:54:02.963 Executor task launch worker for task 14 DEBUG BlockManager: Getting remote block rdd_19_5 20/01/23 18:54:02.964 Executor task launch worker for task 14 DEBUG BlockManager: Block rdd_19_5 is unknown by block manager master 20/01/23 18:54:02.964 Executor task launch worker for task 14 DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 0, partitions 5-6 20/01/23 18:54:02.964 Executor task launch worker for task 13 INFO Executor: Finished task 4.0 in stage 3.0 (TID 13). 2222 bytes result sent to driver 20/01/23 18:54:02.964 Executor task launch worker for task 14 DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 20/01/23 18:54:02.964 Executor task launch worker for task 13 DEBUG ExecutorMetricsPoller: stageTCMP: (3, 0) -> 1 20/01/23 18:54:02.964 Executor task launch worker for task 14 INFO ShuffleBlockFetcherIterator: Getting 0 (0.0 B) non-empty blocks including 0 (0.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 20/01/23 18:54:02.965 Executor task launch worker for task 14 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 20/01/23 18:54:02.965 Executor task launch worker for task 14 DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: 20/01/23 18:54:02.965 Executor task launch worker for task 14 DEBUG ShuffleBlockFetcherIterator: Got local blocks in 0 ms 20/01/23 18:54:02.965 dispatcher-event-loop-1 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_3.0, runningTasks: 3 20/01/23 18:54:02.965 Executor task launch worker for task 14 DEBUG TaskMemoryManager: Task 14 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@761cb3b1 20/01/23 18:54:02.966 dispatcher-event-loop-1 INFO TaskSetManager: Starting task 7.0 in stage 3.0 (TID 16, 192.168.1.225, executor driver, partition 7, PROCESS_LOCAL, 7143 bytes) 20/01/23 18:54:02.966 Executor task launch worker for task 14 INFO MemoryStore: Block rdd_19_5 stored as values in memory (estimated size 16.0 B, free 366.2 MiB) 20/01/23 18:54:02.966 task-result-getter-2 INFO TaskSetManager: Finished task 0.0 in stage 3.0 (TID 9) in 86 ms on 192.168.1.225 (executor driver) (4/10) 20/01/23 18:54:02.967 task-result-getter-2 INFO TaskSetManager: Finished task 4.0 in stage 3.0 (TID 13) in 51 ms on 192.168.1.225 (executor driver) (5/10) 20/01/23 18:54:02.967 Executor task launch worker for task 16 INFO Executor: Running task 7.0 in stage 3.0 (TID 16) 20/01/23 18:54:02.967 Executor task launch worker for task 16 DEBUG ExecutorMetricsPoller: stageTCMP: (3, 0) -> 2 20/01/23 18:54:02.969 dispatcher-event-loop-1 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_3.0, runningTasks: 3 20/01/23 18:54:02.969 dispatcher-event-loop-1 INFO TaskSetManager: Starting task 8.0 in stage 3.0 (TID 17, 192.168.1.225, executor driver, partition 8, PROCESS_LOCAL, 7143 bytes) 20/01/23 18:54:02.970 Executor task launch worker for task 15 INFO Executor: Running task 6.0 in stage 3.0 (TID 15) 20/01/23 18:54:02.970 Executor task launch worker for task 15 DEBUG ExecutorMetricsPoller: stageTCMP: (3, 0) -> 3 20/01/23 18:54:02.970 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_19_5 in memory on 192.168.1.225:35863 (size: 16.0 B, free: 366.3 MiB) 20/01/23 18:54:02.971 Executor task launch worker for task 14 DEBUG BlockManagerMaster: Updated info of block rdd_19_5 20/01/23 18:54:02.971 Executor task launch worker for task 14 DEBUG BlockManager: Told master about block rdd_19_5 20/01/23 18:54:02.971 Executor task launch worker for task 14 DEBUG BlockManager: Put block rdd_19_5 locally took 7 ms 20/01/23 18:54:02.971 Executor task launch worker for task 14 DEBUG BlockManager: Putting block rdd_19_5 without replication took 7 ms 20/01/23 18:54:02.971 Executor task launch worker for task 14 DEBUG BlockManager: Getting local block rdd_19_5 20/01/23 18:54:02.971 Executor task launch worker for task 14 DEBUG BlockManager: Level for block rdd_19_5 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:02.972 Executor task launch worker for task 14 INFO MemoryStore: Block rdd_20_5 stored as values in memory (estimated size 16.0 B, free 366.2 MiB) 20/01/23 18:54:02.973 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_20_5 in memory on 192.168.1.225:35863 (size: 16.0 B, free: 366.3 MiB) 20/01/23 18:54:02.973 Executor task launch worker for task 14 DEBUG BlockManagerMaster: Updated info of block rdd_20_5 20/01/23 18:54:02.973 Executor task launch worker for task 14 DEBUG BlockManager: Told master about block rdd_20_5 20/01/23 18:54:02.973 Executor task launch worker for task 14 DEBUG BlockManager: Put block rdd_20_5 locally took 10 ms 20/01/23 18:54:02.973 Executor task launch worker for task 14 DEBUG BlockManager: Putting block rdd_20_5 without replication took 10 ms 20/01/23 18:54:02.974 Executor task launch worker for task 14 DEBUG BlockManager: Getting local block rdd_20_5 20/01/23 18:54:02.974 Executor task launch worker for task 14 DEBUG BlockManager: Level for block rdd_20_5 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:02.974 Executor task launch worker for task 17 INFO Executor: Running task 8.0 in stage 3.0 (TID 17) 20/01/23 18:54:02.974 Executor task launch worker for task 17 DEBUG ExecutorMetricsPoller: stageTCMP: (3, 0) -> 4 20/01/23 18:54:02.975 Executor task launch worker for task 14 INFO Executor: Finished task 5.0 in stage 3.0 (TID 14). 2222 bytes result sent to driver 20/01/23 18:54:02.975 Executor task launch worker for task 14 DEBUG ExecutorMetricsPoller: stageTCMP: (3, 0) -> 3 20/01/23 18:54:02.976 dispatcher-event-loop-0 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_3.0, runningTasks: 3 20/01/23 18:54:02.976 dispatcher-event-loop-0 INFO TaskSetManager: Starting task 9.0 in stage 3.0 (TID 18, 192.168.1.225, executor driver, partition 9, PROCESS_LOCAL, 7143 bytes) 20/01/23 18:54:02.977 task-result-getter-1 INFO TaskSetManager: Finished task 5.0 in stage 3.0 (TID 14) in 26 ms on 192.168.1.225 (executor driver) (6/10) 20/01/23 18:54:02.977 Executor task launch worker for task 18 INFO Executor: Running task 9.0 in stage 3.0 (TID 18) 20/01/23 18:54:02.977 Executor task launch worker for task 18 DEBUG ExecutorMetricsPoller: stageTCMP: (3, 0) -> 4 20/01/23 18:54:02.989 Executor task launch worker for task 16 DEBUG BlockManager: Getting local block rdd_20_7 20/01/23 18:54:02.989 Executor task launch worker for task 16 DEBUG BlockManager: Block rdd_20_7 was not found 20/01/23 18:54:02.990 Executor task launch worker for task 16 DEBUG BlockManager: Getting remote block rdd_20_7 20/01/23 18:54:03.001 Executor task launch worker for task 17 DEBUG BlockManager: Getting local block rdd_20_8 20/01/23 18:54:03.001 Executor task launch worker for task 17 DEBUG BlockManager: Block rdd_20_8 was not found 20/01/23 18:54:03.001 Executor task launch worker for task 17 DEBUG BlockManager: Getting remote block rdd_20_8 20/01/23 18:54:03.003 Executor task launch worker for task 15 DEBUG BlockManager: Getting local block rdd_20_6 20/01/23 18:54:03.003 Executor task launch worker for task 15 DEBUG BlockManager: Block rdd_20_6 was not found 20/01/23 18:54:03.003 Executor task launch worker for task 15 DEBUG BlockManager: Getting remote block rdd_20_6 20/01/23 18:54:03.005 Executor task launch worker for task 16 DEBUG BlockManager: Block rdd_20_7 is unknown by block manager master 20/01/23 18:54:03.005 Executor task launch worker for task 15 DEBUG BlockManager: Block rdd_20_6 is unknown by block manager master 20/01/23 18:54:03.005 Executor task launch worker for task 16 DEBUG BlockManager: Getting local block rdd_19_7 20/01/23 18:54:03.006 Executor task launch worker for task 16 DEBUG BlockManager: Block rdd_19_7 was not found 20/01/23 18:54:03.006 Executor task launch worker for task 15 DEBUG BlockManager: Getting local block rdd_19_6 20/01/23 18:54:03.006 Executor task launch worker for task 16 DEBUG BlockManager: Getting remote block rdd_19_7 20/01/23 18:54:03.006 Executor task launch worker for task 15 DEBUG BlockManager: Block rdd_19_6 was not found 20/01/23 18:54:03.006 Executor task launch worker for task 15 DEBUG BlockManager: Getting remote block rdd_19_6 20/01/23 18:54:03.006 Executor task launch worker for task 15 DEBUG BlockManager: Block rdd_19_6 is unknown by block manager master 20/01/23 18:54:03.006 Executor task launch worker for task 16 DEBUG BlockManager: Block rdd_19_7 is unknown by block manager master 20/01/23 18:54:03.006 Executor task launch worker for task 15 DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 0, partitions 6-7 20/01/23 18:54:03.006 Executor task launch worker for task 16 DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 0, partitions 7-8 20/01/23 18:54:03.007 Executor task launch worker for task 15 DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 20/01/23 18:54:03.007 Executor task launch worker for task 16 DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 20/01/23 18:54:03.007 Executor task launch worker for task 15 INFO ShuffleBlockFetcherIterator: Getting 0 (0.0 B) non-empty blocks including 0 (0.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 20/01/23 18:54:03.007 Executor task launch worker for task 16 INFO ShuffleBlockFetcherIterator: Getting 0 (0.0 B) non-empty blocks including 0 (0.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 20/01/23 18:54:03.007 Executor task launch worker for task 15 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 20/01/23 18:54:03.007 Executor task launch worker for task 16 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 20/01/23 18:54:03.007 Executor task launch worker for task 15 DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: 20/01/23 18:54:03.007 Executor task launch worker for task 16 DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: 20/01/23 18:54:03.007 Executor task launch worker for task 15 DEBUG ShuffleBlockFetcherIterator: Got local blocks in 0 ms 20/01/23 18:54:03.007 Executor task launch worker for task 16 DEBUG ShuffleBlockFetcherIterator: Got local blocks in 0 ms 20/01/23 18:54:03.008 Executor task launch worker for task 15 DEBUG TaskMemoryManager: Task 15 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@471611cc 20/01/23 18:54:03.008 Executor task launch worker for task 16 DEBUG TaskMemoryManager: Task 16 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@32f7c2a8 20/01/23 18:54:03.009 Executor task launch worker for task 15 INFO MemoryStore: Block rdd_19_6 stored as values in memory (estimated size 16.0 B, free 366.2 MiB) 20/01/23 18:54:03.009 Executor task launch worker for task 16 INFO MemoryStore: Block rdd_19_7 stored as values in memory (estimated size 16.0 B, free 366.2 MiB) 20/01/23 18:54:03.010 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_19_6 in memory on 192.168.1.225:35863 (size: 16.0 B, free: 366.3 MiB) 20/01/23 18:54:03.010 Executor task launch worker for task 15 DEBUG BlockManagerMaster: Updated info of block rdd_19_6 20/01/23 18:54:03.010 Executor task launch worker for task 15 DEBUG BlockManager: Told master about block rdd_19_6 20/01/23 18:54:03.010 Executor task launch worker for task 15 DEBUG BlockManager: Put block rdd_19_6 locally took 3 ms 20/01/23 18:54:03.010 Executor task launch worker for task 15 DEBUG BlockManager: Putting block rdd_19_6 without replication took 3 ms 20/01/23 18:54:03.010 Executor task launch worker for task 15 DEBUG BlockManager: Getting local block rdd_19_6 20/01/23 18:54:03.010 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_19_7 in memory on 192.168.1.225:35863 (size: 16.0 B, free: 366.3 MiB) 20/01/23 18:54:03.010 Executor task launch worker for task 15 DEBUG BlockManager: Level for block rdd_19_6 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:03.011 Executor task launch worker for task 18 DEBUG BlockManager: Getting local block rdd_20_9 20/01/23 18:54:03.011 Executor task launch worker for task 17 DEBUG BlockManager: Block rdd_20_8 is unknown by block manager master 20/01/23 18:54:03.011 Executor task launch worker for task 15 INFO MemoryStore: Block rdd_20_6 stored as values in memory (estimated size 16.0 B, free 366.2 MiB) 20/01/23 18:54:03.011 Executor task launch worker for task 17 DEBUG BlockManager: Getting local block rdd_19_8 20/01/23 18:54:03.011 Executor task launch worker for task 17 DEBUG BlockManager: Block rdd_19_8 was not found 20/01/23 18:54:03.011 Executor task launch worker for task 17 DEBUG BlockManager: Getting remote block rdd_19_8 20/01/23 18:54:03.011 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_20_6 in memory on 192.168.1.225:35863 (size: 16.0 B, free: 366.3 MiB) 20/01/23 18:54:03.012 Executor task launch worker for task 15 DEBUG BlockManagerMaster: Updated info of block rdd_20_6 20/01/23 18:54:03.012 Executor task launch worker for task 15 DEBUG BlockManager: Told master about block rdd_20_6 20/01/23 18:54:03.012 Executor task launch worker for task 15 DEBUG BlockManager: Put block rdd_20_6 locally took 6 ms 20/01/23 18:54:03.012 Executor task launch worker for task 15 DEBUG BlockManager: Putting block rdd_20_6 without replication took 6 ms 20/01/23 18:54:03.012 Executor task launch worker for task 15 DEBUG BlockManager: Getting local block rdd_20_6 20/01/23 18:54:03.012 Executor task launch worker for task 15 DEBUG BlockManager: Level for block rdd_20_6 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:03.013 Executor task launch worker for task 17 DEBUG BlockManager: Block rdd_19_8 is unknown by block manager master 20/01/23 18:54:03.013 Executor task launch worker for task 17 DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 0, partitions 8-9 20/01/23 18:54:03.013 Executor task launch worker for task 15 INFO Executor: Finished task 6.0 in stage 3.0 (TID 15). 2222 bytes result sent to driver 20/01/23 18:54:03.014 Executor task launch worker for task 15 DEBUG ExecutorMetricsPoller: stageTCMP: (3, 0) -> 3 20/01/23 18:54:03.014 dispatcher-event-loop-3 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_3.0, runningTasks: 3 20/01/23 18:54:03.014 Executor task launch worker for task 17 DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 20/01/23 18:54:03.015 Executor task launch worker for task 17 INFO ShuffleBlockFetcherIterator: Getting 0 (0.0 B) non-empty blocks including 0 (0.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 20/01/23 18:54:03.015 Executor task launch worker for task 17 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 1 ms 20/01/23 18:54:03.015 Executor task launch worker for task 17 DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: 20/01/23 18:54:03.015 Executor task launch worker for task 17 DEBUG ShuffleBlockFetcherIterator: Got local blocks in 1 ms 20/01/23 18:54:03.016 Executor task launch worker for task 17 DEBUG TaskMemoryManager: Task 17 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@1a613b8e 20/01/23 18:54:03.016 Executor task launch worker for task 17 INFO MemoryStore: Block rdd_19_8 stored as values in memory (estimated size 16.0 B, free 366.2 MiB) 20/01/23 18:54:03.017 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_19_8 in memory on 192.168.1.225:35863 (size: 16.0 B, free: 366.3 MiB) 20/01/23 18:54:03.017 Executor task launch worker for task 17 DEBUG BlockManagerMaster: Updated info of block rdd_19_8 20/01/23 18:54:03.017 Executor task launch worker for task 17 DEBUG BlockManager: Told master about block rdd_19_8 20/01/23 18:54:03.017 Executor task launch worker for task 17 DEBUG BlockManager: Put block rdd_19_8 locally took 4 ms 20/01/23 18:54:03.017 Executor task launch worker for task 17 DEBUG BlockManager: Putting block rdd_19_8 without replication took 4 ms 20/01/23 18:54:03.017 Executor task launch worker for task 17 DEBUG BlockManager: Getting local block rdd_19_8 20/01/23 18:54:03.018 Executor task launch worker for task 17 DEBUG BlockManager: Level for block rdd_19_8 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:03.018 Executor task launch worker for task 17 INFO MemoryStore: Block rdd_20_8 stored as values in memory (estimated size 16.0 B, free 366.2 MiB) 20/01/23 18:54:03.019 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_20_8 in memory on 192.168.1.225:35863 (size: 16.0 B, free: 366.3 MiB) 20/01/23 18:54:03.019 Executor task launch worker for task 17 DEBUG BlockManagerMaster: Updated info of block rdd_20_8 20/01/23 18:54:03.019 Executor task launch worker for task 17 DEBUG BlockManager: Told master about block rdd_20_8 20/01/23 18:54:03.019 Executor task launch worker for task 17 DEBUG BlockManager: Put block rdd_20_8 locally took 8 ms 20/01/23 18:54:03.019 Executor task launch worker for task 17 DEBUG BlockManager: Putting block rdd_20_8 without replication took 8 ms 20/01/23 18:54:03.019 Executor task launch worker for task 17 DEBUG BlockManager: Getting local block rdd_20_8 20/01/23 18:54:03.019 Executor task launch worker for task 17 DEBUG BlockManager: Level for block rdd_20_8 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:03.021 Executor task launch worker for task 17 INFO Executor: Finished task 8.0 in stage 3.0 (TID 17). 2222 bytes result sent to driver 20/01/23 18:54:03.021 Executor task launch worker for task 17 DEBUG ExecutorMetricsPoller: stageTCMP: (3, 0) -> 2 20/01/23 18:54:03.011 Executor task launch worker for task 18 DEBUG BlockManager: Block rdd_20_9 was not found 20/01/23 18:54:03.015 task-result-getter-0 INFO TaskSetManager: Finished task 6.0 in stage 3.0 (TID 15) in 59 ms on 192.168.1.225 (executor driver) (7/10) 20/01/23 18:54:03.011 Executor task launch worker for task 16 DEBUG BlockManagerMaster: Updated info of block rdd_19_7 20/01/23 18:54:03.027 Executor task launch worker for task 18 DEBUG BlockManager: Getting remote block rdd_20_9 20/01/23 18:54:03.027 Executor task launch worker for task 16 DEBUG BlockManager: Told master about block rdd_19_7 20/01/23 18:54:03.027 Executor task launch worker for task 16 DEBUG BlockManager: Put block rdd_19_7 locally took 20 ms 20/01/23 18:54:03.027 Executor task launch worker for task 16 DEBUG BlockManager: Putting block rdd_19_7 without replication took 20 ms 20/01/23 18:54:03.027 Executor task launch worker for task 16 DEBUG BlockManager: Getting local block rdd_19_7 20/01/23 18:54:03.027 Executor task launch worker for task 16 DEBUG BlockManager: Level for block rdd_19_7 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:03.028 dispatcher-event-loop-0 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_3.0, runningTasks: 2 20/01/23 18:54:03.028 Executor task launch worker for task 16 INFO MemoryStore: Block rdd_20_7 stored as values in memory (estimated size 16.0 B, free 366.2 MiB) 20/01/23 18:54:03.029 Executor task launch worker for task 18 DEBUG BlockManager: Block rdd_20_9 is unknown by block manager master 20/01/23 18:54:03.029 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_20_7 in memory on 192.168.1.225:35863 (size: 16.0 B, free: 366.3 MiB) 20/01/23 18:54:03.029 Executor task launch worker for task 18 DEBUG BlockManager: Getting local block rdd_19_9 20/01/23 18:54:03.029 Executor task launch worker for task 18 DEBUG BlockManager: Block rdd_19_9 was not found 20/01/23 18:54:03.029 Executor task launch worker for task 18 DEBUG BlockManager: Getting remote block rdd_19_9 20/01/23 18:54:03.029 Executor task launch worker for task 18 DEBUG BlockManager: Block rdd_19_9 is unknown by block manager master 20/01/23 18:54:03.029 task-result-getter-2 INFO TaskSetManager: Finished task 8.0 in stage 3.0 (TID 17) in 60 ms on 192.168.1.225 (executor driver) (8/10) 20/01/23 18:54:03.029 Executor task launch worker for task 18 DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 0, partitions 9-10 20/01/23 18:54:03.029 Executor task launch worker for task 16 DEBUG BlockManagerMaster: Updated info of block rdd_20_7 20/01/23 18:54:03.029 Executor task launch worker for task 16 DEBUG BlockManager: Told master about block rdd_20_7 20/01/23 18:54:03.029 Executor task launch worker for task 16 DEBUG BlockManager: Put block rdd_20_7 locally took 24 ms 20/01/23 18:54:03.030 Executor task launch worker for task 16 DEBUG BlockManager: Putting block rdd_20_7 without replication took 24 ms 20/01/23 18:54:03.030 Executor task launch worker for task 16 DEBUG BlockManager: Getting local block rdd_20_7 20/01/23 18:54:03.030 Executor task launch worker for task 16 DEBUG BlockManager: Level for block rdd_20_7 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:03.030 Executor task launch worker for task 18 DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 20/01/23 18:54:03.030 Executor task launch worker for task 18 INFO ShuffleBlockFetcherIterator: Getting 0 (0.0 B) non-empty blocks including 0 (0.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 20/01/23 18:54:03.030 Executor task launch worker for task 18 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 20/01/23 18:54:03.030 Executor task launch worker for task 18 DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: 20/01/23 18:54:03.030 Executor task launch worker for task 18 DEBUG ShuffleBlockFetcherIterator: Got local blocks in 0 ms 20/01/23 18:54:03.031 Executor task launch worker for task 16 INFO Executor: Finished task 7.0 in stage 3.0 (TID 16). 2222 bytes result sent to driver 20/01/23 18:54:03.031 Executor task launch worker for task 18 DEBUG TaskMemoryManager: Task 18 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@3efaa9da 20/01/23 18:54:03.031 Executor task launch worker for task 16 DEBUG ExecutorMetricsPoller: stageTCMP: (3, 0) -> 1 20/01/23 18:54:03.031 Executor task launch worker for task 18 INFO MemoryStore: Block rdd_19_9 stored as values in memory (estimated size 16.0 B, free 366.2 MiB) 20/01/23 18:54:03.031 dispatcher-event-loop-2 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_3.0, runningTasks: 1 20/01/23 18:54:03.032 task-result-getter-3 INFO TaskSetManager: Finished task 7.0 in stage 3.0 (TID 16) in 66 ms on 192.168.1.225 (executor driver) (9/10) 20/01/23 18:54:03.033 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_19_9 in memory on 192.168.1.225:35863 (size: 16.0 B, free: 366.3 MiB) 20/01/23 18:54:03.033 Executor task launch worker for task 18 DEBUG BlockManagerMaster: Updated info of block rdd_19_9 20/01/23 18:54:03.033 Executor task launch worker for task 18 DEBUG BlockManager: Told master about block rdd_19_9 20/01/23 18:54:03.033 Executor task launch worker for task 18 DEBUG BlockManager: Put block rdd_19_9 locally took 4 ms 20/01/23 18:54:03.033 Executor task launch worker for task 18 DEBUG BlockManager: Putting block rdd_19_9 without replication took 4 ms 20/01/23 18:54:03.033 Executor task launch worker for task 18 DEBUG BlockManager: Getting local block rdd_19_9 20/01/23 18:54:03.034 Executor task launch worker for task 18 DEBUG BlockManager: Level for block rdd_19_9 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:03.034 Executor task launch worker for task 18 INFO MemoryStore: Block rdd_20_9 stored as values in memory (estimated size 16.0 B, free 366.2 MiB) 20/01/23 18:54:03.035 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_20_9 in memory on 192.168.1.225:35863 (size: 16.0 B, free: 366.3 MiB) 20/01/23 18:54:03.035 Executor task launch worker for task 18 DEBUG BlockManagerMaster: Updated info of block rdd_20_9 20/01/23 18:54:03.035 Executor task launch worker for task 18 DEBUG BlockManager: Told master about block rdd_20_9 20/01/23 18:54:03.035 Executor task launch worker for task 18 DEBUG BlockManager: Put block rdd_20_9 locally took 6 ms 20/01/23 18:54:03.035 Executor task launch worker for task 18 DEBUG BlockManager: Putting block rdd_20_9 without replication took 6 ms 20/01/23 18:54:03.036 Executor task launch worker for task 18 DEBUG BlockManager: Getting local block rdd_20_9 20/01/23 18:54:03.036 Executor task launch worker for task 18 DEBUG BlockManager: Level for block rdd_20_9 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:03.037 Executor task launch worker for task 18 INFO Executor: Finished task 9.0 in stage 3.0 (TID 18). 2222 bytes result sent to driver 20/01/23 18:54:03.037 dispatcher-event-loop-1 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_3.0, runningTasks: 0 20/01/23 18:54:03.038 Executor task launch worker for task 18 DEBUG ExecutorMetricsPoller: removing (3, 0) from stageTCMP 20/01/23 18:54:03.038 task-result-getter-1 INFO TaskSetManager: Finished task 9.0 in stage 3.0 (TID 18) in 62 ms on 192.168.1.225 (executor driver) (10/10) 20/01/23 18:54:03.038 task-result-getter-1 INFO TaskSchedulerImpl: Removed TaskSet 3.0, whose tasks have all completed, from pool 20/01/23 18:54:03.039 dag-scheduler-event-loop INFO DAGScheduler: ResultStage 3 (count at ALS.scala:944) finished in 0.171 s 20/01/23 18:54:03.039 dag-scheduler-event-loop DEBUG DAGScheduler: After removal of stage 2, remaining stages = 2 20/01/23 18:54:03.040 dag-scheduler-event-loop DEBUG DAGScheduler: After removal of stage 1, remaining stages = 1 20/01/23 18:54:03.040 dag-scheduler-event-loop DEBUG DAGScheduler: After removal of stage 3, remaining stages = 0 20/01/23 18:54:03.040 dag-scheduler-event-loop INFO DAGScheduler: Job 1 is finished. Cancelling potential speculative or zombie tasks for this job 20/01/23 18:54:03.040 dag-scheduler-event-loop INFO TaskSchedulerImpl: Killing all running tasks in stage 3: Stage finished 20/01/23 18:54:03.040 Thread-4 INFO DAGScheduler: Job 1 finished: count at ALS.scala:944, took 1.504291 s 20/01/23 18:54:03.042 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$train$3 20/01/23 18:54:03.044 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$train$3) is now cleaned +++ 20/01/23 18:54:03.046 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$makeBlocks$1 20/01/23 18:54:03.048 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$makeBlocks$1) is now cleaned +++ 20/01/23 18:54:03.049 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$groupByKey$2 20/01/23 18:54:03.053 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$groupByKey$2) is now cleaned +++ 20/01/23 18:54:03.054 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$groupByKey$3 20/01/23 18:54:03.057 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$groupByKey$3) is now cleaned +++ 20/01/23 18:54:03.058 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$groupByKey$4 20/01/23 18:54:03.061 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$groupByKey$4) is now cleaned +++ 20/01/23 18:54:03.062 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$makeBlocks$5 20/01/23 18:54:03.064 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$makeBlocks$5) is now cleaned +++ 20/01/23 18:54:03.069 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$makeBlocks$7 20/01/23 18:54:03.071 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$makeBlocks$7) is now cleaned +++ 20/01/23 18:54:03.072 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$count$1$adapted 20/01/23 18:54:03.077 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$count$1$adapted) is now cleaned +++ 20/01/23 18:54:03.078 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$runJob$5 20/01/23 18:54:03.083 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$runJob$5) is now cleaned +++ 20/01/23 18:54:03.083 Thread-4 INFO SparkContext: Starting job: count at ALS.scala:951 20/01/23 18:54:03.086 dag-scheduler-event-loop INFO DAGScheduler: Registering RDD 22 (map at ALS.scala:1585) as input to shuffle 2 20/01/23 18:54:03.086 dag-scheduler-event-loop INFO DAGScheduler: Got job 2 (count at ALS.scala:951) with 10 output partitions 20/01/23 18:54:03.086 dag-scheduler-event-loop INFO DAGScheduler: Final stage: ResultStage 6 (count at ALS.scala:951) 20/01/23 18:54:03.086 dag-scheduler-event-loop INFO DAGScheduler: Parents of final stage: List(ShuffleMapStage 5) 20/01/23 18:54:03.087 dag-scheduler-event-loop INFO DAGScheduler: Missing parents: List(ShuffleMapStage 5) 20/01/23 18:54:03.087 dag-scheduler-event-loop DEBUG DAGScheduler: submitStage(ResultStage 6 (name=count at ALS.scala:951;jobs=2)) 20/01/23 18:54:03.088 dag-scheduler-event-loop DEBUG DAGScheduler: missing: List(ShuffleMapStage 5) 20/01/23 18:54:03.088 dag-scheduler-event-loop DEBUG DAGScheduler: submitStage(ShuffleMapStage 5 (name=map at ALS.scala:1585;jobs=2)) 20/01/23 18:54:03.089 dag-scheduler-event-loop DEBUG DAGScheduler: missing: List() 20/01/23 18:54:03.089 dag-scheduler-event-loop INFO DAGScheduler: Submitting ShuffleMapStage 5 (MapPartitionsRDD[22] at map at ALS.scala:1585), which has no missing parents 20/01/23 18:54:03.089 dag-scheduler-event-loop DEBUG DAGScheduler: submitMissingTasks(ShuffleMapStage 5) 20/01/23 18:54:03.093 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_4 stored as values in memory (estimated size 29.1 KiB, free 366.2 MiB) 20/01/23 18:54:03.093 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_4 locally took 1 ms 20/01/23 18:54:03.094 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_4 without replication took 1 ms 20/01/23 18:54:03.095 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_4_piece0 stored as bytes in memory (estimated size 13.9 KiB, free 366.2 MiB) 20/01/23 18:54:03.096 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added broadcast_4_piece0 in memory on 192.168.1.225:35863 (size: 13.9 KiB, free: 366.3 MiB) 20/01/23 18:54:03.096 dag-scheduler-event-loop DEBUG BlockManagerMaster: Updated info of block broadcast_4_piece0 20/01/23 18:54:03.096 dag-scheduler-event-loop DEBUG BlockManager: Told master about block broadcast_4_piece0 20/01/23 18:54:03.096 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_4_piece0 locally took 1 ms 20/01/23 18:54:03.096 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_4_piece0 without replication took 1 ms 20/01/23 18:54:03.097 dag-scheduler-event-loop INFO SparkContext: Created broadcast 4 from broadcast at DAGScheduler.scala:1200 20/01/23 18:54:03.098 dag-scheduler-event-loop INFO DAGScheduler: Submitting 4 missing tasks from ShuffleMapStage 5 (MapPartitionsRDD[22] at map at ALS.scala:1585) (first 15 tasks are for partitions Vector(0, 1, 2, 3)) 20/01/23 18:54:03.098 dag-scheduler-event-loop INFO TaskSchedulerImpl: Adding task set 5.0 with 4 tasks 20/01/23 18:54:03.099 dag-scheduler-event-loop DEBUG TaskSetManager: Epoch for TaskSet 5.0: 2 20/01/23 18:54:03.100 dag-scheduler-event-loop DEBUG TaskSetManager: Adding pending tasks took 0 ms 20/01/23 18:54:03.100 dag-scheduler-event-loop DEBUG TaskSetManager: Valid locality levels for TaskSet 5.0: PROCESS_LOCAL, NODE_LOCAL, ANY 20/01/23 18:54:03.101 dispatcher-event-loop-3 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_5.0, runningTasks: 0 20/01/23 18:54:03.102 dispatcher-event-loop-3 INFO TaskSetManager: Starting task 0.0 in stage 5.0 (TID 19, 192.168.1.225, executor driver, partition 0, PROCESS_LOCAL, 7132 bytes) 20/01/23 18:54:03.102 dispatcher-event-loop-3 INFO TaskSetManager: Starting task 1.0 in stage 5.0 (TID 20, 192.168.1.225, executor driver, partition 1, PROCESS_LOCAL, 7132 bytes) 20/01/23 18:54:03.102 dispatcher-event-loop-3 INFO TaskSetManager: Starting task 2.0 in stage 5.0 (TID 21, 192.168.1.225, executor driver, partition 2, PROCESS_LOCAL, 7132 bytes) 20/01/23 18:54:03.103 dispatcher-event-loop-3 INFO TaskSetManager: Starting task 3.0 in stage 5.0 (TID 22, 192.168.1.225, executor driver, partition 3, PROCESS_LOCAL, 7132 bytes) 20/01/23 18:54:03.103 Executor task launch worker for task 20 INFO Executor: Running task 1.0 in stage 5.0 (TID 20) 20/01/23 18:54:03.103 Executor task launch worker for task 21 INFO Executor: Running task 2.0 in stage 5.0 (TID 21) 20/01/23 18:54:03.103 Executor task launch worker for task 19 INFO Executor: Running task 0.0 in stage 5.0 (TID 19) 20/01/23 18:54:03.103 Executor task launch worker for task 22 INFO Executor: Running task 3.0 in stage 5.0 (TID 22) 20/01/23 18:54:03.103 Executor task launch worker for task 20 DEBUG ExecutorMetricsPoller: stageTCMP: (5, 0) -> 1 20/01/23 18:54:03.104 Executor task launch worker for task 22 DEBUG ExecutorMetricsPoller: stageTCMP: (5, 0) -> 2 20/01/23 18:54:03.104 Executor task launch worker for task 21 DEBUG ExecutorMetricsPoller: stageTCMP: (5, 0) -> 4 20/01/23 18:54:03.104 Executor task launch worker for task 19 DEBUG ExecutorMetricsPoller: stageTCMP: (5, 0) -> 3 20/01/23 18:54:03.104 Executor task launch worker for task 19 DEBUG BlockManager: Getting local block broadcast_4 20/01/23 18:54:03.105 Executor task launch worker for task 19 DEBUG BlockManager: Level for block broadcast_4 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:03.111 Executor task launch worker for task 19 DEBUG BlockManager: Getting local block rdd_16_0 20/01/23 18:54:03.111 Executor task launch worker for task 19 DEBUG BlockManager: Level for block rdd_16_0 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:03.111 Executor task launch worker for task 19 INFO BlockManager: Found block rdd_16_0 locally 20/01/23 18:54:03.122 Executor task launch worker for task 22 DEBUG BlockManager: Getting local block rdd_16_3 20/01/23 18:54:03.122 Executor task launch worker for task 22 DEBUG BlockManager: Level for block rdd_16_3 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:03.122 Executor task launch worker for task 22 INFO BlockManager: Found block rdd_16_3 locally 20/01/23 18:54:03.123 Executor task launch worker for task 20 DEBUG BlockManager: Getting local block rdd_16_1 20/01/23 18:54:03.123 Executor task launch worker for task 20 DEBUG BlockManager: Level for block rdd_16_1 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:03.123 Executor task launch worker for task 20 INFO BlockManager: Found block rdd_16_1 locally 20/01/23 18:54:03.131 Executor task launch worker for task 21 DEBUG BlockManager: Getting local block rdd_16_2 20/01/23 18:54:03.132 Executor task launch worker for task 21 DEBUG BlockManager: Level for block rdd_16_2 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:03.132 Executor task launch worker for task 21 INFO BlockManager: Found block rdd_16_2 locally 20/01/23 18:54:03.134 Executor task launch worker for task 19 DEBUG ALS: Converting to local indices took 0.001731241 seconds. 20/01/23 18:54:03.137 Executor task launch worker for task 22 DEBUG ALS: Converting to local indices took 0.001661881 seconds. 20/01/23 18:54:03.137 Executor task launch worker for task 19 INFO Executor: Finished task 0.0 in stage 5.0 (TID 19). 2045 bytes result sent to driver 20/01/23 18:54:03.141 Executor task launch worker for task 22 DEBUG ALS: Converting to local indices took 0.002378455 seconds. 20/01/23 18:54:03.142 Executor task launch worker for task 20 DEBUG ALS: Converting to local indices took 0.002403375 seconds. 20/01/23 18:54:03.143 Executor task launch worker for task 19 DEBUG ExecutorMetricsPoller: stageTCMP: (5, 0) -> 3 20/01/23 18:54:03.143 dispatcher-event-loop-0 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_5.0, runningTasks: 3 20/01/23 18:54:03.143 dispatcher-event-loop-0 DEBUG TaskSetManager: No tasks for locality level PROCESS_LOCAL, so moving to locality level NODE_LOCAL 20/01/23 18:54:03.143 dispatcher-event-loop-0 DEBUG TaskSetManager: No tasks for locality level NODE_LOCAL, so moving to locality level ANY 20/01/23 18:54:03.144 task-result-getter-0 INFO TaskSetManager: Finished task 0.0 in stage 5.0 (TID 19) in 43 ms on 192.168.1.225 (executor driver) (1/4) 20/01/23 18:54:03.146 Executor task launch worker for task 21 DEBUG ALS: Converting to local indices took 0.001511699 seconds. 20/01/23 18:54:03.148 Executor task launch worker for task 21 DEBUG ALS: Converting to local indices took 0.001109307 seconds. 20/01/23 18:54:03.150 dag-scheduler-event-loop DEBUG DAGScheduler: ShuffleMapTask finished on driver 20/01/23 18:54:03.152 Executor task launch worker for task 20 INFO Executor: Finished task 1.0 in stage 5.0 (TID 20). 2045 bytes result sent to driver 20/01/23 18:54:03.153 dispatcher-event-loop-2 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_5.0, runningTasks: 2 20/01/23 18:54:03.154 Executor task launch worker for task 20 DEBUG ExecutorMetricsPoller: stageTCMP: (5, 0) -> 2 20/01/23 18:54:03.155 Executor task launch worker for task 21 INFO Executor: Finished task 2.0 in stage 5.0 (TID 21). 2045 bytes result sent to driver 20/01/23 18:54:03.155 Executor task launch worker for task 21 DEBUG ExecutorMetricsPoller: stageTCMP: (5, 0) -> 1 20/01/23 18:54:03.156 dispatcher-event-loop-1 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_5.0, runningTasks: 1 20/01/23 18:54:03.156 task-result-getter-3 INFO TaskSetManager: Finished task 2.0 in stage 5.0 (TID 21) in 54 ms on 192.168.1.225 (executor driver) (2/4) 20/01/23 18:54:03.157 dag-scheduler-event-loop DEBUG DAGScheduler: ShuffleMapTask finished on driver 20/01/23 18:54:03.169 task-result-getter-2 INFO TaskSetManager: Finished task 1.0 in stage 5.0 (TID 20) in 66 ms on 192.168.1.225 (executor driver) (3/4) 20/01/23 18:54:03.170 dag-scheduler-event-loop DEBUG DAGScheduler: ShuffleMapTask finished on driver 20/01/23 18:54:03.170 Executor task launch worker for task 22 INFO Executor: Finished task 3.0 in stage 5.0 (TID 22). 2045 bytes result sent to driver 20/01/23 18:54:03.170 Executor task launch worker for task 22 DEBUG ExecutorMetricsPoller: removing (5, 0) from stageTCMP 20/01/23 18:54:03.171 dispatcher-event-loop-3 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_5.0, runningTasks: 0 20/01/23 18:54:03.171 task-result-getter-1 INFO TaskSetManager: Finished task 3.0 in stage 5.0 (TID 22) in 69 ms on 192.168.1.225 (executor driver) (4/4) 20/01/23 18:54:03.171 task-result-getter-1 INFO TaskSchedulerImpl: Removed TaskSet 5.0, whose tasks have all completed, from pool 20/01/23 18:54:03.172 dag-scheduler-event-loop DEBUG DAGScheduler: ShuffleMapTask finished on driver 20/01/23 18:54:03.172 dag-scheduler-event-loop INFO DAGScheduler: ShuffleMapStage 5 (map at ALS.scala:1585) finished in 0.082 s 20/01/23 18:54:03.172 dag-scheduler-event-loop INFO DAGScheduler: looking for newly runnable stages 20/01/23 18:54:03.173 dag-scheduler-event-loop INFO DAGScheduler: running: Set() 20/01/23 18:54:03.173 dag-scheduler-event-loop INFO DAGScheduler: waiting: Set(ResultStage 6) 20/01/23 18:54:03.173 dag-scheduler-event-loop INFO DAGScheduler: failed: Set() 20/01/23 18:54:03.173 dag-scheduler-event-loop DEBUG MapOutputTrackerMaster: Increasing epoch to 3 20/01/23 18:54:03.173 dag-scheduler-event-loop DEBUG DAGScheduler: submitStage(ResultStage 6 (name=count at ALS.scala:951;jobs=2)) 20/01/23 18:54:03.174 dag-scheduler-event-loop DEBUG DAGScheduler: missing: List() 20/01/23 18:54:03.174 dag-scheduler-event-loop INFO DAGScheduler: Submitting ResultStage 6 (itemOutBlocks MapPartitionsRDD[25] at mapValues at ALS.scala:1622), which has no missing parents 20/01/23 18:54:03.174 dag-scheduler-event-loop DEBUG DAGScheduler: submitMissingTasks(ResultStage 6) 20/01/23 18:54:03.179 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_5 stored as values in memory (estimated size 30.4 KiB, free 366.1 MiB) 20/01/23 18:54:03.180 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_5 locally took 1 ms 20/01/23 18:54:03.180 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_5 without replication took 1 ms 20/01/23 18:54:03.181 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_5_piece0 stored as bytes in memory (estimated size 14.3 KiB, free 366.1 MiB) 20/01/23 18:54:03.182 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added broadcast_5_piece0 in memory on 192.168.1.225:35863 (size: 14.3 KiB, free: 366.2 MiB) 20/01/23 18:54:03.182 dag-scheduler-event-loop DEBUG BlockManagerMaster: Updated info of block broadcast_5_piece0 20/01/23 18:54:03.182 dag-scheduler-event-loop DEBUG BlockManager: Told master about block broadcast_5_piece0 20/01/23 18:54:03.182 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_5_piece0 locally took 1 ms 20/01/23 18:54:03.182 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_5_piece0 without replication took 1 ms 20/01/23 18:54:03.182 dag-scheduler-event-loop INFO SparkContext: Created broadcast 5 from broadcast at DAGScheduler.scala:1200 20/01/23 18:54:03.183 dag-scheduler-event-loop INFO DAGScheduler: Submitting 10 missing tasks from ResultStage 6 (itemOutBlocks MapPartitionsRDD[25] at mapValues at ALS.scala:1622) (first 15 tasks are for partitions Vector(0, 1, 2, 3, 4, 5, 6, 7, 8, 9)) 20/01/23 18:54:03.183 dag-scheduler-event-loop INFO TaskSchedulerImpl: Adding task set 6.0 with 10 tasks 20/01/23 18:54:03.184 dag-scheduler-event-loop DEBUG TaskSetManager: Epoch for TaskSet 6.0: 3 20/01/23 18:54:03.184 dag-scheduler-event-loop DEBUG TaskSetManager: Adding pending tasks took 0 ms 20/01/23 18:54:03.184 dag-scheduler-event-loop DEBUG TaskSetManager: Valid locality levels for TaskSet 6.0: NODE_LOCAL, NO_PREF, ANY 20/01/23 18:54:03.184 dispatcher-event-loop-0 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_6.0, runningTasks: 0 20/01/23 18:54:03.185 dispatcher-event-loop-0 INFO TaskSetManager: Starting task 0.0 in stage 6.0 (TID 23, 192.168.1.225, executor driver, partition 0, NODE_LOCAL, 7143 bytes) 20/01/23 18:54:03.185 dispatcher-event-loop-0 INFO TaskSetManager: Starting task 1.0 in stage 6.0 (TID 24, 192.168.1.225, executor driver, partition 1, NODE_LOCAL, 7143 bytes) 20/01/23 18:54:03.186 dispatcher-event-loop-0 INFO TaskSetManager: Starting task 2.0 in stage 6.0 (TID 25, 192.168.1.225, executor driver, partition 2, NODE_LOCAL, 7143 bytes) 20/01/23 18:54:03.186 dispatcher-event-loop-0 DEBUG TaskSetManager: No tasks for locality level NODE_LOCAL, so moving to locality level NO_PREF 20/01/23 18:54:03.186 dispatcher-event-loop-0 DEBUG TaskSetManager: Moving to ANY after waiting for 0ms 20/01/23 18:54:03.186 dispatcher-event-loop-0 INFO TaskSetManager: Starting task 3.0 in stage 6.0 (TID 26, 192.168.1.225, executor driver, partition 3, PROCESS_LOCAL, 7143 bytes) 20/01/23 18:54:03.186 Executor task launch worker for task 23 INFO Executor: Running task 0.0 in stage 6.0 (TID 23) 20/01/23 18:54:03.187 Executor task launch worker for task 24 INFO Executor: Running task 1.0 in stage 6.0 (TID 24) 20/01/23 18:54:03.187 Executor task launch worker for task 25 INFO Executor: Running task 2.0 in stage 6.0 (TID 25) 20/01/23 18:54:03.187 Executor task launch worker for task 26 INFO Executor: Running task 3.0 in stage 6.0 (TID 26) 20/01/23 18:54:03.187 Executor task launch worker for task 23 DEBUG ExecutorMetricsPoller: stageTCMP: (6, 0) -> 1 20/01/23 18:54:03.187 Executor task launch worker for task 25 DEBUG ExecutorMetricsPoller: stageTCMP: (6, 0) -> 3 20/01/23 18:54:03.187 Executor task launch worker for task 26 DEBUG ExecutorMetricsPoller: stageTCMP: (6, 0) -> 2 20/01/23 18:54:03.188 Executor task launch worker for task 23 DEBUG BlockManager: Getting local block broadcast_5 20/01/23 18:54:03.188 Executor task launch worker for task 23 DEBUG BlockManager: Level for block broadcast_5 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:03.188 Executor task launch worker for task 24 DEBUG ExecutorMetricsPoller: stageTCMP: (6, 0) -> 4 20/01/23 18:54:03.194 Executor task launch worker for task 23 DEBUG BlockManager: Getting local block rdd_25_0 20/01/23 18:54:03.194 Executor task launch worker for task 23 DEBUG BlockManager: Block rdd_25_0 was not found 20/01/23 18:54:03.194 Executor task launch worker for task 23 DEBUG BlockManager: Getting remote block rdd_25_0 20/01/23 18:54:03.198 Executor task launch worker for task 26 DEBUG BlockManager: Getting local block rdd_25_3 20/01/23 18:54:03.198 Executor task launch worker for task 26 DEBUG BlockManager: Block rdd_25_3 was not found 20/01/23 18:54:03.198 Executor task launch worker for task 26 DEBUG BlockManager: Getting remote block rdd_25_3 20/01/23 18:54:03.199 Executor task launch worker for task 26 DEBUG BlockManager: Block rdd_25_3 is unknown by block manager master 20/01/23 18:54:03.199 Executor task launch worker for task 26 DEBUG BlockManager: Getting local block rdd_24_3 20/01/23 18:54:03.199 Executor task launch worker for task 26 DEBUG BlockManager: Block rdd_24_3 was not found 20/01/23 18:54:03.199 Executor task launch worker for task 26 DEBUG BlockManager: Getting remote block rdd_24_3 20/01/23 18:54:03.199 Executor task launch worker for task 26 DEBUG BlockManager: Block rdd_24_3 is unknown by block manager master 20/01/23 18:54:03.199 Executor task launch worker for task 26 DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 2, partitions 3-4 20/01/23 18:54:03.200 Executor task launch worker for task 26 DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 20/01/23 18:54:03.200 Executor task launch worker for task 26 INFO ShuffleBlockFetcherIterator: Getting 0 (0.0 B) non-empty blocks including 0 (0.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 20/01/23 18:54:03.200 Executor task launch worker for task 26 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 20/01/23 18:54:03.200 Executor task launch worker for task 26 DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: 20/01/23 18:54:03.200 Executor task launch worker for task 26 DEBUG ShuffleBlockFetcherIterator: Got local blocks in 0 ms 20/01/23 18:54:03.201 Executor task launch worker for task 23 DEBUG BlockManager: Block rdd_25_0 is unknown by block manager master 20/01/23 18:54:03.201 Executor task launch worker for task 23 DEBUG BlockManager: Getting local block rdd_24_0 20/01/23 18:54:03.201 Executor task launch worker for task 23 DEBUG BlockManager: Block rdd_24_0 was not found 20/01/23 18:54:03.201 Executor task launch worker for task 23 DEBUG BlockManager: Getting remote block rdd_24_0 20/01/23 18:54:03.201 Executor task launch worker for task 23 DEBUG BlockManager: Block rdd_24_0 is unknown by block manager master 20/01/23 18:54:03.201 Executor task launch worker for task 23 DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 2, partitions 0-1 20/01/23 18:54:03.202 Executor task launch worker for task 23 DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 20/01/23 18:54:03.202 Executor task launch worker for task 23 INFO ShuffleBlockFetcherIterator: Getting 1 (251.0 B) non-empty blocks including 1 (251.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 20/01/23 18:54:03.203 Executor task launch worker for task 23 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 20/01/23 18:54:03.203 Executor task launch worker for task 23 DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_2_20_0,1) 20/01/23 18:54:03.203 Executor task launch worker for task 23 DEBUG ShuffleBlockFetcherIterator: Got local blocks in 1 ms 20/01/23 18:54:03.198 Executor task launch worker for task 25 DEBUG BlockManager: Getting local block rdd_25_2 20/01/23 18:54:03.201 Executor task launch worker for task 26 DEBUG TaskMemoryManager: Task 26 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@3805706e 20/01/23 18:54:03.204 Executor task launch worker for task 24 DEBUG BlockManager: Getting local block rdd_25_1 20/01/23 18:54:03.204 Executor task launch worker for task 24 DEBUG BlockManager: Block rdd_25_1 was not found 20/01/23 18:54:03.204 Executor task launch worker for task 24 DEBUG BlockManager: Getting remote block rdd_25_1 20/01/23 18:54:03.204 Executor task launch worker for task 23 DEBUG ALS: Start sorting an uncompressed in-block of size 1. (sortId = -2059903202) 20/01/23 18:54:03.204 Executor task launch worker for task 23 DEBUG ALS: Sorting took 1.336E-5 seconds. (sortId = -2059903202) 20/01/23 18:54:03.205 Executor task launch worker for task 26 INFO MemoryStore: Block rdd_24_3 stored as values in memory (estimated size 16.0 B, free 366.1 MiB) 20/01/23 18:54:03.205 Executor task launch worker for task 23 DEBUG TaskMemoryManager: Task 23 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@6c313dcb 20/01/23 18:54:03.205 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_24_3 in memory on 192.168.1.225:35863 (size: 16.0 B, free: 366.2 MiB) 20/01/23 18:54:03.205 Executor task launch worker for task 23 INFO MemoryStore: Block rdd_24_0 stored as values in memory (estimated size 192.0 B, free 366.1 MiB) 20/01/23 18:54:03.206 Executor task launch worker for task 26 DEBUG BlockManagerMaster: Updated info of block rdd_24_3 20/01/23 18:54:03.206 Executor task launch worker for task 26 DEBUG BlockManager: Told master about block rdd_24_3 20/01/23 18:54:03.206 Executor task launch worker for task 26 DEBUG BlockManager: Put block rdd_24_3 locally took 6 ms 20/01/23 18:54:03.206 Executor task launch worker for task 26 DEBUG BlockManager: Putting block rdd_24_3 without replication took 6 ms 20/01/23 18:54:03.206 Executor task launch worker for task 26 DEBUG BlockManager: Getting local block rdd_24_3 20/01/23 18:54:03.206 Executor task launch worker for task 26 DEBUG BlockManager: Level for block rdd_24_3 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:03.207 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_24_0 in memory on 192.168.1.225:35863 (size: 192.0 B, free: 366.2 MiB) 20/01/23 18:54:03.208 Executor task launch worker for task 26 INFO MemoryStore: Block rdd_25_3 stored as values in memory (estimated size 16.0 B, free 366.1 MiB) 20/01/23 18:54:03.208 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_25_3 in memory on 192.168.1.225:35863 (size: 16.0 B, free: 366.2 MiB) 20/01/23 18:54:03.208 Executor task launch worker for task 8 DEBUG GenerateSafeProjection: code for createexternalrow(staticinvoke(class scala.collection.mutable.WrappedArray$, ObjectType(interface scala.collection.Seq), make, mapobjects(lambdavariable(MapObject, StringType, true, -1), lambdavariable(MapObject, StringType, true, -1).toString, input[0, array, true], None).array, true, false), StructField(words,ArrayType(StringType,true),true)): /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificSafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificSafeProjection extends org.apache.spark.sql.catalyst.expressions.codegen.BaseProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private InternalRow mutableRow; /* 009 */ private boolean resultIsNull_0; /* 010 */ private UTF8String value_MapObject_lambda_variable_1; /* 011 */ private boolean isNull_MapObject_lambda_variable_1; /* 012 */ private boolean globalIsNull_0; /* 013 */ private java.lang.Object[] mutableStateArray_0 = new java.lang.Object[1]; /* 014 */ /* 015 */ public SpecificSafeProjection(Object[] references) { /* 016 */ this.references = references; /* 017 */ mutableRow = (InternalRow) references[references.length - 1]; /* 018 */ /* 019 */ /* 020 */ } /* 021 */ /* 022 */ public void initialize(int partitionIndex) { /* 023 */ /* 024 */ } /* 025 */ /* 026 */ public java.lang.Object apply(java.lang.Object _i) { /* 027 */ InternalRow i = (InternalRow) _i; /* 028 */ Object[] values_0 = new Object[1]; /* 029 */ /* 030 */ resultIsNull_0 = false; /* 031 */ if (!resultIsNull_0) { /* 032 */ ArrayData value_6 = MapObjects_0(i); /* 033 */ boolean isNull_2 = true; /* 034 */ java.lang.Object value_2 = null; /* 035 */ if (!globalIsNull_0) { /* 036 */ /* 037 */ isNull_2 = false; /* 038 */ if (!isNull_2) { /* 039 */ /* 040 */ Object funcResult_1 = null; /* 041 */ funcResult_1 = value_6.array(); /* 042 */ value_2 = (java.lang.Object) funcResult_1; /* 043 */ /* 044 */ } /* 045 */ } /* 046 */ resultIsNull_0 = isNull_2; /* 047 */ mutableStateArray_0[0] = value_2; /* 048 */ } /* 049 */ /* 050 */ boolean isNull_1 = resultIsNull_0; /* 051 */ scala.collection.Seq value_1 = null; /* 052 */ if (!resultIsNull_0) { /* 053 */ value_1 = scala.collection.mutable.WrappedArray.make(mutableStateArray_0[0]); /* 054 */ } /* 055 */ if (isNull_1) { /* 056 */ values_0[0] = null; /* 057 */ } else { /* 058 */ values_0[0] = value_1; /* 059 */ } /* 060 */ /* 061 */ final org.apache.spark.sql.Row value_0 = new org.apache.spark.sql.catalyst.expressions.GenericRowWithSchema(values_0, ((org.apache.spark.sql.types.StructType) references[0] /* schema */)); /* 062 */ if (false) { /* 063 */ mutableRow.setNullAt(0); /* 064 */ } else { /* 065 */ /* 066 */ mutableRow.update(0, value_0); /* 067 */ } /* 068 */ /* 069 */ return mutableRow; /* 070 */ } /* 071 */ /* 072 */ /* 073 */ private ArrayData MapObjects_0(InternalRow i) { /* 074 */ boolean isNull_4 = i.isNullAt(0); /* 075 */ ArrayData value_4 = isNull_4 ? /* 076 */ null : (i.getArray(0)); /* 077 */ ArrayData value_3 = null; /* 078 */ /* 079 */ if (!isNull_4) { /* 080 */ /* 081 */ int dataLength_0 = value_4.numElements(); /* 082 */ /* 083 */ java.lang.String[] convertedArray_0 = null; /* 084 */ convertedArray_0 = new java.lang.String[dataLength_0]; /* 085 */ /* 086 */ /* 087 */ int loopIndex_0 = 0; /* 088 */ /* 089 */ while (loopIndex_0 < dataLength_0) { /* 090 */ value_MapObject_lambda_variable_1 = (UTF8String) (value_4.getUTF8String(loopIndex_0)); /* 091 */ isNull_MapObject_lambda_variable_1 = value_4.isNullAt(loopIndex_0); /* 092 */ /* 093 */ boolean isNull_5 = true; /* 094 */ java.lang.String value_5 = null; /* 095 */ if (!isNull_MapObject_lambda_variable_1) { /* 096 */ /* 097 */ isNull_5 = false; /* 098 */ if (!isNull_5) { /* 099 */ /* 100 */ Object funcResult_0 = null; /* 101 */ funcResult_0 = value_MapObject_lambda_variable_1.toString(); /* 102 */ value_5 = (java.lang.String) funcResult_0; /* 103 */ /* 104 */ } /* 105 */ } /* 106 */ if (isNull_5) { /* 107 */ convertedArray_0[loopIndex_0] = null; /* 108 */ } else { /* 109 */ convertedArray_0[loopIndex_0] = value_5; /* 110 */ } /* 111 */ /* 112 */ loopIndex_0 += 1; /* 113 */ } /* 114 */ /* 115 */ value_3 = new org.apache.spark.sql.catalyst.util.GenericArrayData(convertedArray_0); /* 116 */ } /* 117 */ globalIsNull_0 = isNull_4; /* 118 */ return value_3; /* 119 */ } /* 120 */ /* 121 */ } 20/01/23 18:54:03.209 Executor task launch worker for task 26 DEBUG BlockManagerMaster: Updated info of block rdd_25_3 20/01/23 18:54:03.209 Executor task launch worker for task 26 DEBUG BlockManager: Told master about block rdd_25_3 20/01/23 18:54:03.209 Executor task launch worker for task 26 DEBUG BlockManager: Put block rdd_25_3 locally took 10 ms 20/01/23 18:54:03.209 Executor task launch worker for task 26 DEBUG BlockManager: Putting block rdd_25_3 without replication took 10 ms 20/01/23 18:54:03.209 Executor task launch worker for task 26 DEBUG BlockManager: Getting local block rdd_25_3 20/01/23 18:54:03.209 Executor task launch worker for task 26 DEBUG BlockManager: Level for block rdd_25_3 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:03.204 Executor task launch worker for task 25 DEBUG BlockManager: Block rdd_25_2 was not found 20/01/23 18:54:03.209 Executor task launch worker for task 23 DEBUG BlockManagerMaster: Updated info of block rdd_24_0 20/01/23 18:54:03.211 Executor task launch worker for task 24 DEBUG BlockManager: Block rdd_25_1 is unknown by block manager master 20/01/23 18:54:03.211 Executor task launch worker for task 24 DEBUG BlockManager: Getting local block rdd_24_1 20/01/23 18:54:03.211 Executor task launch worker for task 24 DEBUG BlockManager: Block rdd_24_1 was not found 20/01/23 18:54:03.211 Executor task launch worker for task 24 DEBUG BlockManager: Getting remote block rdd_24_1 20/01/23 18:54:03.211 Executor task launch worker for task 24 DEBUG BlockManager: Block rdd_24_1 is unknown by block manager master 20/01/23 18:54:03.211 Executor task launch worker for task 24 DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 2, partitions 1-2 20/01/23 18:54:03.212 Executor task launch worker for task 24 DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 20/01/23 18:54:03.212 Executor task launch worker for task 24 INFO ShuffleBlockFetcherIterator: Getting 3 (753.0 B) non-empty blocks including 3 (753.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 20/01/23 18:54:03.212 Executor task launch worker for task 24 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 20/01/23 18:54:03.212 Executor task launch worker for task 24 DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_2_19_1,0), (shuffle_2_21_1,2), (shuffle_2_22_1,3) 20/01/23 18:54:03.213 Executor task launch worker for task 24 DEBUG ShuffleBlockFetcherIterator: Got local blocks in 1 ms 20/01/23 18:54:03.211 Executor task launch worker for task 26 INFO Executor: Finished task 3.0 in stage 6.0 (TID 26). 2222 bytes result sent to driver 20/01/23 18:54:03.214 dispatcher-event-loop-2 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_6.0, runningTasks: 3 20/01/23 18:54:03.215 dispatcher-event-loop-2 INFO TaskSetManager: Starting task 4.0 in stage 6.0 (TID 27, 192.168.1.225, executor driver, partition 4, PROCESS_LOCAL, 7143 bytes) 20/01/23 18:54:03.210 Executor task launch worker for task 25 DEBUG BlockManager: Getting remote block rdd_25_2 20/01/23 18:54:03.215 task-result-getter-0 INFO TaskSetManager: Finished task 3.0 in stage 6.0 (TID 26) in 29 ms on 192.168.1.225 (executor driver) (1/10) 20/01/23 18:54:03.211 Executor task launch worker for task 23 DEBUG BlockManager: Told master about block rdd_24_0 20/01/23 18:54:03.215 Executor task launch worker for task 26 DEBUG ExecutorMetricsPoller: stageTCMP: (6, 0) -> 3 20/01/23 18:54:03.215 Executor task launch worker for task 27 INFO Executor: Running task 4.0 in stage 6.0 (TID 27) 20/01/23 18:54:03.215 Executor task launch worker for task 23 DEBUG BlockManager: Put block rdd_24_0 locally took 13 ms 20/01/23 18:54:03.216 Executor task launch worker for task 23 DEBUG BlockManager: Putting block rdd_24_0 without replication took 14 ms 20/01/23 18:54:03.216 Executor task launch worker for task 23 DEBUG BlockManager: Getting local block rdd_24_0 20/01/23 18:54:03.216 Executor task launch worker for task 25 DEBUG BlockManager: Block rdd_25_2 is unknown by block manager master 20/01/23 18:54:03.216 Executor task launch worker for task 23 DEBUG BlockManager: Level for block rdd_24_0 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:03.216 Executor task launch worker for task 25 DEBUG BlockManager: Getting local block rdd_24_2 20/01/23 18:54:03.216 Executor task launch worker for task 25 DEBUG BlockManager: Block rdd_24_2 was not found 20/01/23 18:54:03.216 Executor task launch worker for task 25 DEBUG BlockManager: Getting remote block rdd_24_2 20/01/23 18:54:03.216 Executor task launch worker for task 27 DEBUG ExecutorMetricsPoller: stageTCMP: (6, 0) -> 4 20/01/23 18:54:03.216 Executor task launch worker for task 25 DEBUG BlockManager: Block rdd_24_2 is unknown by block manager master 20/01/23 18:54:03.216 Executor task launch worker for task 25 DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 2, partitions 2-3 20/01/23 18:54:03.217 Executor task launch worker for task 25 DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 20/01/23 18:54:03.218 Executor task launch worker for task 25 INFO ShuffleBlockFetcherIterator: Getting 2 (527.0 B) non-empty blocks including 2 (527.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 20/01/23 18:54:03.218 Executor task launch worker for task 25 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 20/01/23 18:54:03.218 Executor task launch worker for task 25 DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_2_21_2,2), (shuffle_2_22_2,3) 20/01/23 18:54:03.218 Executor task launch worker for task 25 DEBUG ShuffleBlockFetcherIterator: Got local blocks in 1 ms 20/01/23 18:54:03.220 Executor task launch worker for task 25 DEBUG ALS: Start sorting an uncompressed in-block of size 2. (sortId = 827117277) 20/01/23 18:54:03.220 Executor task launch worker for task 25 DEBUG ALS: Sorting took 3.644E-5 seconds. (sortId = 827117277) 20/01/23 18:54:03.220 Executor task launch worker for task 25 DEBUG TaskMemoryManager: Task 25 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@7a288254 20/01/23 18:54:03.221 Executor task launch worker for task 25 INFO MemoryStore: Block rdd_24_2 stored as values in memory (estimated size 192.0 B, free 366.1 MiB) 20/01/23 18:54:03.221 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_24_2 in memory on 192.168.1.225:35863 (size: 192.0 B, free: 366.2 MiB) 20/01/23 18:54:03.222 Executor task launch worker for task 23 INFO MemoryStore: Block rdd_25_0 stored as values in memory (estimated size 288.0 B, free 366.1 MiB) 20/01/23 18:54:03.222 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_25_0 in memory on 192.168.1.225:35863 (size: 288.0 B, free: 366.2 MiB) 20/01/23 18:54:03.222 Executor task launch worker for task 27 DEBUG BlockManager: Getting local block rdd_25_4 20/01/23 18:54:03.222 Executor task launch worker for task 27 DEBUG BlockManager: Block rdd_25_4 was not found 20/01/23 18:54:03.222 Executor task launch worker for task 27 DEBUG BlockManager: Getting remote block rdd_25_4 20/01/23 18:54:03.222 Executor task launch worker for task 23 DEBUG BlockManagerMaster: Updated info of block rdd_25_0 20/01/23 18:54:03.222 Executor task launch worker for task 23 DEBUG BlockManager: Told master about block rdd_25_0 20/01/23 18:54:03.222 Executor task launch worker for task 23 DEBUG BlockManager: Put block rdd_25_0 locally took 21 ms 20/01/23 18:54:03.223 Executor task launch worker for task 23 DEBUG BlockManager: Putting block rdd_25_0 without replication took 21 ms 20/01/23 18:54:03.223 Executor task launch worker for task 23 DEBUG BlockManager: Getting local block rdd_25_0 20/01/23 18:54:03.223 Executor task launch worker for task 23 DEBUG BlockManager: Level for block rdd_25_0 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:03.223 Executor task launch worker for task 25 DEBUG BlockManagerMaster: Updated info of block rdd_24_2 20/01/23 18:54:03.223 Executor task launch worker for task 25 DEBUG BlockManager: Told master about block rdd_24_2 20/01/23 18:54:03.223 Executor task launch worker for task 25 DEBUG BlockManager: Put block rdd_24_2 locally took 6 ms 20/01/23 18:54:03.223 Executor task launch worker for task 27 DEBUG BlockManager: Block rdd_25_4 is unknown by block manager master 20/01/23 18:54:03.210 Executor task launch worker for task 8 DEBUG CodeGenerator: /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificSafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificSafeProjection extends org.apache.spark.sql.catalyst.expressions.codegen.BaseProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private InternalRow mutableRow; /* 009 */ private boolean resultIsNull_0; /* 010 */ private UTF8String value_MapObject_lambda_variable_1; /* 011 */ private boolean isNull_MapObject_lambda_variable_1; /* 012 */ private boolean globalIsNull_0; /* 013 */ private java.lang.Object[] mutableStateArray_0 = new java.lang.Object[1]; /* 014 */ /* 015 */ public SpecificSafeProjection(Object[] references) { /* 016 */ this.references = references; /* 017 */ mutableRow = (InternalRow) references[references.length - 1]; /* 018 */ /* 019 */ /* 020 */ } /* 021 */ /* 022 */ public void initialize(int partitionIndex) { /* 023 */ /* 024 */ } /* 025 */ /* 026 */ public java.lang.Object apply(java.lang.Object _i) { /* 027 */ InternalRow i = (InternalRow) _i; /* 028 */ Object[] values_0 = new Object[1]; /* 029 */ /* 030 */ resultIsNull_0 = false; /* 031 */ if (!resultIsNull_0) { /* 032 */ ArrayData value_6 = MapObjects_0(i); /* 033 */ boolean isNull_2 = true; /* 034 */ java.lang.Object value_2 = null; /* 035 */ if (!globalIsNull_0) { /* 036 */ /* 037 */ isNull_2 = false; /* 038 */ if (!isNull_2) { /* 039 */ /* 040 */ Object funcResult_1 = null; /* 041 */ funcResult_1 = value_6.array(); /* 042 */ value_2 = (java.lang.Object) funcResult_1; /* 043 */ /* 044 */ } /* 045 */ } /* 046 */ resultIsNull_0 = isNull_2; /* 047 */ mutableStateArray_0[0] = value_2; /* 048 */ } /* 049 */ /* 050 */ boolean isNull_1 = resultIsNull_0; /* 051 */ scala.collection.Seq value_1 = null; /* 052 */ if (!resultIsNull_0) { /* 053 */ value_1 = scala.collection.mutable.WrappedArray.make(mutableStateArray_0[0]); /* 054 */ } /* 055 */ if (isNull_1) { /* 056 */ values_0[0] = null; /* 057 */ } else { /* 058 */ values_0[0] = value_1; /* 059 */ } /* 060 */ /* 061 */ final org.apache.spark.sql.Row value_0 = new org.apache.spark.sql.catalyst.expressions.GenericRowWithSchema(values_0, ((org.apache.spark.sql.types.StructType) references[0] /* schema */)); /* 062 */ if (false) { /* 063 */ mutableRow.setNullAt(0); /* 064 */ } else { /* 065 */ /* 066 */ mutableRow.update(0, value_0); /* 067 */ } /* 068 */ /* 069 */ return mutableRow; /* 070 */ } /* 071 */ /* 072 */ /* 073 */ private ArrayData MapObjects_0(InternalRow i) { /* 074 */ boolean isNull_4 = i.isNullAt(0); /* 075 */ ArrayData value_4 = isNull_4 ? /* 076 */ null : (i.getArray(0)); /* 077 */ ArrayData value_3 = null; /* 078 */ /* 079 */ if (!isNull_4) { /* 080 */ /* 081 */ int dataLength_0 = value_4.numElements(); /* 082 */ /* 083 */ java.lang.String[] convertedArray_0 = null; /* 084 */ convertedArray_0 = new java.lang.String[dataLength_0]; /* 085 */ /* 086 */ /* 087 */ int loopIndex_0 = 0; /* 088 */ /* 089 */ while (loopIndex_0 < dataLength_0) { /* 090 */ value_MapObject_lambda_variable_1 = (UTF8String) (value_4.getUTF8String(loopIndex_0)); /* 091 */ isNull_MapObject_lambda_variable_1 = value_4.isNullAt(loopIndex_0); /* 092 */ /* 093 */ boolean isNull_5 = true; /* 094 */ java.lang.String value_5 = null; /* 095 */ if (!isNull_MapObject_lambda_variable_1) { /* 096 */ /* 097 */ isNull_5 = false; /* 098 */ if (!isNull_5) { /* 099 */ /* 100 */ Object funcResult_0 = null; /* 101 */ funcResult_0 = value_MapObject_lambda_variable_1.toString(); /* 102 */ value_5 = (java.lang.String) funcResult_0; /* 103 */ /* 104 */ } /* 105 */ } /* 106 */ if (isNull_5) { /* 107 */ convertedArray_0[loopIndex_0] = null; /* 108 */ } else { /* 109 */ convertedArray_0[loopIndex_0] = value_5; /* 110 */ } /* 111 */ /* 112 */ loopIndex_0 += 1; /* 113 */ } /* 114 */ /* 115 */ value_3 = new org.apache.spark.sql.catalyst.util.GenericArrayData(convertedArray_0); /* 116 */ } /* 117 */ globalIsNull_0 = isNull_4; /* 118 */ return value_3; /* 119 */ } /* 120 */ /* 121 */ } 20/01/23 18:54:03.218 Executor task launch worker for task 10 DEBUG GenerateSafeProjection: code for createexternalrow(staticinvoke(class scala.collection.mutable.WrappedArray$, ObjectType(interface scala.collection.Seq), make, mapobjects(lambdavariable(MapObject, StringType, true, -1), lambdavariable(MapObject, StringType, true, -1).toString, input[0, array, true], None).array, true, false), StructField(words,ArrayType(StringType,true),true)): /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificSafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificSafeProjection extends org.apache.spark.sql.catalyst.expressions.codegen.BaseProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private InternalRow mutableRow; /* 009 */ private boolean resultIsNull_0; /* 010 */ private UTF8String value_MapObject_lambda_variable_1; /* 011 */ private boolean isNull_MapObject_lambda_variable_1; /* 012 */ private boolean globalIsNull_0; /* 013 */ private java.lang.Object[] mutableStateArray_0 = new java.lang.Object[1]; /* 014 */ /* 015 */ public SpecificSafeProjection(Object[] references) { /* 016 */ this.references = references; /* 017 */ mutableRow = (InternalRow) references[references.length - 1]; /* 018 */ /* 019 */ /* 020 */ } /* 021 */ /* 022 */ public void initialize(int partitionIndex) { /* 023 */ /* 024 */ } /* 025 */ /* 026 */ public java.lang.Object apply(java.lang.Object _i) { /* 027 */ InternalRow i = (InternalRow) _i; /* 028 */ Object[] values_0 = new Object[1]; /* 029 */ /* 030 */ resultIsNull_0 = false; /* 031 */ if (!resultIsNull_0) { /* 032 */ ArrayData value_6 = MapObjects_0(i); /* 033 */ boolean isNull_2 = true; /* 034 */ java.lang.Object value_2 = null; /* 035 */ if (!globalIsNull_0) { /* 036 */ /* 037 */ isNull_2 = false; /* 038 */ if (!isNull_2) { /* 039 */ /* 040 */ Object funcResult_1 = null; /* 041 */ funcResult_1 = value_6.array(); /* 042 */ value_2 = (java.lang.Object) funcResult_1; /* 043 */ /* 044 */ } /* 045 */ } /* 046 */ resultIsNull_0 = isNull_2; /* 047 */ mutableStateArray_0[0] = value_2; /* 048 */ } /* 049 */ /* 050 */ boolean isNull_1 = resultIsNull_0; /* 051 */ scala.collection.Seq value_1 = null; /* 052 */ if (!resultIsNull_0) { /* 053 */ value_1 = scala.collection.mutable.WrappedArray.make(mutableStateArray_0[0]); /* 054 */ } /* 055 */ if (isNull_1) { /* 056 */ values_0[0] = null; /* 057 */ } else { /* 058 */ values_0[0] = value_1; /* 059 */ } /* 060 */ /* 061 */ final org.apache.spark.sql.Row value_0 = new org.apache.spark.sql.catalyst.expressions.GenericRowWithSchema(values_0, ((org.apache.spark.sql.types.StructType) references[0] /* schema */)); /* 062 */ if (false) { /* 063 */ mutableRow.setNullAt(0); /* 064 */ } else { /* 065 */ /* 066 */ mutableRow.update(0, value_0); /* 067 */ } /* 068 */ /* 069 */ return mutableRow; /* 070 */ } /* 071 */ /* 072 */ /* 073 */ private ArrayData MapObjects_0(InternalRow i) { /* 074 */ boolean isNull_4 = i.isNullAt(0); /* 075 */ ArrayData value_4 = isNull_4 ? /* 076 */ null : (i.getArray(0)); /* 077 */ ArrayData value_3 = null; /* 078 */ /* 079 */ if (!isNull_4) { /* 080 */ /* 081 */ int dataLength_0 = value_4.numElements(); /* 082 */ /* 083 */ java.lang.String[] convertedArray_0 = null; /* 084 */ convertedArray_0 = new java.lang.String[dataLength_0]; /* 085 */ /* 086 */ /* 087 */ int loopIndex_0 = 0; /* 088 */ /* 089 */ while (loopIndex_0 < dataLength_0) { /* 090 */ value_MapObject_lambda_variable_1 = (UTF8String) (value_4.getUTF8String(loopIndex_0)); /* 091 */ isNull_MapObject_lambda_variable_1 = value_4.isNullAt(loopIndex_0); /* 092 */ /* 093 */ boolean isNull_5 = true; /* 094 */ java.lang.String value_5 = null; /* 095 */ if (!isNull_MapObject_lambda_variable_1) { /* 096 */ /* 097 */ isNull_5 = false; /* 098 */ if (!isNull_5) { /* 099 */ /* 100 */ Object funcResult_0 = null; /* 101 */ funcResult_0 = value_MapObject_lambda_variable_1.toString(); /* 102 */ value_5 = (java.lang.String) funcResult_0; /* 103 */ /* 104 */ } /* 105 */ } /* 106 */ if (isNull_5) { /* 107 */ convertedArray_0[loopIndex_0] = null; /* 108 */ } else { /* 109 */ convertedArray_0[loopIndex_0] = value_5; /* 110 */ } /* 111 */ /* 112 */ loopIndex_0 += 1; /* 113 */ } /* 114 */ /* 115 */ value_3 = new org.apache.spark.sql.catalyst.util.GenericArrayData(convertedArray_0); /* 116 */ } /* 117 */ globalIsNull_0 = isNull_4; /* 118 */ return value_3; /* 119 */ } /* 120 */ /* 121 */ } 20/01/23 18:54:03.226 Executor task launch worker for task 11 DEBUG GenerateSafeProjection: code for createexternalrow(staticinvoke(class scala.collection.mutable.WrappedArray$, ObjectType(interface scala.collection.Seq), make, mapobjects(lambdavariable(MapObject, StringType, true, -1), lambdavariable(MapObject, StringType, true, -1).toString, input[0, array, true], None).array, true, false), StructField(words,ArrayType(StringType,true),true)): /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificSafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificSafeProjection extends org.apache.spark.sql.catalyst.expressions.codegen.BaseProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private InternalRow mutableRow; /* 009 */ private boolean resultIsNull_0; /* 010 */ private UTF8String value_MapObject_lambda_variable_1; /* 011 */ private boolean isNull_MapObject_lambda_variable_1; /* 012 */ private boolean globalIsNull_0; /* 013 */ private java.lang.Object[] mutableStateArray_0 = new java.lang.Object[1]; /* 014 */ /* 015 */ public SpecificSafeProjection(Object[] references) { /* 016 */ this.references = references; /* 017 */ mutableRow = (InternalRow) references[references.length - 1]; /* 018 */ /* 019 */ /* 020 */ } /* 021 */ /* 022 */ public void initialize(int partitionIndex) { /* 023 */ /* 024 */ } /* 025 */ /* 026 */ public java.lang.Object apply(java.lang.Object _i) { /* 027 */ InternalRow i = (InternalRow) _i; /* 028 */ Object[] values_0 = new Object[1]; /* 029 */ /* 030 */ resultIsNull_0 = false; /* 031 */ if (!resultIsNull_0) { /* 032 */ ArrayData value_6 = MapObjects_0(i); /* 033 */ boolean isNull_2 = true; /* 034 */ java.lang.Object value_2 = null; /* 035 */ if (!globalIsNull_0) { /* 036 */ /* 037 */ isNull_2 = false; /* 038 */ if (!isNull_2) { /* 039 */ /* 040 */ Object funcResult_1 = null; /* 041 */ funcResult_1 = value_6.array(); /* 042 */ value_2 = (java.lang.Object) funcResult_1; /* 043 */ /* 044 */ } /* 045 */ } /* 046 */ resultIsNull_0 = isNull_2; /* 047 */ mutableStateArray_0[0] = value_2; /* 048 */ } /* 049 */ /* 050 */ boolean isNull_1 = resultIsNull_0; /* 051 */ scala.collection.Seq value_1 = null; /* 052 */ if (!resultIsNull_0) { /* 053 */ value_1 = scala.collection.mutable.WrappedArray.make(mutableStateArray_0[0]); /* 054 */ } /* 055 */ if (isNull_1) { /* 056 */ values_0[0] = null; /* 057 */ } else { /* 058 */ values_0[0] = value_1; /* 059 */ } /* 060 */ /* 061 */ final org.apache.spark.sql.Row value_0 = new org.apache.spark.sql.catalyst.expressions.GenericRowWithSchema(values_0, ((org.apache.spark.sql.types.StructType) references[0] /* schema */)); /* 062 */ if (false) { /* 063 */ mutableRow.setNullAt(0); /* 064 */ } else { /* 065 */ /* 066 */ mutableRow.update(0, value_0); /* 067 */ } /* 068 */ /* 069 */ return mutableRow; /* 070 */ } /* 071 */ /* 072 */ /* 073 */ private ArrayData MapObjects_0(InternalRow i) { /* 074 */ boolean isNull_4 = i.isNullAt(0); /* 075 */ ArrayData value_4 = isNull_4 ? /* 076 */ null : (i.getArray(0)); /* 077 */ ArrayData value_3 = null; /* 078 */ /* 079 */ if (!isNull_4) { /* 080 */ /* 081 */ int dataLength_0 = value_4.numElements(); /* 082 */ /* 083 */ java.lang.String[] convertedArray_0 = null; /* 084 */ convertedArray_0 = new java.lang.String[dataLength_0]; /* 085 */ /* 086 */ /* 087 */ int loopIndex_0 = 0; /* 088 */ /* 089 */ while (loopIndex_0 < dataLength_0) { /* 090 */ value_MapObject_lambda_variable_1 = (UTF8String) (value_4.getUTF8String(loopIndex_0)); /* 091 */ isNull_MapObject_lambda_variable_1 = value_4.isNullAt(loopIndex_0); /* 092 */ /* 093 */ boolean isNull_5 = true; /* 094 */ java.lang.String value_5 = null; /* 095 */ if (!isNull_MapObject_lambda_variable_1) { /* 096 */ /* 097 */ isNull_5 = false; /* 098 */ if (!isNull_5) { /* 099 */ /* 100 */ Object funcResult_0 = null; /* 101 */ funcResult_0 = value_MapObject_lambda_variable_1.toString(); /* 102 */ value_5 = (java.lang.String) funcResult_0; /* 103 */ /* 104 */ } /* 105 */ } /* 106 */ if (isNull_5) { /* 107 */ convertedArray_0[loopIndex_0] = null; /* 108 */ } else { /* 109 */ convertedArray_0[loopIndex_0] = value_5; /* 110 */ } /* 111 */ /* 112 */ loopIndex_0 += 1; /* 113 */ } /* 114 */ /* 115 */ value_3 = new org.apache.spark.sql.catalyst.util.GenericArrayData(convertedArray_0); /* 116 */ } /* 117 */ globalIsNull_0 = isNull_4; /* 118 */ return value_3; /* 119 */ } /* 120 */ /* 121 */ } 20/01/23 18:54:03.227 Executor task launch worker for task 9 DEBUG GenerateSafeProjection: code for createexternalrow(staticinvoke(class scala.collection.mutable.WrappedArray$, ObjectType(interface scala.collection.Seq), make, mapobjects(lambdavariable(MapObject, StringType, true, -1), lambdavariable(MapObject, StringType, true, -1).toString, input[0, array, true], None).array, true, false), StructField(words,ArrayType(StringType,true),true)): /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificSafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificSafeProjection extends org.apache.spark.sql.catalyst.expressions.codegen.BaseProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private InternalRow mutableRow; /* 009 */ private boolean resultIsNull_0; /* 010 */ private UTF8String value_MapObject_lambda_variable_1; /* 011 */ private boolean isNull_MapObject_lambda_variable_1; /* 012 */ private boolean globalIsNull_0; /* 013 */ private java.lang.Object[] mutableStateArray_0 = new java.lang.Object[1]; /* 014 */ /* 015 */ public SpecificSafeProjection(Object[] references) { /* 016 */ this.references = references; /* 017 */ mutableRow = (InternalRow) references[references.length - 1]; /* 018 */ /* 019 */ /* 020 */ } /* 021 */ /* 022 */ public void initialize(int partitionIndex) { /* 023 */ /* 024 */ } /* 025 */ /* 026 */ public java.lang.Object apply(java.lang.Object _i) { /* 027 */ InternalRow i = (InternalRow) _i; /* 028 */ Object[] values_0 = new Object[1]; /* 029 */ /* 030 */ resultIsNull_0 = false; /* 031 */ if (!resultIsNull_0) { /* 032 */ ArrayData value_6 = MapObjects_0(i); /* 033 */ boolean isNull_2 = true; /* 034 */ java.lang.Object value_2 = null; /* 035 */ if (!globalIsNull_0) { /* 036 */ /* 037 */ isNull_2 = false; /* 038 */ if (!isNull_2) { /* 039 */ /* 040 */ Object funcResult_1 = null; /* 041 */ funcResult_1 = value_6.array(); /* 042 */ value_2 = (java.lang.Object) funcResult_1; /* 043 */ /* 044 */ } /* 045 */ } /* 046 */ resultIsNull_0 = isNull_2; /* 047 */ mutableStateArray_0[0] = value_2; /* 048 */ } /* 049 */ /* 050 */ boolean isNull_1 = resultIsNull_0; /* 051 */ scala.collection.Seq value_1 = null; /* 052 */ if (!resultIsNull_0) { /* 053 */ value_1 = scala.collection.mutable.WrappedArray.make(mutableStateArray_0[0]); /* 054 */ } /* 055 */ if (isNull_1) { /* 056 */ values_0[0] = null; /* 057 */ } else { /* 058 */ values_0[0] = value_1; /* 059 */ } /* 060 */ /* 061 */ final org.apache.spark.sql.Row value_0 = new org.apache.spark.sql.catalyst.expressions.GenericRowWithSchema(values_0, ((org.apache.spark.sql.types.StructType) references[0] /* schema */)); /* 062 */ if (false) { /* 063 */ mutableRow.setNullAt(0); /* 064 */ } else { /* 065 */ /* 066 */ mutableRow.update(0, value_0); /* 067 */ } /* 068 */ /* 069 */ return mutableRow; /* 070 */ } /* 071 */ /* 072 */ /* 073 */ private ArrayData MapObjects_0(InternalRow i) { /* 074 */ boolean isNull_4 = i.isNullAt(0); /* 075 */ ArrayData value_4 = isNull_4 ? /* 076 */ null : (i.getArray(0)); /* 077 */ ArrayData value_3 = null; /* 078 */ /* 079 */ if (!isNull_4) { /* 080 */ /* 081 */ int dataLength_0 = value_4.numElements(); /* 082 */ /* 083 */ java.lang.String[] convertedArray_0 = null; /* 084 */ convertedArray_0 = new java.lang.String[dataLength_0]; /* 085 */ /* 086 */ /* 087 */ int loopIndex_0 = 0; /* 088 */ /* 089 */ while (loopIndex_0 < dataLength_0) { /* 090 */ value_MapObject_lambda_variable_1 = (UTF8String) (value_4.getUTF8String(loopIndex_0)); /* 091 */ isNull_MapObject_lambda_variable_1 = value_4.isNullAt(loopIndex_0); /* 092 */ /* 093 */ boolean isNull_5 = true; /* 094 */ java.lang.String value_5 = null; /* 095 */ if (!isNull_MapObject_lambda_variable_1) { /* 096 */ /* 097 */ isNull_5 = false; /* 098 */ if (!isNull_5) { /* 099 */ /* 100 */ Object funcResult_0 = null; /* 101 */ funcResult_0 = value_MapObject_lambda_variable_1.toString(); /* 102 */ value_5 = (java.lang.String) funcResult_0; /* 103 */ /* 104 */ } /* 105 */ } /* 106 */ if (isNull_5) { /* 107 */ convertedArray_0[loopIndex_0] = null; /* 108 */ } else { /* 109 */ convertedArray_0[loopIndex_0] = value_5; /* 110 */ } /* 111 */ /* 112 */ loopIndex_0 += 1; /* 113 */ } /* 114 */ /* 115 */ value_3 = new org.apache.spark.sql.catalyst.util.GenericArrayData(convertedArray_0); /* 116 */ } /* 117 */ globalIsNull_0 = isNull_4; /* 118 */ return value_3; /* 119 */ } /* 120 */ /* 121 */ } 20/01/23 18:54:03.229 Executor task launch worker for task 23 INFO Executor: Finished task 0.0 in stage 6.0 (TID 23). 2222 bytes result sent to driver 20/01/23 18:54:03.229 Executor task launch worker for task 23 DEBUG ExecutorMetricsPoller: stageTCMP: (6, 0) -> 3 20/01/23 18:54:03.230 dispatcher-event-loop-3 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_6.0, runningTasks: 3 20/01/23 18:54:03.230 dispatcher-event-loop-3 INFO TaskSetManager: Starting task 5.0 in stage 6.0 (TID 28, 192.168.1.225, executor driver, partition 5, PROCESS_LOCAL, 7143 bytes) 20/01/23 18:54:03.230 Executor task launch worker for task 28 INFO Executor: Running task 5.0 in stage 6.0 (TID 28) 20/01/23 18:54:03.230 task-result-getter-3 INFO TaskSetManager: Finished task 0.0 in stage 6.0 (TID 23) in 45 ms on 192.168.1.225 (executor driver) (2/10) 20/01/23 18:54:03.231 Executor task launch worker for task 28 DEBUG ExecutorMetricsPoller: stageTCMP: (6, 0) -> 4 20/01/23 18:54:03.231 Executor task launch worker for task 25 DEBUG BlockManager: Putting block rdd_24_2 without replication took 14 ms 20/01/23 18:54:03.231 Executor task launch worker for task 25 DEBUG BlockManager: Getting local block rdd_24_2 20/01/23 18:54:03.231 Executor task launch worker for task 27 DEBUG BlockManager: Getting local block rdd_24_4 20/01/23 18:54:03.231 Executor task launch worker for task 25 DEBUG BlockManager: Level for block rdd_24_2 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:03.231 Executor task launch worker for task 27 DEBUG BlockManager: Block rdd_24_4 was not found 20/01/23 18:54:03.231 Executor task launch worker for task 27 DEBUG BlockManager: Getting remote block rdd_24_4 20/01/23 18:54:03.232 Executor task launch worker for task 27 DEBUG BlockManager: Block rdd_24_4 is unknown by block manager master 20/01/23 18:54:03.232 Executor task launch worker for task 27 DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 2, partitions 4-5 20/01/23 18:54:03.232 Executor task launch worker for task 27 DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 20/01/23 18:54:03.233 Executor task launch worker for task 25 INFO MemoryStore: Block rdd_25_2 stored as values in memory (estimated size 296.0 B, free 366.1 MiB) 20/01/23 18:54:03.233 Executor task launch worker for task 27 INFO ShuffleBlockFetcherIterator: Getting 0 (0.0 B) non-empty blocks including 0 (0.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 20/01/23 18:54:03.233 Executor task launch worker for task 27 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 20/01/23 18:54:03.233 Executor task launch worker for task 27 DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: 20/01/23 18:54:03.233 Executor task launch worker for task 27 DEBUG ShuffleBlockFetcherIterator: Got local blocks in 0 ms 20/01/23 18:54:03.233 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_25_2 in memory on 192.168.1.225:35863 (size: 296.0 B, free: 366.2 MiB) 20/01/23 18:54:03.233 Executor task launch worker for task 27 DEBUG TaskMemoryManager: Task 27 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@52e13cf4 20/01/23 18:54:03.234 Executor task launch worker for task 25 DEBUG BlockManagerMaster: Updated info of block rdd_25_2 20/01/23 18:54:03.234 Executor task launch worker for task 25 DEBUG BlockManager: Told master about block rdd_25_2 20/01/23 18:54:03.234 Executor task launch worker for task 25 DEBUG BlockManager: Put block rdd_25_2 locally took 17 ms 20/01/23 18:54:03.234 Executor task launch worker for task 25 DEBUG BlockManager: Putting block rdd_25_2 without replication took 18 ms 20/01/23 18:54:03.234 Executor task launch worker for task 25 DEBUG BlockManager: Getting local block rdd_25_2 20/01/23 18:54:03.234 Executor task launch worker for task 25 DEBUG BlockManager: Level for block rdd_25_2 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:03.234 Executor task launch worker for task 27 INFO MemoryStore: Block rdd_24_4 stored as values in memory (estimated size 16.0 B, free 366.1 MiB) 20/01/23 18:54:03.234 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_24_4 in memory on 192.168.1.225:35863 (size: 16.0 B, free: 366.2 MiB) 20/01/23 18:54:03.236 Executor task launch worker for task 27 DEBUG BlockManagerMaster: Updated info of block rdd_24_4 20/01/23 18:54:03.236 Executor task launch worker for task 27 DEBUG BlockManager: Told master about block rdd_24_4 20/01/23 18:54:03.236 Executor task launch worker for task 27 DEBUG BlockManager: Put block rdd_24_4 locally took 4 ms 20/01/23 18:54:03.236 Executor task launch worker for task 24 DEBUG ALS: Start sorting an uncompressed in-block of size 3. (sortId = -444964534) 20/01/23 18:54:03.236 Executor task launch worker for task 27 DEBUG BlockManager: Putting block rdd_24_4 without replication took 4 ms 20/01/23 18:54:03.236 Executor task launch worker for task 27 DEBUG BlockManager: Getting local block rdd_24_4 20/01/23 18:54:03.236 Executor task launch worker for task 24 DEBUG ALS: Sorting took 3.834E-5 seconds. (sortId = -444964534) 20/01/23 18:54:03.236 Executor task launch worker for task 27 DEBUG BlockManager: Level for block rdd_24_4 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:03.236 Executor task launch worker for task 24 DEBUG TaskMemoryManager: Task 24 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@7073a43e 20/01/23 18:54:03.237 Executor task launch worker for task 27 INFO MemoryStore: Block rdd_25_4 stored as values in memory (estimated size 16.0 B, free 366.1 MiB) 20/01/23 18:54:03.237 Executor task launch worker for task 24 INFO MemoryStore: Block rdd_24_1 stored as values in memory (estimated size 208.0 B, free 366.1 MiB) 20/01/23 18:54:03.237 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_25_4 in memory on 192.168.1.225:35863 (size: 16.0 B, free: 366.2 MiB) 20/01/23 18:54:03.237 Executor task launch worker for task 27 DEBUG BlockManagerMaster: Updated info of block rdd_25_4 20/01/23 18:54:03.238 Executor task launch worker for task 27 DEBUG BlockManager: Told master about block rdd_25_4 20/01/23 18:54:03.238 Executor task launch worker for task 27 DEBUG BlockManager: Put block rdd_25_4 locally took 6 ms 20/01/23 18:54:03.238 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_24_1 in memory on 192.168.1.225:35863 (size: 208.0 B, free: 366.2 MiB) 20/01/23 18:54:03.238 Executor task launch worker for task 27 DEBUG BlockManager: Putting block rdd_25_4 without replication took 6 ms 20/01/23 18:54:03.238 Executor task launch worker for task 27 DEBUG BlockManager: Getting local block rdd_25_4 20/01/23 18:54:03.238 Executor task launch worker for task 27 DEBUG BlockManager: Level for block rdd_25_4 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:03.238 Executor task launch worker for task 24 DEBUG BlockManagerMaster: Updated info of block rdd_24_1 20/01/23 18:54:03.238 Executor task launch worker for task 24 DEBUG BlockManager: Told master about block rdd_24_1 20/01/23 18:54:03.238 Executor task launch worker for task 24 DEBUG BlockManager: Put block rdd_24_1 locally took 27 ms 20/01/23 18:54:03.238 Executor task launch worker for task 24 DEBUG BlockManager: Putting block rdd_24_1 without replication took 27 ms 20/01/23 18:54:03.238 Executor task launch worker for task 24 DEBUG BlockManager: Getting local block rdd_24_1 20/01/23 18:54:03.238 Executor task launch worker for task 24 DEBUG BlockManager: Level for block rdd_24_1 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:03.253 Executor task launch worker for task 28 DEBUG BlockManager: Getting local block rdd_25_5 20/01/23 18:54:03.253 Executor task launch worker for task 28 DEBUG BlockManager: Block rdd_25_5 was not found 20/01/23 18:54:03.253 Executor task launch worker for task 28 DEBUG BlockManager: Getting remote block rdd_25_5 20/01/23 18:54:03.254 Executor task launch worker for task 28 DEBUG BlockManager: Block rdd_25_5 is unknown by block manager master 20/01/23 18:54:03.254 Executor task launch worker for task 28 DEBUG BlockManager: Getting local block rdd_24_5 20/01/23 18:54:03.254 Executor task launch worker for task 28 DEBUG BlockManager: Block rdd_24_5 was not found 20/01/23 18:54:03.254 Executor task launch worker for task 28 DEBUG BlockManager: Getting remote block rdd_24_5 20/01/23 18:54:03.254 Executor task launch worker for task 25 INFO Executor: Finished task 2.0 in stage 6.0 (TID 25). 2222 bytes result sent to driver 20/01/23 18:54:03.254 Executor task launch worker for task 25 DEBUG ExecutorMetricsPoller: stageTCMP: (6, 0) -> 3 20/01/23 18:54:03.255 Executor task launch worker for task 24 INFO MemoryStore: Block rdd_25_1 stored as values in memory (estimated size 304.0 B, free 366.1 MiB) 20/01/23 18:54:03.255 Executor task launch worker for task 27 INFO Executor: Finished task 4.0 in stage 6.0 (TID 27). 2222 bytes result sent to driver 20/01/23 18:54:03.255 Executor task launch worker for task 27 DEBUG ExecutorMetricsPoller: stageTCMP: (6, 0) -> 2 20/01/23 18:54:03.256 dispatcher-event-loop-2 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_6.0, runningTasks: 3 20/01/23 18:54:03.256 dispatcher-event-loop-2 INFO TaskSetManager: Starting task 6.0 in stage 6.0 (TID 29, 192.168.1.225, executor driver, partition 6, PROCESS_LOCAL, 7143 bytes) 20/01/23 18:54:03.257 task-result-getter-2 INFO TaskSetManager: Finished task 2.0 in stage 6.0 (TID 25) in 71 ms on 192.168.1.225 (executor driver) (3/10) 20/01/23 18:54:03.259 dispatcher-event-loop-2 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_6.0, runningTasks: 3 20/01/23 18:54:03.259 dispatcher-event-loop-2 INFO TaskSetManager: Starting task 7.0 in stage 6.0 (TID 30, 192.168.1.225, executor driver, partition 7, PROCESS_LOCAL, 7143 bytes) 20/01/23 18:54:03.260 Executor task launch worker for task 30 INFO Executor: Running task 7.0 in stage 6.0 (TID 30) 20/01/23 18:54:03.260 task-result-getter-1 INFO TaskSetManager: Finished task 4.0 in stage 6.0 (TID 27) in 45 ms on 192.168.1.225 (executor driver) (4/10) 20/01/23 18:54:03.258 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_25_1 in memory on 192.168.1.225:35863 (size: 304.0 B, free: 366.2 MiB) 20/01/23 18:54:03.258 Executor task launch worker for task 28 DEBUG BlockManager: Block rdd_24_5 is unknown by block manager master 20/01/23 18:54:03.258 Executor task launch worker for task 29 INFO Executor: Running task 6.0 in stage 6.0 (TID 29) 20/01/23 18:54:03.260 Executor task launch worker for task 30 DEBUG ExecutorMetricsPoller: stageTCMP: (6, 0) -> 3 20/01/23 18:54:03.260 Executor task launch worker for task 24 DEBUG BlockManagerMaster: Updated info of block rdd_25_1 20/01/23 18:54:03.260 Executor task launch worker for task 29 DEBUG ExecutorMetricsPoller: stageTCMP: (6, 0) -> 4 20/01/23 18:54:03.260 Executor task launch worker for task 28 DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 2, partitions 5-6 20/01/23 18:54:03.261 Executor task launch worker for task 28 DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 20/01/23 18:54:03.262 Executor task launch worker for task 28 INFO ShuffleBlockFetcherIterator: Getting 0 (0.0 B) non-empty blocks including 0 (0.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 20/01/23 18:54:03.262 Executor task launch worker for task 28 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 20/01/23 18:54:03.262 Executor task launch worker for task 28 DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: 20/01/23 18:54:03.262 Executor task launch worker for task 28 DEBUG ShuffleBlockFetcherIterator: Got local blocks in 0 ms 20/01/23 18:54:03.262 Executor task launch worker for task 24 DEBUG BlockManager: Told master about block rdd_25_1 20/01/23 18:54:03.263 Executor task launch worker for task 28 DEBUG TaskMemoryManager: Task 28 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@30d56e70 20/01/23 18:54:03.263 Executor task launch worker for task 24 DEBUG BlockManager: Put block rdd_25_1 locally took 51 ms 20/01/23 18:54:03.263 Executor task launch worker for task 24 DEBUG BlockManager: Putting block rdd_25_1 without replication took 52 ms 20/01/23 18:54:03.263 Executor task launch worker for task 24 DEBUG BlockManager: Getting local block rdd_25_1 20/01/23 18:54:03.263 Executor task launch worker for task 24 DEBUG BlockManager: Level for block rdd_25_1 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:03.263 Executor task launch worker for task 28 INFO MemoryStore: Block rdd_24_5 stored as values in memory (estimated size 16.0 B, free 366.1 MiB) 20/01/23 18:54:03.264 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_24_5 in memory on 192.168.1.225:35863 (size: 16.0 B, free: 366.2 MiB) 20/01/23 18:54:03.264 Executor task launch worker for task 28 DEBUG BlockManagerMaster: Updated info of block rdd_24_5 20/01/23 18:54:03.264 Executor task launch worker for task 28 DEBUG BlockManager: Told master about block rdd_24_5 20/01/23 18:54:03.264 Executor task launch worker for task 28 DEBUG BlockManager: Put block rdd_24_5 locally took 3 ms 20/01/23 18:54:03.264 Executor task launch worker for task 28 DEBUG BlockManager: Putting block rdd_24_5 without replication took 3 ms 20/01/23 18:54:03.264 Executor task launch worker for task 28 DEBUG BlockManager: Getting local block rdd_24_5 20/01/23 18:54:03.264 Executor task launch worker for task 28 DEBUG BlockManager: Level for block rdd_24_5 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:03.265 Executor task launch worker for task 28 INFO MemoryStore: Block rdd_25_5 stored as values in memory (estimated size 16.0 B, free 366.1 MiB) 20/01/23 18:54:03.265 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_25_5 in memory on 192.168.1.225:35863 (size: 16.0 B, free: 366.2 MiB) 20/01/23 18:54:03.265 Executor task launch worker for task 28 DEBUG BlockManagerMaster: Updated info of block rdd_25_5 20/01/23 18:54:03.266 Executor task launch worker for task 28 DEBUG BlockManager: Told master about block rdd_25_5 20/01/23 18:54:03.266 Executor task launch worker for task 28 DEBUG BlockManager: Put block rdd_25_5 locally took 11 ms 20/01/23 18:54:03.266 Executor task launch worker for task 28 DEBUG BlockManager: Putting block rdd_25_5 without replication took 11 ms 20/01/23 18:54:03.266 Executor task launch worker for task 28 DEBUG BlockManager: Getting local block rdd_25_5 20/01/23 18:54:03.266 Executor task launch worker for task 28 DEBUG BlockManager: Level for block rdd_25_5 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:03.266 Executor task launch worker for task 24 INFO Executor: Finished task 1.0 in stage 6.0 (TID 24). 2222 bytes result sent to driver 20/01/23 18:54:03.266 Executor task launch worker for task 30 DEBUG BlockManager: Getting local block rdd_25_7 20/01/23 18:54:03.266 Executor task launch worker for task 24 DEBUG ExecutorMetricsPoller: stageTCMP: (6, 0) -> 3 20/01/23 18:54:03.266 Executor task launch worker for task 30 DEBUG BlockManager: Block rdd_25_7 was not found 20/01/23 18:54:03.266 Executor task launch worker for task 30 DEBUG BlockManager: Getting remote block rdd_25_7 20/01/23 18:54:03.267 Executor task launch worker for task 30 DEBUG BlockManager: Block rdd_25_7 is unknown by block manager master 20/01/23 18:54:03.267 dispatcher-event-loop-1 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_6.0, runningTasks: 3 20/01/23 18:54:03.267 Executor task launch worker for task 29 DEBUG BlockManager: Getting local block rdd_25_6 20/01/23 18:54:03.267 Executor task launch worker for task 29 DEBUG BlockManager: Block rdd_25_6 was not found 20/01/23 18:54:03.267 Executor task launch worker for task 29 DEBUG BlockManager: Getting remote block rdd_25_6 20/01/23 18:54:03.267 dispatcher-event-loop-1 INFO TaskSetManager: Starting task 8.0 in stage 6.0 (TID 31, 192.168.1.225, executor driver, partition 8, PROCESS_LOCAL, 7143 bytes) 20/01/23 18:54:03.267 Executor task launch worker for task 31 INFO Executor: Running task 8.0 in stage 6.0 (TID 31) 20/01/23 18:54:03.267 task-result-getter-0 INFO TaskSetManager: Finished task 1.0 in stage 6.0 (TID 24) in 82 ms on 192.168.1.225 (executor driver) (5/10) 20/01/23 18:54:03.267 Executor task launch worker for task 30 DEBUG BlockManager: Getting local block rdd_24_7 20/01/23 18:54:03.267 Executor task launch worker for task 30 DEBUG BlockManager: Block rdd_24_7 was not found 20/01/23 18:54:03.267 Executor task launch worker for task 30 DEBUG BlockManager: Getting remote block rdd_24_7 20/01/23 18:54:03.268 Executor task launch worker for task 31 DEBUG ExecutorMetricsPoller: stageTCMP: (6, 0) -> 4 20/01/23 18:54:03.269 Executor task launch worker for task 30 DEBUG BlockManager: Block rdd_24_7 is unknown by block manager master 20/01/23 18:54:03.269 Executor task launch worker for task 30 DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 2, partitions 7-8 20/01/23 18:54:03.267 Executor task launch worker for task 28 INFO Executor: Finished task 5.0 in stage 6.0 (TID 28). 2222 bytes result sent to driver 20/01/23 18:54:03.270 dispatcher-event-loop-0 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_6.0, runningTasks: 3 20/01/23 18:54:03.270 dispatcher-event-loop-0 INFO TaskSetManager: Starting task 9.0 in stage 6.0 (TID 32, 192.168.1.225, executor driver, partition 9, PROCESS_LOCAL, 7143 bytes) 20/01/23 18:54:03.270 Executor task launch worker for task 30 DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 20/01/23 18:54:03.270 Executor task launch worker for task 28 DEBUG ExecutorMetricsPoller: stageTCMP: (6, 0) -> 3 20/01/23 18:54:03.271 Executor task launch worker for task 32 INFO Executor: Running task 9.0 in stage 6.0 (TID 32) 20/01/23 18:54:03.271 Executor task launch worker for task 29 DEBUG BlockManager: Block rdd_25_6 is unknown by block manager master 20/01/23 18:54:03.271 Executor task launch worker for task 29 DEBUG BlockManager: Getting local block rdd_24_6 20/01/23 18:54:03.271 Executor task launch worker for task 29 DEBUG BlockManager: Block rdd_24_6 was not found 20/01/23 18:54:03.271 Executor task launch worker for task 30 INFO ShuffleBlockFetcherIterator: Getting 0 (0.0 B) non-empty blocks including 0 (0.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 20/01/23 18:54:03.271 Executor task launch worker for task 32 DEBUG ExecutorMetricsPoller: stageTCMP: (6, 0) -> 4 20/01/23 18:54:03.271 Executor task launch worker for task 29 DEBUG BlockManager: Getting remote block rdd_24_6 20/01/23 18:54:03.271 Executor task launch worker for task 29 DEBUG BlockManager: Block rdd_24_6 is unknown by block manager master 20/01/23 18:54:03.271 Executor task launch worker for task 29 DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 2, partitions 6-7 20/01/23 18:54:03.272 Executor task launch worker for task 29 DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 20/01/23 18:54:03.272 Executor task launch worker for task 29 INFO ShuffleBlockFetcherIterator: Getting 0 (0.0 B) non-empty blocks including 0 (0.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 20/01/23 18:54:03.272 Executor task launch worker for task 29 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 20/01/23 18:54:03.272 Executor task launch worker for task 29 DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: 20/01/23 18:54:03.272 Executor task launch worker for task 29 DEBUG ShuffleBlockFetcherIterator: Got local blocks in 0 ms 20/01/23 18:54:03.273 Executor task launch worker for task 29 DEBUG TaskMemoryManager: Task 29 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@c68825f 20/01/23 18:54:03.273 Executor task launch worker for task 29 INFO MemoryStore: Block rdd_24_6 stored as values in memory (estimated size 16.0 B, free 366.1 MiB) 20/01/23 18:54:03.271 Executor task launch worker for task 30 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 20/01/23 18:54:03.274 Executor task launch worker for task 30 DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: 20/01/23 18:54:03.274 Executor task launch worker for task 30 DEBUG ShuffleBlockFetcherIterator: Got local blocks in 3 ms 20/01/23 18:54:03.274 Executor task launch worker for task 30 DEBUG TaskMemoryManager: Task 30 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@5b080e1c 20/01/23 18:54:03.275 Executor task launch worker for task 30 INFO MemoryStore: Block rdd_24_7 stored as values in memory (estimated size 16.0 B, free 366.1 MiB) 20/01/23 18:54:03.275 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_24_6 in memory on 192.168.1.225:35863 (size: 16.0 B, free: 366.2 MiB) 20/01/23 18:54:03.276 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_24_7 in memory on 192.168.1.225:35863 (size: 16.0 B, free: 366.2 MiB) 20/01/23 18:54:03.277 Executor task launch worker for task 30 DEBUG BlockManagerMaster: Updated info of block rdd_24_7 20/01/23 18:54:03.277 Executor task launch worker for task 30 DEBUG BlockManager: Told master about block rdd_24_7 20/01/23 18:54:03.271 task-result-getter-3 INFO TaskSetManager: Finished task 5.0 in stage 6.0 (TID 28) in 40 ms on 192.168.1.225 (executor driver) (6/10) 20/01/23 18:54:03.278 Executor task launch worker for task 31 DEBUG BlockManager: Getting local block rdd_25_8 20/01/23 18:54:03.277 Executor task launch worker for task 30 DEBUG BlockManager: Put block rdd_24_7 locally took 8 ms 20/01/23 18:54:03.279 Executor task launch worker for task 30 DEBUG BlockManager: Putting block rdd_24_7 without replication took 9 ms 20/01/23 18:54:03.279 Executor task launch worker for task 30 DEBUG BlockManager: Getting local block rdd_24_7 20/01/23 18:54:03.279 Executor task launch worker for task 30 DEBUG BlockManager: Level for block rdd_24_7 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:03.279 Executor task launch worker for task 30 INFO MemoryStore: Block rdd_25_7 stored as values in memory (estimated size 16.0 B, free 366.1 MiB) 20/01/23 18:54:03.277 Executor task launch worker for task 29 DEBUG BlockManagerMaster: Updated info of block rdd_24_6 20/01/23 18:54:03.277 Executor task launch worker for task 32 DEBUG BlockManager: Getting local block rdd_25_9 20/01/23 18:54:03.280 Executor task launch worker for task 32 DEBUG BlockManager: Block rdd_25_9 was not found 20/01/23 18:54:03.280 Executor task launch worker for task 32 DEBUG BlockManager: Getting remote block rdd_25_9 20/01/23 18:54:03.280 Executor task launch worker for task 29 DEBUG BlockManager: Told master about block rdd_24_6 20/01/23 18:54:03.280 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_25_7 in memory on 192.168.1.225:35863 (size: 16.0 B, free: 366.2 MiB) 20/01/23 18:54:03.281 Executor task launch worker for task 32 DEBUG BlockManager: Block rdd_25_9 is unknown by block manager master 20/01/23 18:54:03.281 Executor task launch worker for task 32 DEBUG BlockManager: Getting local block rdd_24_9 20/01/23 18:54:03.281 Executor task launch worker for task 32 DEBUG BlockManager: Block rdd_24_9 was not found 20/01/23 18:54:03.281 Executor task launch worker for task 32 DEBUG BlockManager: Getting remote block rdd_24_9 20/01/23 18:54:03.281 Executor task launch worker for task 32 DEBUG BlockManager: Block rdd_24_9 is unknown by block manager master 20/01/23 18:54:03.281 Executor task launch worker for task 32 DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 2, partitions 9-10 20/01/23 18:54:03.282 Executor task launch worker for task 32 DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 20/01/23 18:54:03.282 Executor task launch worker for task 32 INFO ShuffleBlockFetcherIterator: Getting 0 (0.0 B) non-empty blocks including 0 (0.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 20/01/23 18:54:03.282 Executor task launch worker for task 32 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 20/01/23 18:54:03.282 Executor task launch worker for task 32 DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: 20/01/23 18:54:03.282 Executor task launch worker for task 32 DEBUG ShuffleBlockFetcherIterator: Got local blocks in 0 ms 20/01/23 18:54:03.283 Executor task launch worker for task 32 DEBUG TaskMemoryManager: Task 32 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@2cc234f7 20/01/23 18:54:03.283 Executor task launch worker for task 32 INFO MemoryStore: Block rdd_24_9 stored as values in memory (estimated size 16.0 B, free 366.1 MiB) 20/01/23 18:54:03.284 Executor task launch worker for task 30 DEBUG BlockManagerMaster: Updated info of block rdd_25_7 20/01/23 18:54:03.284 Executor task launch worker for task 30 DEBUG BlockManager: Told master about block rdd_25_7 20/01/23 18:54:03.284 Executor task launch worker for task 30 DEBUG BlockManager: Put block rdd_25_7 locally took 16 ms 20/01/23 18:54:03.284 Executor task launch worker for task 30 DEBUG BlockManager: Putting block rdd_25_7 without replication took 16 ms 20/01/23 18:54:03.284 Executor task launch worker for task 30 DEBUG BlockManager: Getting local block rdd_25_7 20/01/23 18:54:03.284 Executor task launch worker for task 30 DEBUG BlockManager: Level for block rdd_25_7 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:03.284 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_24_9 in memory on 192.168.1.225:35863 (size: 16.0 B, free: 366.2 MiB) 20/01/23 18:54:03.285 Executor task launch worker for task 32 DEBUG BlockManagerMaster: Updated info of block rdd_24_9 20/01/23 18:54:03.285 Executor task launch worker for task 32 DEBUG BlockManager: Told master about block rdd_24_9 20/01/23 18:54:03.285 Executor task launch worker for task 32 DEBUG BlockManager: Put block rdd_24_9 locally took 3 ms 20/01/23 18:54:03.285 Executor task launch worker for task 32 DEBUG BlockManager: Putting block rdd_24_9 without replication took 3 ms 20/01/23 18:54:03.285 Executor task launch worker for task 32 DEBUG BlockManager: Getting local block rdd_24_9 20/01/23 18:54:03.285 Executor task launch worker for task 32 DEBUG BlockManager: Level for block rdd_24_9 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:03.286 Executor task launch worker for task 30 INFO Executor: Finished task 7.0 in stage 6.0 (TID 30). 2222 bytes result sent to driver 20/01/23 18:54:03.287 Executor task launch worker for task 32 INFO MemoryStore: Block rdd_25_9 stored as values in memory (estimated size 16.0 B, free 366.1 MiB) 20/01/23 18:54:03.287 dispatcher-event-loop-1 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_6.0, runningTasks: 3 20/01/23 18:54:03.288 task-result-getter-2 INFO TaskSetManager: Finished task 7.0 in stage 6.0 (TID 30) in 29 ms on 192.168.1.225 (executor driver) (7/10) 20/01/23 18:54:03.288 Executor task launch worker for task 30 DEBUG ExecutorMetricsPoller: stageTCMP: (6, 0) -> 3 20/01/23 18:54:03.278 Executor task launch worker for task 31 DEBUG BlockManager: Block rdd_25_8 was not found 20/01/23 18:54:03.289 Executor task launch worker for task 31 DEBUG BlockManager: Getting remote block rdd_25_8 20/01/23 18:54:03.289 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_25_9 in memory on 192.168.1.225:35863 (size: 16.0 B, free: 366.2 MiB) 20/01/23 18:54:03.280 Executor task launch worker for task 29 DEBUG BlockManager: Put block rdd_24_6 locally took 8 ms 20/01/23 18:54:03.289 Executor task launch worker for task 29 DEBUG BlockManager: Putting block rdd_24_6 without replication took 18 ms 20/01/23 18:54:03.289 Executor task launch worker for task 31 DEBUG BlockManager: Block rdd_25_8 is unknown by block manager master 20/01/23 18:54:03.288 Thread-4 DEBUG WholeStageCodegenExec: /* 001 */ public Object generate(Object[] references) { /* 002 */ return new GeneratedIteratorForCodegenStage2(references); /* 003 */ } /* 004 */ /* 005 */ // codegenStageId=2 /* 006 */ final class GeneratedIteratorForCodegenStage2 extends org.apache.spark.sql.execution.BufferedRowIterator { /* 007 */ private Object[] references; /* 008 */ private scala.collection.Iterator[] inputs; /* 009 */ private boolean agg_initAgg_0; /* 010 */ private boolean agg_bufIsNull_0; /* 011 */ private double agg_bufValue_0; /* 012 */ private boolean agg_bufIsNull_1; /* 013 */ private long agg_bufValue_1; /* 014 */ private scala.collection.Iterator inputadapter_input_0; /* 015 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[] agg_mutableStateArray_0 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[1]; /* 016 */ /* 017 */ public GeneratedIteratorForCodegenStage2(Object[] references) { /* 018 */ this.references = references; /* 019 */ } /* 020 */ /* 021 */ public void init(int index, scala.collection.Iterator[] inputs) { /* 022 */ partitionIndex = index; /* 023 */ this.inputs = inputs; /* 024 */ /* 025 */ inputadapter_input_0 = inputs[0]; /* 026 */ agg_mutableStateArray_0[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(1, 0); /* 027 */ /* 028 */ } /* 029 */ /* 030 */ private void agg_doAggregateWithoutKey_0() throws java.io.IOException { /* 031 */ // initialize aggregation buffer /* 032 */ agg_bufIsNull_0 = false; /* 033 */ agg_bufValue_0 = 0.0D; /* 034 */ agg_bufIsNull_1 = false; /* 035 */ agg_bufValue_1 = 0L; /* 036 */ /* 037 */ while ( inputadapter_input_0.hasNext()) { /* 038 */ InternalRow inputadapter_row_0 = (InternalRow) inputadapter_input_0.next(); /* 039 */ /* 040 */ boolean inputadapter_isNull_0 = inputadapter_row_0.isNullAt(0); /* 041 */ double inputadapter_value_0 = inputadapter_isNull_0 ? /* 042 */ -1.0 : (inputadapter_row_0.getDouble(0)); /* 043 */ boolean inputadapter_isNull_1 = inputadapter_row_0.isNullAt(1); /* 044 */ long inputadapter_value_1 = inputadapter_isNull_1 ? /* 045 */ -1L : (inputadapter_row_0.getLong(1)); /* 046 */ /* 047 */ agg_doConsume_0(inputadapter_row_0, inputadapter_value_0, inputadapter_isNull_0, inputadapter_value_1, inputadapter_isNull_1); /* 048 */ // shouldStop check is eliminated /* 049 */ } /* 050 */ /* 051 */ } /* 052 */ /* 053 */ private void agg_doConsume_0(InternalRow inputadapter_row_0, double agg_expr_0_0, boolean agg_exprIsNull_0_0, long agg_expr_1_0, boolean agg_exprIsNull_1_0) throws java.io.IOException { /* 054 */ // do aggregate /* 055 */ // common sub-expressions /* 056 */ /* 057 */ // evaluate aggregate functions and update aggregation buffers /* 058 */ /* 059 */ // do aggregate for avg /* 060 */ // evaluate aggregate function /* 061 */ boolean agg_isNull_7 = true; /* 062 */ double agg_value_7 = -1.0; /* 063 */ /* 064 */ if (!agg_bufIsNull_0) { /* 065 */ if (!agg_exprIsNull_0_0) { /* 066 */ agg_isNull_7 = false; // resultCode could change nullability. /* 067 */ /* 068 */ agg_value_7 = agg_bufValue_0 + agg_expr_0_0; /* 069 */ /* 070 */ } /* 071 */ /* 072 */ } /* 073 */ boolean agg_isNull_10 = true; /* 074 */ long agg_value_10 = -1L; /* 075 */ /* 076 */ if (!agg_bufIsNull_1) { /* 077 */ if (!agg_exprIsNull_1_0) { /* 078 */ agg_isNull_10 = false; // resultCode could change nullability. /* 079 */ /* 080 */ agg_value_10 = agg_bufValue_1 + agg_expr_1_0; /* 081 */ /* 082 */ } /* 083 */ /* 084 */ } /* 085 */ // update aggregation buffers /* 086 */ agg_bufIsNull_0 = agg_isNull_7; /* 087 */ agg_bufValue_0 = agg_value_7; /* 088 */ /* 089 */ agg_bufIsNull_1 = agg_isNull_10; /* 090 */ agg_bufValue_1 = agg_value_10; /* 091 */ /* 092 */ } /* 093 */ /* 094 */ protected void processNext() throws java.io.IOException { /* 095 */ while (!agg_initAgg_0) { /* 096 */ agg_initAgg_0 = true; /* 097 */ long agg_beforeAgg_0 = System.nanoTime(); /* 098 */ agg_doAggregateWithoutKey_0(); /* 099 */ ((org.apache.spark.sql.execution.metric.SQLMetric) references[1] /* aggTime */).add((System.nanoTime() - agg_beforeAgg_0) / 1000000); /* 100 */ /* 101 */ // output the result /* 102 */ boolean agg_isNull_4 = agg_bufIsNull_1; /* 103 */ double agg_value_4 = -1.0; /* 104 */ if (!agg_bufIsNull_1) { /* 105 */ agg_value_4 = (double) agg_bufValue_1; /* 106 */ } /* 107 */ boolean agg_isNull_2 = false; /* 108 */ double agg_value_2 = -1.0; /* 109 */ if (agg_isNull_4 || agg_value_4 == 0) { /* 110 */ agg_isNull_2 = true; /* 111 */ } else { /* 112 */ if (agg_bufIsNull_0) { /* 113 */ agg_isNull_2 = true; /* 114 */ } else { /* 115 */ agg_value_2 = (double)(agg_bufValue_0 / agg_value_4); /* 116 */ } /* 117 */ } /* 118 */ /* 119 */ ((org.apache.spark.sql.execution.metric.SQLMetric) references[0] /* numOutputRows */).add(1); /* 120 */ agg_mutableStateArray_0[0].reset(); /* 121 */ /* 122 */ agg_mutableStateArray_0[0].zeroOutNullBytes(); /* 123 */ /* 124 */ if (agg_isNull_2) { /* 125 */ agg_mutableStateArray_0[0].setNullAt(0); /* 126 */ } else { /* 127 */ agg_mutableStateArray_0[0].write(0, agg_value_2); /* 128 */ } /* 129 */ append((agg_mutableStateArray_0[0].getRow())); /* 130 */ } /* 131 */ } /* 132 */ /* 133 */ } 20/01/23 18:54:03.289 Executor task launch worker for task 29 DEBUG BlockManager: Getting local block rdd_24_6 20/01/23 18:54:03.290 Executor task launch worker for task 31 DEBUG BlockManager: Getting local block rdd_24_8 20/01/23 18:54:03.290 Executor task launch worker for task 29 DEBUG BlockManager: Level for block rdd_24_6 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:03.290 Executor task launch worker for task 29 INFO MemoryStore: Block rdd_25_6 stored as values in memory (estimated size 16.0 B, free 366.1 MiB) 20/01/23 18:54:03.289 Executor task launch worker for task 32 DEBUG BlockManagerMaster: Updated info of block rdd_25_9 20/01/23 18:54:03.290 Executor task launch worker for task 31 DEBUG BlockManager: Block rdd_24_8 was not found 20/01/23 18:54:03.291 Executor task launch worker for task 31 DEBUG BlockManager: Getting remote block rdd_24_8 20/01/23 18:54:03.291 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_25_6 in memory on 192.168.1.225:35863 (size: 16.0 B, free: 366.2 MiB) 20/01/23 18:54:03.291 Executor task launch worker for task 29 DEBUG BlockManagerMaster: Updated info of block rdd_25_6 20/01/23 18:54:03.291 Executor task launch worker for task 29 DEBUG BlockManager: Told master about block rdd_25_6 20/01/23 18:54:03.291 Executor task launch worker for task 29 DEBUG BlockManager: Put block rdd_25_6 locally took 20 ms 20/01/23 18:54:03.291 Executor task launch worker for task 31 DEBUG BlockManager: Block rdd_24_8 is unknown by block manager master 20/01/23 18:54:03.291 Executor task launch worker for task 29 DEBUG BlockManager: Putting block rdd_25_6 without replication took 20 ms 20/01/23 18:54:03.291 Executor task launch worker for task 29 DEBUG BlockManager: Getting local block rdd_25_6 20/01/23 18:54:03.291 Executor task launch worker for task 31 DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 2, partitions 8-9 20/01/23 18:54:03.292 Executor task launch worker for task 31 DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 20/01/23 18:54:03.292 Executor task launch worker for task 31 INFO ShuffleBlockFetcherIterator: Getting 0 (0.0 B) non-empty blocks including 0 (0.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 20/01/23 18:54:03.291 Executor task launch worker for task 32 DEBUG BlockManager: Told master about block rdd_25_9 20/01/23 18:54:03.292 Executor task launch worker for task 31 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 20/01/23 18:54:03.293 Executor task launch worker for task 32 DEBUG BlockManager: Put block rdd_25_9 locally took 11 ms 20/01/23 18:54:03.292 Executor task launch worker for task 29 DEBUG BlockManager: Level for block rdd_25_6 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:03.293 Executor task launch worker for task 31 DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: 20/01/23 18:54:03.293 Executor task launch worker for task 31 DEBUG ShuffleBlockFetcherIterator: Got local blocks in 0 ms 20/01/23 18:54:03.293 Executor task launch worker for task 32 DEBUG BlockManager: Putting block rdd_25_9 without replication took 11 ms 20/01/23 18:54:03.293 Executor task launch worker for task 32 DEBUG BlockManager: Getting local block rdd_25_9 20/01/23 18:54:03.293 Executor task launch worker for task 32 DEBUG BlockManager: Level for block rdd_25_9 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:03.293 Executor task launch worker for task 31 DEBUG TaskMemoryManager: Task 31 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@f012859 20/01/23 18:54:03.294 Executor task launch worker for task 29 INFO Executor: Finished task 6.0 in stage 6.0 (TID 29). 2222 bytes result sent to driver 20/01/23 18:54:03.294 Executor task launch worker for task 31 INFO MemoryStore: Block rdd_24_8 stored as values in memory (estimated size 16.0 B, free 366.1 MiB) 20/01/23 18:54:03.294 dispatcher-event-loop-2 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_6.0, runningTasks: 2 20/01/23 18:54:03.295 Executor task launch worker for task 29 DEBUG ExecutorMetricsPoller: stageTCMP: (6, 0) -> 2 20/01/23 18:54:03.295 task-result-getter-1 INFO TaskSetManager: Finished task 6.0 in stage 6.0 (TID 29) in 39 ms on 192.168.1.225 (executor driver) (8/10) 20/01/23 18:54:03.295 Executor task launch worker for task 32 INFO Executor: Finished task 9.0 in stage 6.0 (TID 32). 2222 bytes result sent to driver 20/01/23 18:54:03.295 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_24_8 in memory on 192.168.1.225:35863 (size: 16.0 B, free: 366.2 MiB) 20/01/23 18:54:03.296 dispatcher-event-loop-0 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_6.0, runningTasks: 1 20/01/23 18:54:03.296 Executor task launch worker for task 31 DEBUG BlockManagerMaster: Updated info of block rdd_24_8 20/01/23 18:54:03.296 Executor task launch worker for task 31 DEBUG BlockManager: Told master about block rdd_24_8 20/01/23 18:54:03.296 Executor task launch worker for task 31 DEBUG BlockManager: Put block rdd_24_8 locally took 4 ms 20/01/23 18:54:03.296 Executor task launch worker for task 31 DEBUG BlockManager: Putting block rdd_24_8 without replication took 4 ms 20/01/23 18:54:03.296 Executor task launch worker for task 31 DEBUG BlockManager: Getting local block rdd_24_8 20/01/23 18:54:03.296 task-result-getter-0 INFO TaskSetManager: Finished task 9.0 in stage 6.0 (TID 32) in 26 ms on 192.168.1.225 (executor driver) (9/10) 20/01/23 18:54:03.297 Thread-4 DEBUG CodeGenerator: /* 001 */ public Object generate(Object[] references) { /* 002 */ return new GeneratedIteratorForCodegenStage2(references); /* 003 */ } /* 004 */ /* 005 */ // codegenStageId=2 /* 006 */ final class GeneratedIteratorForCodegenStage2 extends org.apache.spark.sql.execution.BufferedRowIterator { /* 007 */ private Object[] references; /* 008 */ private scala.collection.Iterator[] inputs; /* 009 */ private boolean agg_initAgg_0; /* 010 */ private boolean agg_bufIsNull_0; /* 011 */ private double agg_bufValue_0; /* 012 */ private boolean agg_bufIsNull_1; /* 013 */ private long agg_bufValue_1; /* 014 */ private scala.collection.Iterator inputadapter_input_0; /* 015 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[] agg_mutableStateArray_0 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[1]; /* 016 */ /* 017 */ public GeneratedIteratorForCodegenStage2(Object[] references) { /* 018 */ this.references = references; /* 019 */ } /* 020 */ /* 021 */ public void init(int index, scala.collection.Iterator[] inputs) { /* 022 */ partitionIndex = index; /* 023 */ this.inputs = inputs; /* 024 */ /* 025 */ inputadapter_input_0 = inputs[0]; /* 026 */ agg_mutableStateArray_0[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(1, 0); /* 027 */ /* 028 */ } /* 029 */ /* 030 */ private void agg_doAggregateWithoutKey_0() throws java.io.IOException { /* 031 */ // initialize aggregation buffer /* 032 */ agg_bufIsNull_0 = false; /* 033 */ agg_bufValue_0 = 0.0D; /* 034 */ agg_bufIsNull_1 = false; /* 035 */ agg_bufValue_1 = 0L; /* 036 */ /* 037 */ while ( inputadapter_input_0.hasNext()) { /* 038 */ InternalRow inputadapter_row_0 = (InternalRow) inputadapter_input_0.next(); /* 039 */ /* 040 */ boolean inputadapter_isNull_0 = inputadapter_row_0.isNullAt(0); /* 041 */ double inputadapter_value_0 = inputadapter_isNull_0 ? /* 042 */ -1.0 : (inputadapter_row_0.getDouble(0)); /* 043 */ boolean inputadapter_isNull_1 = inputadapter_row_0.isNullAt(1); /* 044 */ long inputadapter_value_1 = inputadapter_isNull_1 ? /* 045 */ -1L : (inputadapter_row_0.getLong(1)); /* 046 */ /* 047 */ agg_doConsume_0(inputadapter_row_0, inputadapter_value_0, inputadapter_isNull_0, inputadapter_value_1, inputadapter_isNull_1); /* 048 */ // shouldStop check is eliminated /* 049 */ } /* 050 */ /* 051 */ } /* 052 */ /* 053 */ private void agg_doConsume_0(InternalRow inputadapter_row_0, double agg_expr_0_0, boolean agg_exprIsNull_0_0, long agg_expr_1_0, boolean agg_exprIsNull_1_0) throws java.io.IOException { /* 054 */ // do aggregate /* 055 */ // common sub-expressions /* 056 */ /* 057 */ // evaluate aggregate functions and update aggregation buffers /* 058 */ /* 059 */ // do aggregate for avg /* 060 */ // evaluate aggregate function /* 061 */ boolean agg_isNull_7 = true; /* 062 */ double agg_value_7 = -1.0; /* 063 */ /* 064 */ if (!agg_bufIsNull_0) { /* 065 */ if (!agg_exprIsNull_0_0) { /* 066 */ agg_isNull_7 = false; // resultCode could change nullability. /* 067 */ /* 068 */ agg_value_7 = agg_bufValue_0 + agg_expr_0_0; /* 069 */ /* 070 */ } /* 071 */ /* 072 */ } /* 073 */ boolean agg_isNull_10 = true; /* 074 */ long agg_value_10 = -1L; /* 075 */ /* 076 */ if (!agg_bufIsNull_1) { /* 077 */ if (!agg_exprIsNull_1_0) { /* 078 */ agg_isNull_10 = false; // resultCode could change nullability. /* 079 */ /* 080 */ agg_value_10 = agg_bufValue_1 + agg_expr_1_0; /* 081 */ /* 082 */ } /* 083 */ /* 084 */ } /* 085 */ // update aggregation buffers /* 086 */ agg_bufIsNull_0 = agg_isNull_7; /* 087 */ agg_bufValue_0 = agg_value_7; /* 088 */ /* 089 */ agg_bufIsNull_1 = agg_isNull_10; /* 090 */ agg_bufValue_1 = agg_value_10; /* 091 */ /* 092 */ } /* 093 */ /* 094 */ protected void processNext() throws java.io.IOException { /* 095 */ while (!agg_initAgg_0) { /* 096 */ agg_initAgg_0 = true; /* 097 */ long agg_beforeAgg_0 = System.nanoTime(); /* 098 */ agg_doAggregateWithoutKey_0(); /* 099 */ ((org.apache.spark.sql.execution.metric.SQLMetric) references[1] /* aggTime */).add((System.nanoTime() - agg_beforeAgg_0) / 1000000); /* 100 */ /* 101 */ // output the result /* 102 */ boolean agg_isNull_4 = agg_bufIsNull_1; /* 103 */ double agg_value_4 = -1.0; /* 104 */ if (!agg_bufIsNull_1) { /* 105 */ agg_value_4 = (double) agg_bufValue_1; /* 106 */ } /* 107 */ boolean agg_isNull_2 = false; /* 108 */ double agg_value_2 = -1.0; /* 109 */ if (agg_isNull_4 || agg_value_4 == 0) { /* 110 */ agg_isNull_2 = true; /* 111 */ } else { /* 112 */ if (agg_bufIsNull_0) { /* 113 */ agg_isNull_2 = true; /* 114 */ } else { /* 115 */ agg_value_2 = (double)(agg_bufValue_0 / agg_value_4); /* 116 */ } /* 117 */ } /* 118 */ /* 119 */ ((org.apache.spark.sql.execution.metric.SQLMetric) references[0] /* numOutputRows */).add(1); /* 120 */ agg_mutableStateArray_0[0].reset(); /* 121 */ /* 122 */ agg_mutableStateArray_0[0].zeroOutNullBytes(); /* 123 */ /* 124 */ if (agg_isNull_2) { /* 125 */ agg_mutableStateArray_0[0].setNullAt(0); /* 126 */ } else { /* 127 */ agg_mutableStateArray_0[0].write(0, agg_value_2); /* 128 */ } /* 129 */ append((agg_mutableStateArray_0[0].getRow())); /* 130 */ } /* 131 */ } /* 132 */ /* 133 */ } 20/01/23 18:54:03.296 Executor task launch worker for task 32 DEBUG ExecutorMetricsPoller: stageTCMP: (6, 0) -> 1 20/01/23 18:54:03.296 Executor task launch worker for task 31 DEBUG BlockManager: Level for block rdd_24_8 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:03.299 Executor task launch worker for task 31 INFO MemoryStore: Block rdd_25_8 stored as values in memory (estimated size 16.0 B, free 366.1 MiB) 20/01/23 18:54:03.300 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_25_8 in memory on 192.168.1.225:35863 (size: 16.0 B, free: 366.2 MiB) 20/01/23 18:54:03.302 Executor task launch worker for task 31 DEBUG BlockManagerMaster: Updated info of block rdd_25_8 20/01/23 18:54:03.302 Executor task launch worker for task 31 DEBUG BlockManager: Told master about block rdd_25_8 20/01/23 18:54:03.302 Executor task launch worker for task 31 DEBUG BlockManager: Put block rdd_25_8 locally took 12 ms 20/01/23 18:54:03.302 Executor task launch worker for task 31 DEBUG BlockManager: Putting block rdd_25_8 without replication took 12 ms 20/01/23 18:54:03.302 Executor task launch worker for task 31 DEBUG BlockManager: Getting local block rdd_25_8 20/01/23 18:54:03.302 Executor task launch worker for task 31 DEBUG BlockManager: Level for block rdd_25_8 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:03.303 Executor task launch worker for task 31 INFO Executor: Finished task 8.0 in stage 6.0 (TID 31). 2222 bytes result sent to driver 20/01/23 18:54:03.303 Executor task launch worker for task 31 DEBUG ExecutorMetricsPoller: removing (6, 0) from stageTCMP 20/01/23 18:54:03.304 dispatcher-event-loop-3 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_6.0, runningTasks: 0 20/01/23 18:54:03.305 task-result-getter-3 INFO TaskSetManager: Finished task 8.0 in stage 6.0 (TID 31) in 38 ms on 192.168.1.225 (executor driver) (10/10) 20/01/23 18:54:03.305 task-result-getter-3 INFO TaskSchedulerImpl: Removed TaskSet 6.0, whose tasks have all completed, from pool 20/01/23 18:54:03.306 dag-scheduler-event-loop INFO DAGScheduler: ResultStage 6 (count at ALS.scala:951) finished in 0.130 s 20/01/23 18:54:03.306 dag-scheduler-event-loop DEBUG DAGScheduler: After removal of stage 5, remaining stages = 2 20/01/23 18:54:03.306 dag-scheduler-event-loop DEBUG DAGScheduler: After removal of stage 4, remaining stages = 1 20/01/23 18:54:03.306 dag-scheduler-event-loop DEBUG DAGScheduler: After removal of stage 6, remaining stages = 0 20/01/23 18:54:03.306 dag-scheduler-event-loop INFO DAGScheduler: Job 2 is finished. Cancelling potential speculative or zombie tasks for this job 20/01/23 18:54:03.307 dag-scheduler-event-loop INFO TaskSchedulerImpl: Killing all running tasks in stage 6: Stage finished 20/01/23 18:54:03.307 Thread-4 INFO DAGScheduler: Job 2 finished: count at ALS.scala:951, took 0.224170 s 20/01/23 18:54:03.310 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$initialize$1 20/01/23 18:54:03.312 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$initialize$1) is now cleaned +++ 20/01/23 18:54:03.313 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$initialize$1 20/01/23 18:54:03.315 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$initialize$1) is now cleaned +++ 20/01/23 18:54:03.331 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$cogroup$4 20/01/23 18:54:03.334 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$cogroup$4) is now cleaned +++ 20/01/23 18:54:03.337 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$join$2 20/01/23 18:54:03.319 Executor task launch worker for task 8 INFO CodeGenerator: Code generated in 110.389978 ms 20/01/23 18:54:03.340 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$join$2) is now cleaned +++ 20/01/23 18:54:03.345 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$computeFactors$1 20/01/23 18:54:03.347 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$computeFactors$1) is now cleaned +++ 20/01/23 18:54:03.349 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$groupByKey$2 20/01/23 18:54:03.352 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$groupByKey$2) is now cleaned +++ 20/01/23 18:54:03.357 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$groupByKey$3 20/01/23 18:54:03.360 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$groupByKey$3) is now cleaned +++ 20/01/23 18:54:03.360 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$groupByKey$4 20/01/23 18:54:03.367 Executor task launch worker for task 11 INFO PythonRunner: Times: total = 61, boot = -1136, init = 1197, finish = 0 20/01/23 18:54:03.372 Executor task launch worker for task 8 INFO PythonRunner: Times: total = 46, boot = -1100, init = 1146, finish = 0 20/01/23 18:54:03.373 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$groupByKey$4) is now cleaned +++ 20/01/23 18:54:03.374 Executor task launch worker for task 9 INFO PythonRunner: Times: total = 129, boot = -1078, init = 1207, finish = 0 20/01/23 18:54:03.379 Executor task launch worker for task 10 INFO PythonRunner: Times: total = 85, boot = -1069, init = 1154, finish = 0 20/01/23 18:54:03.380 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$cogroup$4 20/01/23 18:54:03.383 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$cogroup$4) is now cleaned +++ 20/01/23 18:54:03.384 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$join$2 20/01/23 18:54:03.401 Thread-4 INFO CodeGenerator: Code generated in 111.064852 ms 20/01/23 18:54:03.415 Executor task launch worker for task 9 DEBUG TaskMemoryManager: Task 9 release 0.0 B from org.apache.spark.util.collection.ExternalSorter@1aecd732 20/01/23 18:54:03.415 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$join$2) is now cleaned +++ 20/01/23 18:54:03.419 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$computeFactors$4 20/01/23 18:54:03.416 Executor task launch worker for task 8 DEBUG TaskMemoryManager: Task 8 release 0.0 B from org.apache.spark.util.collection.ExternalSorter@2d8e41b2 20/01/23 18:54:03.427 Executor task launch worker for task 9 INFO Executor: Finished task 1.0 in stage 2.0 (TID 9). 2125 bytes result sent to driver 20/01/23 18:54:03.427 Executor task launch worker for task 9 DEBUG ExecutorMetricsPoller: stageTCMP: (2, 0) -> 3 20/01/23 18:54:03.427 Executor task launch worker for task 8 INFO Executor: Finished task 0.0 in stage 2.0 (TID 8). 2082 bytes result sent to driver 20/01/23 18:54:03.427 Executor task launch worker for task 8 DEBUG ExecutorMetricsPoller: stageTCMP: (2, 0) -> 2 20/01/23 18:54:03.427 dispatcher-event-loop-2 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_2.0, runningTasks: 3 20/01/23 18:54:03.428 dispatcher-event-loop-2 DEBUG TaskSetManager: No tasks for locality level NO_PREF, so moving to locality level ANY 20/01/23 18:54:03.428 dispatcher-event-loop-2 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_2.0, runningTasks: 2 20/01/23 18:54:03.429 task-result-getter-1 INFO TaskSetManager: Finished task 1.0 in stage 2.0 (TID 9) in 598 ms on 192.168.1.225 (executor driver) (1/4) 20/01/23 18:54:03.430 task-result-getter-1 INFO TaskSetManager: Finished task 0.0 in stage 2.0 (TID 8) in 601 ms on 192.168.1.225 (executor driver) (2/4) 20/01/23 18:54:03.431 dag-scheduler-event-loop DEBUG DAGScheduler: ShuffleMapTask finished on driver 20/01/23 18:54:03.433 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$computeFactors$4) is now cleaned +++ 20/01/23 18:54:03.433 dag-scheduler-event-loop DEBUG DAGScheduler: ShuffleMapTask finished on driver 20/01/23 18:54:03.437 Executor task launch worker for task 10 DEBUG TaskMemoryManager: Task 10 release 0.0 B from org.apache.spark.util.collection.ExternalSorter@6d66527a 20/01/23 18:54:03.439 Executor task launch worker for task 10 INFO Executor: Finished task 2.0 in stage 2.0 (TID 10). 2082 bytes result sent to driver 20/01/23 18:54:03.439 Executor task launch worker for task 10 DEBUG ExecutorMetricsPoller: stageTCMP: (2, 0) -> 1 20/01/23 18:54:03.439 dispatcher-event-loop-3 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_2.0, runningTasks: 1 20/01/23 18:54:03.440 task-result-getter-2 INFO TaskSetManager: Finished task 2.0 in stage 2.0 (TID 10) in 609 ms on 192.168.1.225 (executor driver) (3/4) 20/01/23 18:54:03.440 Executor task launch worker for task 11 DEBUG TaskMemoryManager: Task 11 release 0.0 B from org.apache.spark.util.collection.ExternalSorter@2536a90c 20/01/23 18:54:03.442 Thread-4 DEBUG CoGroupedRDD: Adding one-to-one dependency with itemInBlocks MapPartitionsRDD[24] at mapValues at ALS.scala:1613 20/01/23 18:54:03.442 Thread-4 DEBUG CoGroupedRDD: Adding one-to-one dependency with ShuffledRDD[32] at groupByKey at ALS.scala:1680 20/01/23 18:54:03.445 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$cogroup$4 20/01/23 18:54:03.445 Executor task launch worker for task 11 INFO Executor: Finished task 3.0 in stage 2.0 (TID 11). 2082 bytes result sent to driver 20/01/23 18:54:03.447 dag-scheduler-event-loop DEBUG DAGScheduler: ShuffleMapTask finished on driver 20/01/23 18:54:03.447 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$cogroup$4) is now cleaned +++ 20/01/23 18:54:03.447 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$join$2 20/01/23 18:54:03.448 dispatcher-event-loop-1 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_2.0, runningTasks: 0 20/01/23 18:54:03.449 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$join$2) is now cleaned +++ 20/01/23 18:54:03.449 Executor task launch worker for task 11 DEBUG ExecutorMetricsPoller: removing (2, 0) from stageTCMP 20/01/23 18:54:03.449 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$computeFactors$1 20/01/23 18:54:03.450 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$computeFactors$1) is now cleaned +++ 20/01/23 18:54:03.451 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$groupByKey$2 20/01/23 18:54:03.452 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$groupByKey$2) is now cleaned +++ 20/01/23 18:54:03.452 task-result-getter-3 INFO TaskSetManager: Finished task 3.0 in stage 2.0 (TID 11) in 621 ms on 192.168.1.225 (executor driver) (4/4) 20/01/23 18:54:03.452 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$groupByKey$3 20/01/23 18:54:03.452 task-result-getter-3 INFO TaskSchedulerImpl: Removed TaskSet 2.0, whose tasks have all completed, from pool 20/01/23 18:54:03.453 dag-scheduler-event-loop DEBUG DAGScheduler: ShuffleMapTask finished on driver 20/01/23 18:54:03.453 dag-scheduler-event-loop INFO DAGScheduler: ShuffleMapStage 2 (flatMap at CountVectorizer.scala:212) finished in 0.781 s 20/01/23 18:54:03.454 dag-scheduler-event-loop INFO DAGScheduler: looking for newly runnable stages 20/01/23 18:54:03.455 dag-scheduler-event-loop INFO DAGScheduler: running: Set() 20/01/23 18:54:03.455 dag-scheduler-event-loop INFO DAGScheduler: waiting: Set(ResultStage 3) 20/01/23 18:54:03.455 dag-scheduler-event-loop INFO DAGScheduler: failed: Set() 20/01/23 18:54:03.456 dag-scheduler-event-loop DEBUG MapOutputTrackerMaster: Increasing epoch to 1 20/01/23 18:54:03.459 dag-scheduler-event-loop DEBUG DAGScheduler: submitStage(ResultStage 3 (name=count at CountVectorizer.scala:233;jobs=2)) 20/01/23 18:54:03.459 dag-scheduler-event-loop DEBUG DAGScheduler: missing: List() 20/01/23 18:54:03.459 dag-scheduler-event-loop INFO DAGScheduler: Submitting ResultStage 3 (MapPartitionsRDD[21] at map at CountVectorizer.scala:230), which has no missing parents 20/01/23 18:54:03.459 dag-scheduler-event-loop DEBUG DAGScheduler: submitMissingTasks(ResultStage 3) 20/01/23 18:54:03.460 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$groupByKey$3) is now cleaned +++ 20/01/23 18:54:03.460 Thread-4 DEBUG WholeStageCodegenExec: /* 001 */ public Object generate(Object[] references) { /* 002 */ return new GeneratedIteratorForCodegenStage1(references); /* 003 */ } /* 004 */ /* 005 */ // codegenStageId=1 /* 006 */ final class GeneratedIteratorForCodegenStage1 extends org.apache.spark.sql.execution.BufferedRowIterator { /* 007 */ private Object[] references; /* 008 */ private scala.collection.Iterator[] inputs; /* 009 */ private boolean agg_initAgg_0; /* 010 */ private boolean agg_bufIsNull_0; /* 011 */ private double agg_bufValue_0; /* 012 */ private boolean agg_bufIsNull_1; /* 013 */ private long agg_bufValue_1; /* 014 */ private scala.collection.Iterator rdd_input_0; /* 015 */ private boolean agg_agg_isNull_14_0; /* 016 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[] rdd_mutableStateArray_0 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[7]; /* 017 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[] rdd_mutableStateArray_1 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[6]; /* 018 */ /* 019 */ public GeneratedIteratorForCodegenStage1(Object[] references) { /* 020 */ this.references = references; /* 021 */ } /* 022 */ /* 023 */ public void init(int index, scala.collection.Iterator[] inputs) { /* 024 */ partitionIndex = index; /* 025 */ this.inputs = inputs; /* 026 */ wholestagecodegen_init_0_0(); /* 027 */ wholestagecodegen_init_0_1(); /* 028 */ /* 029 */ } /* 030 */ /* 031 */ private void agg_doAggregateWithoutKey_0() throws java.io.IOException { /* 032 */ // initialize aggregation buffer /* 033 */ agg_bufIsNull_0 = false; /* 034 */ agg_bufValue_0 = 0.0D; /* 035 */ agg_bufIsNull_1 = false; /* 036 */ agg_bufValue_1 = 0L; /* 037 */ /* 038 */ while ( rdd_input_0.hasNext()) { /* 039 */ InternalRow rdd_row_0 = (InternalRow) rdd_input_0.next(); /* 040 */ ((org.apache.spark.sql.execution.metric.SQLMetric) references[0] /* numOutputRows */).add(1); /* 041 */ boolean rdd_isNull_1 = rdd_row_0.isNullAt(1); /* 042 */ double rdd_value_1 = rdd_isNull_1 ? /* 043 */ -1.0 : (rdd_row_0.getDouble(1)); /* 044 */ boolean rdd_isNull_0 = rdd_row_0.isNullAt(0); /* 045 */ InternalRow rdd_value_0 = rdd_isNull_0 ? /* 046 */ null : (rdd_row_0.getStruct(0, 4)); /* 047 */ Object project_arg_0 = rdd_isNull_0 ? null : ((scala.Function1[]) references[1] /* converters */)[0].apply(rdd_value_0); /* 048 */ /* 049 */ InternalRow project_result_0 = null; /* 050 */ try { /* 051 */ project_result_0 = (InternalRow)((scala.Function1[]) references[1] /* converters */)[1].apply(((scala.Function1) references[3] /* udf */).apply(project_arg_0)); /* 052 */ } catch (Exception e) { /* 053 */ throw new org.apache.spark.SparkException(((java.lang.String) references[2] /* errMsg */), e); /* 054 */ } /* 055 */ /* 056 */ boolean project_isNull_1 = project_result_0 == null; /* 057 */ InternalRow project_value_1 = null; /* 058 */ if (!project_isNull_1) { /* 059 */ project_value_1 = project_result_0; /* 060 */ } /* 061 */ /* 062 */ agg_doConsume_0(rdd_value_1, rdd_isNull_1, project_value_1, project_isNull_1); /* 063 */ // shouldStop check is eliminated /* 064 */ } /* 065 */ /* 066 */ } /* 067 */ /* 068 */ private void project_writeFields_0_1(InternalRow project_tmpInput_0) { /* 069 */ if ((project_tmpInput_0.isNullAt(3))) { /* 070 */ rdd_mutableStateArray_0[3].setNullAt(3); /* 071 */ } else { /* 072 */ // Remember the current cursor so that we can calculate how many bytes are /* 073 */ // written later. /* 074 */ final int project_previousCursor_2 = rdd_mutableStateArray_0[3].cursor(); /* 075 */ /* 076 */ final ArrayData project_tmpInput_2 = (project_tmpInput_0.getArray(3)); /* 077 */ if (project_tmpInput_2 instanceof UnsafeArrayData) { /* 078 */ rdd_mutableStateArray_0[3].write((UnsafeArrayData) project_tmpInput_2); /* 079 */ } else { /* 080 */ final int project_numElements_1 = project_tmpInput_2.numElements(); /* 081 */ rdd_mutableStateArray_1[3].initialize(project_numElements_1); /* 082 */ /* 083 */ for (int project_index_1 = 0; project_index_1 < project_numElements_1; project_index_1++) { /* 084 */ rdd_mutableStateArray_1[3].write(project_index_1, project_tmpInput_2.getDouble(project_index_1)); /* 085 */ } /* 086 */ } /* 087 */ /* 088 */ rdd_mutableStateArray_0[3].setOffsetAndSizeFromPreviousCursor(3, project_previousCursor_2); /* 089 */ } /* 090 */ /* 091 */ } /* 092 */ /* 093 */ private void wholestagecodegen_init_0_1() { /* 094 */ rdd_mutableStateArray_0[4] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(2, 32); /* 095 */ rdd_mutableStateArray_0[5] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(rdd_mutableStateArray_0[4], 4); /* 096 */ rdd_mutableStateArray_1[4] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[5], 4); /* 097 */ rdd_mutableStateArray_1[5] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[5], 8); /* 098 */ rdd_mutableStateArray_0[6] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(2, 0); /* 099 */ /* 100 */ } /* 101 */ /* 102 */ private void agg_doConsume_0(double agg_expr_0_0, boolean agg_exprIsNull_0_0, InternalRow agg_expr_1_0, boolean agg_exprIsNull_1_0) throws java.io.IOException { /* 103 */ // do aggregate /* 104 */ // common sub-expressions /* 105 */ boolean agg_isNull_4 = false; /* 106 */ double agg_value_4 = -1.0; /* 107 */ if (!false && agg_exprIsNull_0_0) { /* 108 */ agg_isNull_4 = true; /* 109 */ agg_value_4 = -1.0; /* 110 */ } else { /* 111 */ Object agg_arg_0 = agg_exprIsNull_1_0 ? null : ((scala.Function1[]) references[4] /* converters */)[0].apply(agg_expr_1_0); /* 112 */ /* 113 */ Double agg_conv_0 = agg_expr_0_0; /* 114 */ Object agg_arg_1 = false ? null : agg_conv_0; /* 115 */ /* 116 */ Double agg_result_0 = null; /* 117 */ try { /* 118 */ agg_result_0 = (Double)((scala.Function2) references[6] /* udf */).apply(agg_arg_0, agg_arg_1); /* 119 */ } catch (Exception e) { /* 120 */ throw new org.apache.spark.SparkException(((java.lang.String) references[5] /* errMsg */), e); /* 121 */ } /* 122 */ /* 123 */ boolean agg_isNull_8 = agg_result_0 == null; /* 124 */ double agg_value_8 = -1.0; /* 125 */ if (!agg_isNull_8) { /* 126 */ agg_value_8 = agg_result_0; /* 127 */ } /* 128 */ agg_isNull_4 = agg_isNull_8; /* 129 */ agg_value_4 = agg_value_8; /* 130 */ } /* 131 */ // evaluate aggregate functions and update aggregation buffers /* 132 */ /* 133 */ // do aggregate for avg /* 134 */ // evaluate aggregate function /* 135 */ boolean agg_isNull_12 = true; /* 136 */ double agg_value_12 = -1.0; /* 137 */ /* 138 */ if (!agg_bufIsNull_0) { /* 139 */ agg_agg_isNull_14_0 = true; /* 140 */ double agg_value_14 = -1.0; /* 141 */ do { /* 142 */ if (!agg_isNull_4) { /* 143 */ agg_agg_isNull_14_0 = false; /* 144 */ agg_value_14 = agg_value_4; /* 145 */ continue; /* 146 */ } /* 147 */ /* 148 */ if (!false) { /* 149 */ agg_agg_isNull_14_0 = false; /* 150 */ agg_value_14 = 0.0D; /* 151 */ continue; /* 152 */ } /* 153 */ /* 154 */ } while (false); /* 155 */ /* 156 */ agg_isNull_12 = false; // resultCode could change nullability. /* 157 */ /* 158 */ agg_value_12 = agg_bufValue_0 + agg_value_14; /* 159 */ /* 160 */ } /* 161 */ boolean agg_isNull_16 = false; /* 162 */ long agg_value_16 = -1L; /* 163 */ if (!false && agg_isNull_4) { /* 164 */ agg_isNull_16 = agg_bufIsNull_1; /* 165 */ agg_value_16 = agg_bufValue_1; /* 166 */ } else { /* 167 */ boolean agg_isNull_19 = true; /* 168 */ long agg_value_19 = -1L; /* 169 */ /* 170 */ if (!agg_bufIsNull_1) { /* 171 */ agg_isNull_19 = false; // resultCode could change nullability. /* 172 */ /* 173 */ agg_value_19 = agg_bufValue_1 + 1L; /* 174 */ /* 175 */ } /* 176 */ agg_isNull_16 = agg_isNull_19; /* 177 */ agg_value_16 = agg_value_19; /* 178 */ } /* 179 */ // update aggregation buffers /* 180 */ agg_bufIsNull_0 = agg_isNull_12; /* 181 */ agg_bufValue_0 = agg_value_12; /* 182 */ /* 183 */ agg_bufIsNull_1 = agg_isNull_16; /* 184 */ agg_bufValue_1 = agg_value_16; /* 185 */ /* 186 */ } /* 187 */ /* 188 */ private void project_writeFields_0_0(InternalRow project_tmpInput_0) { /* 189 */ rdd_mutableStateArray_0[3].write(0, (project_tmpInput_0.getByte(0))); /* 190 */ /* 191 */ if ((project_tmpInput_0.isNullAt(1))) { /* 192 */ rdd_mutableStateArray_0[3].setNullAt(1); /* 193 */ } else { /* 194 */ rdd_mutableStateArray_0[3].write(1, (project_tmpInput_0.getInt(1))); /* 195 */ } /* 196 */ /* 197 */ if ((project_tmpInput_0.isNullAt(2))) { /* 198 */ rdd_mutableStateArray_0[3].setNullAt(2); /* 199 */ } else { /* 200 */ // Remember the current cursor so that we can calculate how many bytes are /* 201 */ // written later. /* 202 */ final int project_previousCursor_1 = rdd_mutableStateArray_0[3].cursor(); /* 203 */ /* 204 */ final ArrayData project_tmpInput_1 = (project_tmpInput_0.getArray(2)); /* 205 */ if (project_tmpInput_1 instanceof UnsafeArrayData) { /* 206 */ rdd_mutableStateArray_0[3].write((UnsafeArrayData) project_tmpInput_1); /* 207 */ } else { /* 208 */ final int project_numElements_0 = project_tmpInput_1.numElements(); /* 209 */ rdd_mutableStateArray_1[2].initialize(project_numElements_0); /* 210 */ /* 211 */ for (int project_index_0 = 0; project_index_0 < project_numElements_0; project_index_0++) { /* 212 */ rdd_mutableStateArray_1[2].write(project_index_0, project_tmpInput_1.getInt(project_index_0)); /* 213 */ } /* 214 */ } /* 215 */ /* 216 */ rdd_mutableStateArray_0[3].setOffsetAndSizeFromPreviousCursor(2, project_previousCursor_1); /* 217 */ } /* 218 */ /* 219 */ } /* 220 */ /* 221 */ protected void processNext() throws java.io.IOException { /* 222 */ while (!agg_initAgg_0) { /* 223 */ agg_initAgg_0 = true; /* 224 */ long agg_beforeAgg_0 = System.nanoTime(); /* 225 */ agg_doAggregateWithoutKey_0(); /* 226 */ ((org.apache.spark.sql.execution.metric.SQLMetric) references[8] /* aggTime */).add((System.nanoTime() - agg_beforeAgg_0) / 1000000); /* 227 */ /* 228 */ // output the result /* 229 */ /* 230 */ ((org.apache.spark.sql.execution.metric.SQLMetric) references[7] /* numOutputRows */).add(1); /* 231 */ rdd_mutableStateArray_0[6].reset(); /* 232 */ /* 233 */ rdd_mutableStateArray_0[6].zeroOutNullBytes(); /* 234 */ /* 235 */ if (agg_bufIsNull_0) { /* 236 */ rdd_mutableStateArray_0[6].setNullAt(0); /* 237 */ } else { /* 238 */ rdd_mutableStateArray_0[6].write(0, agg_bufValue_0); /* 239 */ } /* 240 */ /* 241 */ if (agg_bufIsNull_1) { /* 242 */ rdd_mutableStateArray_0[6].setNullAt(1); /* 243 */ } else { /* 244 */ rdd_mutableStateArray_0[6].write(1, agg_bufValue_1); /* 245 */ } /* 246 */ append((rdd_mutableStateArray_0[6].getRow())); /* 247 */ } /* 248 */ } /* 249 */ /* 250 */ private void wholestagecodegen_init_0_0() { /* 251 */ rdd_input_0 = inputs[0]; /* 252 */ rdd_mutableStateArray_0[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(2, 32); /* 253 */ rdd_mutableStateArray_0[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(rdd_mutableStateArray_0[0], 4); /* 254 */ rdd_mutableStateArray_1[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[1], 4); /* 255 */ rdd_mutableStateArray_1[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[1], 8); /* 256 */ rdd_mutableStateArray_0[2] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(2, 32); /* 257 */ rdd_mutableStateArray_0[3] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(rdd_mutableStateArray_0[2], 4); /* 258 */ rdd_mutableStateArray_1[2] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[3], 4); /* 259 */ rdd_mutableStateArray_1[3] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[3], 8); /* 260 */ /* 261 */ } /* 262 */ /* 263 */ } 20/01/23 18:54:03.461 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$groupByKey$4 20/01/23 18:54:03.464 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$groupByKey$4) is now cleaned +++ 20/01/23 18:54:03.465 Thread-4 DEBUG CodeGenerator: /* 001 */ public Object generate(Object[] references) { /* 002 */ return new GeneratedIteratorForCodegenStage1(references); /* 003 */ } /* 004 */ /* 005 */ // codegenStageId=1 /* 006 */ final class GeneratedIteratorForCodegenStage1 extends org.apache.spark.sql.execution.BufferedRowIterator { /* 007 */ private Object[] references; /* 008 */ private scala.collection.Iterator[] inputs; /* 009 */ private boolean agg_initAgg_0; /* 010 */ private boolean agg_bufIsNull_0; /* 011 */ private double agg_bufValue_0; /* 012 */ private boolean agg_bufIsNull_1; /* 013 */ private long agg_bufValue_1; /* 014 */ private scala.collection.Iterator rdd_input_0; /* 015 */ private boolean agg_agg_isNull_14_0; /* 016 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[] rdd_mutableStateArray_0 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[7]; /* 017 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[] rdd_mutableStateArray_1 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[6]; /* 018 */ /* 019 */ public GeneratedIteratorForCodegenStage1(Object[] references) { /* 020 */ this.references = references; /* 021 */ } /* 022 */ /* 023 */ public void init(int index, scala.collection.Iterator[] inputs) { /* 024 */ partitionIndex = index; /* 025 */ this.inputs = inputs; /* 026 */ wholestagecodegen_init_0_0(); /* 027 */ wholestagecodegen_init_0_1(); /* 028 */ /* 029 */ } /* 030 */ /* 031 */ private void agg_doAggregateWithoutKey_0() throws java.io.IOException { /* 032 */ // initialize aggregation buffer /* 033 */ agg_bufIsNull_0 = false; /* 034 */ agg_bufValue_0 = 0.0D; /* 035 */ agg_bufIsNull_1 = false; /* 036 */ agg_bufValue_1 = 0L; /* 037 */ /* 038 */ while ( rdd_input_0.hasNext()) { /* 039 */ InternalRow rdd_row_0 = (InternalRow) rdd_input_0.next(); /* 040 */ ((org.apache.spark.sql.execution.metric.SQLMetric) references[0] /* numOutputRows */).add(1); /* 041 */ boolean rdd_isNull_1 = rdd_row_0.isNullAt(1); /* 042 */ double rdd_value_1 = rdd_isNull_1 ? /* 043 */ -1.0 : (rdd_row_0.getDouble(1)); /* 044 */ boolean rdd_isNull_0 = rdd_row_0.isNullAt(0); /* 045 */ InternalRow rdd_value_0 = rdd_isNull_0 ? /* 046 */ null : (rdd_row_0.getStruct(0, 4)); /* 047 */ Object project_arg_0 = rdd_isNull_0 ? null : ((scala.Function1[]) references[1] /* converters */)[0].apply(rdd_value_0); /* 048 */ /* 049 */ InternalRow project_result_0 = null; /* 050 */ try { /* 051 */ project_result_0 = (InternalRow)((scala.Function1[]) references[1] /* converters */)[1].apply(((scala.Function1) references[3] /* udf */).apply(project_arg_0)); /* 052 */ } catch (Exception e) { /* 053 */ throw new org.apache.spark.SparkException(((java.lang.String) references[2] /* errMsg */), e); /* 054 */ } /* 055 */ /* 056 */ boolean project_isNull_1 = project_result_0 == null; /* 057 */ InternalRow project_value_1 = null; /* 058 */ if (!project_isNull_1) { /* 059 */ project_value_1 = project_result_0; /* 060 */ } /* 061 */ /* 062 */ agg_doConsume_0(rdd_value_1, rdd_isNull_1, project_value_1, project_isNull_1); /* 063 */ // shouldStop check is eliminated /* 064 */ } /* 065 */ /* 066 */ } /* 067 */ /* 068 */ private void project_writeFields_0_1(InternalRow project_tmpInput_0) { /* 069 */ if ((project_tmpInput_0.isNullAt(3))) { /* 070 */ rdd_mutableStateArray_0[3].setNullAt(3); /* 071 */ } else { /* 072 */ // Remember the current cursor so that we can calculate how many bytes are /* 073 */ // written later. /* 074 */ final int project_previousCursor_2 = rdd_mutableStateArray_0[3].cursor(); /* 075 */ /* 076 */ final ArrayData project_tmpInput_2 = (project_tmpInput_0.getArray(3)); /* 077 */ if (project_tmpInput_2 instanceof UnsafeArrayData) { /* 078 */ rdd_mutableStateArray_0[3].write((UnsafeArrayData) project_tmpInput_2); /* 079 */ } else { /* 080 */ final int project_numElements_1 = project_tmpInput_2.numElements(); /* 081 */ rdd_mutableStateArray_1[3].initialize(project_numElements_1); /* 082 */ /* 083 */ for (int project_index_1 = 0; project_index_1 < project_numElements_1; project_index_1++) { /* 084 */ rdd_mutableStateArray_1[3].write(project_index_1, project_tmpInput_2.getDouble(project_index_1)); /* 085 */ } /* 086 */ } /* 087 */ /* 088 */ rdd_mutableStateArray_0[3].setOffsetAndSizeFromPreviousCursor(3, project_previousCursor_2); /* 089 */ } /* 090 */ /* 091 */ } /* 092 */ /* 093 */ private void wholestagecodegen_init_0_1() { /* 094 */ rdd_mutableStateArray_0[4] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(2, 32); /* 095 */ rdd_mutableStateArray_0[5] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(rdd_mutableStateArray_0[4], 4); /* 096 */ rdd_mutableStateArray_1[4] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[5], 4); /* 097 */ rdd_mutableStateArray_1[5] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[5], 8); /* 098 */ rdd_mutableStateArray_0[6] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(2, 0); /* 099 */ /* 100 */ } /* 101 */ /* 102 */ private void agg_doConsume_0(double agg_expr_0_0, boolean agg_exprIsNull_0_0, InternalRow agg_expr_1_0, boolean agg_exprIsNull_1_0) throws java.io.IOException { /* 103 */ // do aggregate /* 104 */ // common sub-expressions /* 105 */ boolean agg_isNull_4 = false; /* 106 */ double agg_value_4 = -1.0; /* 107 */ if (!false && agg_exprIsNull_0_0) { /* 108 */ agg_isNull_4 = true; /* 109 */ agg_value_4 = -1.0; /* 110 */ } else { /* 111 */ Object agg_arg_0 = agg_exprIsNull_1_0 ? null : ((scala.Function1[]) references[4] /* converters */)[0].apply(agg_expr_1_0); /* 112 */ /* 113 */ Double agg_conv_0 = agg_expr_0_0; /* 114 */ Object agg_arg_1 = false ? null : agg_conv_0; /* 115 */ /* 116 */ Double agg_result_0 = null; /* 117 */ try { /* 118 */ agg_result_0 = (Double)((scala.Function2) references[6] /* udf */).apply(agg_arg_0, agg_arg_1); /* 119 */ } catch (Exception e) { /* 120 */ throw new org.apache.spark.SparkException(((java.lang.String) references[5] /* errMsg */), e); /* 121 */ } /* 122 */ /* 123 */ boolean agg_isNull_8 = agg_result_0 == null; /* 124 */ double agg_value_8 = -1.0; /* 125 */ if (!agg_isNull_8) { /* 126 */ agg_value_8 = agg_result_0; /* 127 */ } /* 128 */ agg_isNull_4 = agg_isNull_8; /* 129 */ agg_value_4 = agg_value_8; /* 130 */ } /* 131 */ // evaluate aggregate functions and update aggregation buffers /* 132 */ /* 133 */ // do aggregate for avg /* 134 */ // evaluate aggregate function /* 135 */ boolean agg_isNull_12 = true; /* 136 */ double agg_value_12 = -1.0; /* 137 */ /* 138 */ if (!agg_bufIsNull_0) { /* 139 */ agg_agg_isNull_14_0 = true; /* 140 */ double agg_value_14 = -1.0; /* 141 */ do { /* 142 */ if (!agg_isNull_4) { /* 143 */ agg_agg_isNull_14_0 = false; /* 144 */ agg_value_14 = agg_value_4; /* 145 */ continue; /* 146 */ } /* 147 */ /* 148 */ if (!false) { /* 149 */ agg_agg_isNull_14_0 = false; /* 150 */ agg_value_14 = 0.0D; /* 151 */ continue; /* 152 */ } /* 153 */ /* 154 */ } while (false); /* 155 */ /* 156 */ agg_isNull_12 = false; // resultCode could change nullability. /* 157 */ /* 158 */ agg_value_12 = agg_bufValue_0 + agg_value_14; /* 159 */ /* 160 */ } /* 161 */ boolean agg_isNull_16 = false; /* 162 */ long agg_value_16 = -1L; /* 163 */ if (!false && agg_isNull_4) { /* 164 */ agg_isNull_16 = agg_bufIsNull_1; /* 165 */ agg_value_16 = agg_bufValue_1; /* 166 */ } else { /* 167 */ boolean agg_isNull_19 = true; /* 168 */ long agg_value_19 = -1L; /* 169 */ /* 170 */ if (!agg_bufIsNull_1) { /* 171 */ agg_isNull_19 = false; // resultCode could change nullability. /* 172 */ /* 173 */ agg_value_19 = agg_bufValue_1 + 1L; /* 174 */ /* 175 */ } /* 176 */ agg_isNull_16 = agg_isNull_19; /* 177 */ agg_value_16 = agg_value_19; /* 178 */ } /* 179 */ // update aggregation buffers /* 180 */ agg_bufIsNull_0 = agg_isNull_12; /* 181 */ agg_bufValue_0 = agg_value_12; /* 182 */ /* 183 */ agg_bufIsNull_1 = agg_isNull_16; /* 184 */ agg_bufValue_1 = agg_value_16; /* 185 */ /* 186 */ } /* 187 */ /* 188 */ private void project_writeFields_0_0(InternalRow project_tmpInput_0) { /* 189 */ rdd_mutableStateArray_0[3].write(0, (project_tmpInput_0.getByte(0))); /* 190 */ /* 191 */ if ((project_tmpInput_0.isNullAt(1))) { /* 192 */ rdd_mutableStateArray_0[3].setNullAt(1); /* 193 */ } else { /* 194 */ rdd_mutableStateArray_0[3].write(1, (project_tmpInput_0.getInt(1))); /* 195 */ } /* 196 */ /* 197 */ if ((project_tmpInput_0.isNullAt(2))) { /* 198 */ rdd_mutableStateArray_0[3].setNullAt(2); /* 199 */ } else { /* 200 */ // Remember the current cursor so that we can calculate how many bytes are /* 201 */ // written later. /* 202 */ final int project_previousCursor_1 = rdd_mutableStateArray_0[3].cursor(); /* 203 */ /* 204 */ final ArrayData project_tmpInput_1 = (project_tmpInput_0.getArray(2)); /* 205 */ if (project_tmpInput_1 instanceof UnsafeArrayData) { /* 206 */ rdd_mutableStateArray_0[3].write((UnsafeArrayData) project_tmpInput_1); /* 207 */ } else { /* 208 */ final int project_numElements_0 = project_tmpInput_1.numElements(); /* 209 */ rdd_mutableStateArray_1[2].initialize(project_numElements_0); /* 210 */ /* 211 */ for (int project_index_0 = 0; project_index_0 < project_numElements_0; project_index_0++) { /* 212 */ rdd_mutableStateArray_1[2].write(project_index_0, project_tmpInput_1.getInt(project_index_0)); /* 213 */ } /* 214 */ } /* 215 */ /* 216 */ rdd_mutableStateArray_0[3].setOffsetAndSizeFromPreviousCursor(2, project_previousCursor_1); /* 217 */ } /* 218 */ /* 219 */ } /* 220 */ /* 221 */ protected void processNext() throws java.io.IOException { /* 222 */ while (!agg_initAgg_0) { /* 223 */ agg_initAgg_0 = true; /* 224 */ long agg_beforeAgg_0 = System.nanoTime(); /* 225 */ agg_doAggregateWithoutKey_0(); /* 226 */ ((org.apache.spark.sql.execution.metric.SQLMetric) references[8] /* aggTime */).add((System.nanoTime() - agg_beforeAgg_0) / 1000000); /* 227 */ /* 228 */ // output the result /* 229 */ /* 230 */ ((org.apache.spark.sql.execution.metric.SQLMetric) references[7] /* numOutputRows */).add(1); /* 231 */ rdd_mutableStateArray_0[6].reset(); /* 232 */ /* 233 */ rdd_mutableStateArray_0[6].zeroOutNullBytes(); /* 234 */ /* 235 */ if (agg_bufIsNull_0) { /* 236 */ rdd_mutableStateArray_0[6].setNullAt(0); /* 237 */ } else { /* 238 */ rdd_mutableStateArray_0[6].write(0, agg_bufValue_0); /* 239 */ } /* 240 */ /* 241 */ if (agg_bufIsNull_1) { /* 242 */ rdd_mutableStateArray_0[6].setNullAt(1); /* 243 */ } else { /* 244 */ rdd_mutableStateArray_0[6].write(1, agg_bufValue_1); /* 245 */ } /* 246 */ append((rdd_mutableStateArray_0[6].getRow())); /* 247 */ } /* 248 */ } /* 249 */ /* 250 */ private void wholestagecodegen_init_0_0() { /* 251 */ rdd_input_0 = inputs[0]; /* 252 */ rdd_mutableStateArray_0[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(2, 32); /* 253 */ rdd_mutableStateArray_0[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(rdd_mutableStateArray_0[0], 4); /* 254 */ rdd_mutableStateArray_1[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[1], 4); /* 255 */ rdd_mutableStateArray_1[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[1], 8); /* 256 */ rdd_mutableStateArray_0[2] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(2, 32); /* 257 */ rdd_mutableStateArray_0[3] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(rdd_mutableStateArray_0[2], 4); /* 258 */ rdd_mutableStateArray_1[2] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[3], 4); /* 259 */ rdd_mutableStateArray_1[3] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[3], 8); /* 260 */ /* 261 */ } /* 262 */ /* 263 */ } 20/01/23 18:54:03.486 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$cogroup$4 20/01/23 18:54:03.487 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$cogroup$4) is now cleaned +++ 20/01/23 18:54:03.488 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$join$2 20/01/23 18:54:03.489 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$join$2) is now cleaned +++ 20/01/23 18:54:03.490 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$computeFactors$4 20/01/23 18:54:03.491 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$computeFactors$4) is now cleaned +++ 20/01/23 18:54:03.492 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$train$11 20/01/23 18:54:03.493 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$train$11) is now cleaned +++ 20/01/23 18:54:03.494 Thread-4 DEBUG CoGroupedRDD: Adding one-to-one dependency with userInBlocks MapPartitionsRDD[19] at mapValues at ALS.scala:1613 20/01/23 18:54:03.494 Thread-4 DEBUG CoGroupedRDD: Adding one-to-one dependency with ShuffledRDD[41] at groupByKey at ALS.scala:1680 20/01/23 18:54:03.499 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$cogroup$4 20/01/23 18:54:03.500 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$cogroup$4) is now cleaned +++ 20/01/23 18:54:03.501 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$join$2 20/01/23 18:54:03.502 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$join$2) is now cleaned +++ 20/01/23 18:54:03.504 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$train$12 20/01/23 18:54:03.505 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$train$12) is now cleaned +++ 20/01/23 18:54:03.513 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_5 stored as values in memory (estimated size 4.9 KiB, free 366.3 MiB) 20/01/23 18:54:03.513 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_5 locally took 15 ms 20/01/23 18:54:03.514 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_5 without replication took 15 ms 20/01/23 18:54:03.515 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_5_piece0 stored as bytes in memory (estimated size 2.7 KiB, free 366.3 MiB) 20/01/23 18:54:03.516 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added broadcast_5_piece0 in memory on 192.168.1.225:41233 (size: 2.7 KiB, free: 366.3 MiB) 20/01/23 18:54:03.516 dag-scheduler-event-loop DEBUG BlockManagerMaster: Updated info of block broadcast_5_piece0 20/01/23 18:54:03.516 dag-scheduler-event-loop DEBUG BlockManager: Told master about block broadcast_5_piece0 20/01/23 18:54:03.516 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_5_piece0 locally took 1 ms 20/01/23 18:54:03.516 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_5_piece0 without replication took 1 ms 20/01/23 18:54:03.517 dag-scheduler-event-loop INFO SparkContext: Created broadcast 5 from broadcast at DAGScheduler.scala:1200 20/01/23 18:54:03.517 dag-scheduler-event-loop INFO DAGScheduler: Submitting 4 missing tasks from ResultStage 3 (MapPartitionsRDD[21] at map at CountVectorizer.scala:230) (first 15 tasks are for partitions Vector(0, 1, 2, 3)) 20/01/23 18:54:03.517 dag-scheduler-event-loop INFO TaskSchedulerImpl: Adding task set 3.0 with 4 tasks 20/01/23 18:54:03.518 dag-scheduler-event-loop DEBUG TaskSetManager: Epoch for TaskSet 3.0: 1 20/01/23 18:54:03.520 dag-scheduler-event-loop DEBUG TaskSetManager: Adding pending tasks took 1 ms 20/01/23 18:54:03.520 dag-scheduler-event-loop DEBUG TaskSetManager: Valid locality levels for TaskSet 3.0: NODE_LOCAL, NO_PREF, ANY 20/01/23 18:54:03.521 dispatcher-event-loop-0 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_3.0, runningTasks: 0 20/01/23 18:54:03.523 dispatcher-event-loop-0 INFO TaskSetManager: Starting task 1.0 in stage 3.0 (TID 12, 192.168.1.225, executor driver, partition 1, NODE_LOCAL, 7143 bytes) 20/01/23 18:54:03.524 dispatcher-event-loop-0 INFO TaskSetManager: Starting task 2.0 in stage 3.0 (TID 13, 192.168.1.225, executor driver, partition 2, NODE_LOCAL, 7143 bytes) 20/01/23 18:54:03.524 dispatcher-event-loop-0 INFO TaskSetManager: Starting task 3.0 in stage 3.0 (TID 14, 192.168.1.225, executor driver, partition 3, NODE_LOCAL, 7143 bytes) 20/01/23 18:54:03.524 dispatcher-event-loop-0 DEBUG TaskSetManager: No tasks for locality level NODE_LOCAL, so moving to locality level NO_PREF 20/01/23 18:54:03.525 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$train$14 20/01/23 18:54:03.526 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$train$14) is now cleaned +++ 20/01/23 18:54:03.532 dispatcher-event-loop-0 DEBUG TaskSetManager: Moving to ANY after waiting for 0ms 20/01/23 18:54:03.533 dispatcher-event-loop-0 INFO TaskSetManager: Starting task 0.0 in stage 3.0 (TID 15, 192.168.1.225, executor driver, partition 0, PROCESS_LOCAL, 7143 bytes) 20/01/23 18:54:03.533 Executor task launch worker for task 14 INFO Executor: Running task 3.0 in stage 3.0 (TID 14) 20/01/23 18:54:03.533 Executor task launch worker for task 12 INFO Executor: Running task 1.0 in stage 3.0 (TID 12) 20/01/23 18:54:03.534 Executor task launch worker for task 12 DEBUG ExecutorMetricsPoller: stageTCMP: (3, 0) -> 1 20/01/23 18:54:03.534 Executor task launch worker for task 14 DEBUG ExecutorMetricsPoller: stageTCMP: (3, 0) -> 2 20/01/23 18:54:03.534 Executor task launch worker for task 15 INFO Executor: Running task 0.0 in stage 3.0 (TID 15) 20/01/23 18:54:03.535 Executor task launch worker for task 12 DEBUG BlockManager: Getting local block broadcast_5 20/01/23 18:54:03.535 Executor task launch worker for task 15 DEBUG ExecutorMetricsPoller: stageTCMP: (3, 0) -> 3 20/01/23 18:54:03.535 Executor task launch worker for task 12 DEBUG BlockManager: Level for block broadcast_5 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:03.535 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$cogroup$4 20/01/23 18:54:03.536 Executor task launch worker for task 13 INFO Executor: Running task 2.0 in stage 3.0 (TID 13) 20/01/23 18:54:03.537 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$cogroup$4) is now cleaned +++ 20/01/23 18:54:03.537 Executor task launch worker for task 13 DEBUG ExecutorMetricsPoller: stageTCMP: (3, 0) -> 4 20/01/23 18:54:03.540 Executor task launch worker for task 12 DEBUG BlockManager: Getting local block rdd_21_1 20/01/23 18:54:03.540 Executor task launch worker for task 12 DEBUG BlockManager: Block rdd_21_1 was not found 20/01/23 18:54:03.541 Executor task launch worker for task 12 DEBUG BlockManager: Getting remote block rdd_21_1 20/01/23 18:54:03.543 Executor task launch worker for task 13 DEBUG BlockManager: Getting local block rdd_21_2 20/01/23 18:54:03.543 Executor task launch worker for task 13 DEBUG BlockManager: Block rdd_21_2 was not found 20/01/23 18:54:03.543 Executor task launch worker for task 13 DEBUG BlockManager: Getting remote block rdd_21_2 20/01/23 18:54:03.544 Executor task launch worker for task 13 DEBUG BlockManager: Block rdd_21_2 is unknown by block manager master 20/01/23 18:54:03.546 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$join$2 20/01/23 18:54:03.546 Executor task launch worker for task 13 DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 0, partitions 2-3 20/01/23 18:54:03.547 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$join$2) is now cleaned +++ 20/01/23 18:54:03.548 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$train$15 20/01/23 18:54:03.549 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$train$15) is now cleaned +++ 20/01/23 18:54:03.553 Executor task launch worker for task 12 DEBUG BlockManager: Block rdd_21_1 is unknown by block manager master 20/01/23 18:54:03.553 Executor task launch worker for task 12 DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 0, partitions 1-2 20/01/23 18:54:03.553 Thread-4 DEBUG CoGroupedRDD: Adding one-to-one dependency with MapPartitionsRDD[46] at mapValues at ALS.scala:1026 20/01/23 18:54:03.553 Thread-4 DEBUG CoGroupedRDD: Adding one-to-one dependency with MapPartitionsRDD[45] at mapValues at ALS.scala:1688 20/01/23 18:54:03.554 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$count$1$adapted 20/01/23 18:54:03.556 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$count$1$adapted) is now cleaned +++ 20/01/23 18:54:03.556 Executor task launch worker for task 14 DEBUG BlockManager: Getting local block rdd_21_3 20/01/23 18:54:03.556 Executor task launch worker for task 14 DEBUG BlockManager: Block rdd_21_3 was not found 20/01/23 18:54:03.556 Executor task launch worker for task 14 DEBUG BlockManager: Getting remote block rdd_21_3 20/01/23 18:54:03.556 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$runJob$5 20/01/23 18:54:03.558 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$runJob$5) is now cleaned +++ 20/01/23 18:54:03.558 Thread-4 INFO SparkContext: Starting job: count at ALS.scala:1048 20/01/23 18:54:03.558 Executor task launch worker for task 15 DEBUG BlockManager: Getting local block rdd_21_0 20/01/23 18:54:03.558 Executor task launch worker for task 15 DEBUG BlockManager: Block rdd_21_0 was not found 20/01/23 18:54:03.558 Executor task launch worker for task 15 DEBUG BlockManager: Getting remote block rdd_21_0 20/01/23 18:54:03.559 Executor task launch worker for task 15 DEBUG BlockManager: Block rdd_21_0 is unknown by block manager master 20/01/23 18:54:03.559 Executor task launch worker for task 15 DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 0, partitions 0-1 20/01/23 18:54:03.559 Executor task launch worker for task 14 DEBUG BlockManager: Block rdd_21_3 is unknown by block manager master 20/01/23 18:54:03.559 Executor task launch worker for task 14 DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 0, partitions 3-4 20/01/23 18:54:03.562 dag-scheduler-event-loop DEBUG CoGroupedRDD: Adding one-to-one dependency with itemOutBlocks MapPartitionsRDD[25] at mapValues at ALS.scala:1622 20/01/23 18:54:03.562 dag-scheduler-event-loop DEBUG CoGroupedRDD: Adding one-to-one dependency with MapPartitionsRDD[36] at mapValues at ALS.scala:1688 20/01/23 18:54:03.563 dag-scheduler-event-loop DEBUG CoGroupedRDD: Adding one-to-one dependency with userOutBlocks MapPartitionsRDD[20] at mapValues at ALS.scala:1622 20/01/23 18:54:03.563 dag-scheduler-event-loop DEBUG CoGroupedRDD: Adding one-to-one dependency with MapPartitionsRDD[26] at mapPartitions at ALS.scala:1249 20/01/23 18:54:03.564 Thread-4 INFO CodeGenerator: Code generated in 103.439505 ms 20/01/23 18:54:03.564 Executor task launch worker for task 15 DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 20/01/23 18:54:03.565 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$doExecute$4$adapted 20/01/23 18:54:03.565 dag-scheduler-event-loop INFO DAGScheduler: Registering RDD 31 (flatMap at ALS.scala:1674) as input to shuffle 4 20/01/23 18:54:03.567 dag-scheduler-event-loop INFO DAGScheduler: Registering RDD 40 (flatMap at ALS.scala:1674) as input to shuffle 3 20/01/23 18:54:03.567 Executor task launch worker for task 15 INFO ShuffleBlockFetcherIterator: Getting 0 (0.0 B) non-empty blocks including 0 (0.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 20/01/23 18:54:03.567 dag-scheduler-event-loop INFO DAGScheduler: Got job 3 (count at ALS.scala:1048) with 10 output partitions 20/01/23 18:54:03.567 dag-scheduler-event-loop INFO DAGScheduler: Final stage: ResultStage 12 (count at ALS.scala:1048) 20/01/23 18:54:03.567 dag-scheduler-event-loop INFO DAGScheduler: Parents of final stage: List(ShuffleMapStage 11, ShuffleMapStage 8) 20/01/23 18:54:03.569 Executor task launch worker for task 15 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 5 ms 20/01/23 18:54:03.569 dag-scheduler-event-loop INFO DAGScheduler: Missing parents: List(ShuffleMapStage 11) 20/01/23 18:54:03.569 Executor task launch worker for task 15 DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: 20/01/23 18:54:03.570 Executor task launch worker for task 15 DEBUG ShuffleBlockFetcherIterator: Got local blocks in 7 ms 20/01/23 18:54:03.575 dag-scheduler-event-loop DEBUG DAGScheduler: submitStage(ResultStage 12 (name=count at ALS.scala:1048;jobs=3)) 20/01/23 18:54:03.575 dag-scheduler-event-loop DEBUG DAGScheduler: missing: List(ShuffleMapStage 11) 20/01/23 18:54:03.575 dag-scheduler-event-loop DEBUG DAGScheduler: submitStage(ShuffleMapStage 11 (name=flatMap at ALS.scala:1674;jobs=3)) 20/01/23 18:54:03.576 dag-scheduler-event-loop DEBUG DAGScheduler: missing: List(ShuffleMapStage 9) 20/01/23 18:54:03.577 dag-scheduler-event-loop DEBUG DAGScheduler: submitStage(ShuffleMapStage 9 (name=flatMap at ALS.scala:1674;jobs=3)) 20/01/23 18:54:03.577 dag-scheduler-event-loop DEBUG DAGScheduler: missing: List() 20/01/23 18:54:03.578 dag-scheduler-event-loop INFO DAGScheduler: Submitting ShuffleMapStage 9 (MapPartitionsRDD[31] at flatMap at ALS.scala:1674), which has no missing parents 20/01/23 18:54:03.578 dag-scheduler-event-loop DEBUG DAGScheduler: submitMissingTasks(ShuffleMapStage 9) 20/01/23 18:54:03.585 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$doExecute$4$adapted) is now cleaned +++ 20/01/23 18:54:03.588 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_6 stored as values in memory (estimated size 31.2 KiB, free 366.1 MiB) 20/01/23 18:54:03.588 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_6 locally took 0 ms 20/01/23 18:54:03.588 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_6 without replication took 1 ms 20/01/23 18:54:03.590 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_6_piece0 stored as bytes in memory (estimated size 14.5 KiB, free 366.1 MiB) 20/01/23 18:54:03.590 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added broadcast_6_piece0 in memory on 192.168.1.225:35863 (size: 14.5 KiB, free: 366.2 MiB) 20/01/23 18:54:03.591 dag-scheduler-event-loop DEBUG BlockManagerMaster: Updated info of block broadcast_6_piece0 20/01/23 18:54:03.591 dag-scheduler-event-loop DEBUG BlockManager: Told master about block broadcast_6_piece0 20/01/23 18:54:03.591 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_6_piece0 locally took 1 ms 20/01/23 18:54:03.591 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_6_piece0 without replication took 1 ms 20/01/23 18:54:03.591 dag-scheduler-event-loop INFO SparkContext: Created broadcast 6 from broadcast at DAGScheduler.scala:1200 20/01/23 18:54:03.592 dag-scheduler-event-loop INFO DAGScheduler: Submitting 10 missing tasks from ShuffleMapStage 9 (MapPartitionsRDD[31] at flatMap at ALS.scala:1674) (first 15 tasks are for partitions Vector(0, 1, 2, 3, 4, 5, 6, 7, 8, 9)) 20/01/23 18:54:03.592 dag-scheduler-event-loop INFO TaskSchedulerImpl: Adding task set 9.0 with 10 tasks 20/01/23 18:54:03.592 dag-scheduler-event-loop DEBUG TaskSetManager: Epoch for TaskSet 9.0: 3 20/01/23 18:54:03.592 dag-scheduler-event-loop DEBUG TaskSetManager: Adding pending tasks took 0 ms 20/01/23 18:54:03.593 dag-scheduler-event-loop DEBUG TaskSetManager: Valid locality levels for TaskSet 9.0: PROCESS_LOCAL, NODE_LOCAL, ANY 20/01/23 18:54:03.593 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$doExecute$4$adapted 20/01/23 18:54:03.594 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$doExecute$4$adapted) is now cleaned +++ 20/01/23 18:54:03.573 Executor task launch worker for task 14 DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 20/01/23 18:54:03.598 Executor task launch worker for task 13 DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 20/01/23 18:54:03.599 Executor task launch worker for task 12 DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 20/01/23 18:54:03.606 Executor task launch worker for task 12 INFO ShuffleBlockFetcherIterator: Getting 3 (567.0 B) non-empty blocks including 3 (567.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 20/01/23 18:54:03.606 Executor task launch worker for task 12 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 7 ms 20/01/23 18:54:03.606 Executor task launch worker for task 12 DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_0_8_1,0), (shuffle_0_9_1,1), (shuffle_0_10_1,2) 20/01/23 18:54:03.609 dispatcher-event-loop-1 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_9.0, runningTasks: 0 20/01/23 18:54:03.609 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$collect$2 20/01/23 18:54:03.609 Executor task launch worker for task 13 INFO ShuffleBlockFetcherIterator: Getting 2 (378.0 B) non-empty blocks including 2 (378.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 20/01/23 18:54:03.610 Executor task launch worker for task 13 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 11 ms 20/01/23 18:54:03.610 Executor task launch worker for task 13 DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_0_8_2,0), (shuffle_0_10_2,2) 20/01/23 18:54:03.611 dispatcher-event-loop-1 INFO TaskSetManager: Starting task 0.0 in stage 9.0 (TID 33, 192.168.1.225, executor driver, partition 0, PROCESS_LOCAL, 7362 bytes) 20/01/23 18:54:03.612 dispatcher-event-loop-1 INFO TaskSetManager: Starting task 1.0 in stage 9.0 (TID 34, 192.168.1.225, executor driver, partition 1, PROCESS_LOCAL, 7362 bytes) 20/01/23 18:54:03.612 dispatcher-event-loop-1 INFO TaskSetManager: Starting task 2.0 in stage 9.0 (TID 35, 192.168.1.225, executor driver, partition 2, PROCESS_LOCAL, 7362 bytes) 20/01/23 18:54:03.612 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$collect$2) is now cleaned +++ 20/01/23 18:54:03.612 dispatcher-event-loop-1 INFO TaskSetManager: Starting task 3.0 in stage 9.0 (TID 36, 192.168.1.225, executor driver, partition 3, PROCESS_LOCAL, 7362 bytes) 20/01/23 18:54:03.613 Executor task launch worker for task 35 INFO Executor: Running task 2.0 in stage 9.0 (TID 35) 20/01/23 18:54:03.613 Executor task launch worker for task 34 INFO Executor: Running task 1.0 in stage 9.0 (TID 34) 20/01/23 18:54:03.613 Executor task launch worker for task 33 INFO Executor: Running task 0.0 in stage 9.0 (TID 33) 20/01/23 18:54:03.613 Executor task launch worker for task 36 INFO Executor: Running task 3.0 in stage 9.0 (TID 36) 20/01/23 18:54:03.613 Executor task launch worker for task 35 DEBUG ExecutorMetricsPoller: stageTCMP: (9, 0) -> 1 20/01/23 18:54:03.614 Executor task launch worker for task 34 DEBUG ExecutorMetricsPoller: stageTCMP: (9, 0) -> 4 20/01/23 18:54:03.614 Executor task launch worker for task 14 INFO ShuffleBlockFetcherIterator: Getting 2 (378.0 B) non-empty blocks including 2 (378.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 20/01/23 18:54:03.614 Executor task launch worker for task 12 DEBUG ShuffleBlockFetcherIterator: Got local blocks in 14 ms 20/01/23 18:54:03.614 Executor task launch worker for task 14 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 40 ms 20/01/23 18:54:03.614 Executor task launch worker for task 14 DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_0_8_3,0), (shuffle_0_11_3,3) 20/01/23 18:54:03.614 Executor task launch worker for task 35 DEBUG BlockManager: Getting local block broadcast_6 20/01/23 18:54:03.613 Executor task launch worker for task 33 DEBUG ExecutorMetricsPoller: stageTCMP: (9, 0) -> 2 20/01/23 18:54:03.613 Executor task launch worker for task 36 DEBUG ExecutorMetricsPoller: stageTCMP: (9, 0) -> 3 20/01/23 18:54:03.614 Executor task launch worker for task 35 DEBUG BlockManager: Level for block broadcast_6 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:03.614 Executor task launch worker for task 14 DEBUG ShuffleBlockFetcherIterator: Got local blocks in 41 ms 20/01/23 18:54:03.615 Executor task launch worker for task 15 DEBUG TaskMemoryManager: Task 15 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@2298e83f 20/01/23 18:54:03.616 Executor task launch worker for task 15 INFO MemoryStore: Block rdd_21_0 stored as values in memory (estimated size 16.0 B, free 366.3 MiB) 20/01/23 18:54:03.616 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_21_0 in memory on 192.168.1.225:41233 (size: 16.0 B, free: 366.3 MiB) 20/01/23 18:54:03.617 Executor task launch worker for task 13 DEBUG ShuffleBlockFetcherIterator: Got local blocks in 18 ms 20/01/23 18:54:03.625 Executor task launch worker for task 15 DEBUG BlockManagerMaster: Updated info of block rdd_21_0 20/01/23 18:54:03.625 Executor task launch worker for task 15 DEBUG BlockManager: Told master about block rdd_21_0 20/01/23 18:54:03.625 Executor task launch worker for task 15 DEBUG BlockManager: Put block rdd_21_0 locally took 66 ms 20/01/23 18:54:03.625 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$runJob$5 20/01/23 18:54:03.626 Executor task launch worker for task 15 DEBUG BlockManager: Putting block rdd_21_0 without replication took 67 ms 20/01/23 18:54:03.626 Executor task launch worker for task 15 DEBUG BlockManager: Getting local block rdd_21_0 20/01/23 18:54:03.628 Executor task launch worker for task 15 DEBUG BlockManager: Level for block rdd_21_0 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:03.628 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$runJob$5) is now cleaned +++ 20/01/23 18:54:03.629 Thread-4 INFO SparkContext: Starting job: collect at ClusteringEvaluator.scala:179 20/01/23 18:54:03.630 Executor task launch worker for task 15 INFO Executor: Finished task 0.0 in stage 3.0 (TID 15). 1305 bytes result sent to driver 20/01/23 18:54:03.633 dispatcher-event-loop-3 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_3.0, runningTasks: 3 20/01/23 18:54:03.634 task-result-getter-1 INFO TaskSetManager: Finished task 0.0 in stage 3.0 (TID 15) in 102 ms on 192.168.1.225 (executor driver) (1/4) 20/01/23 18:54:03.634 Executor task launch worker for task 15 DEBUG ExecutorMetricsPoller: stageTCMP: (3, 0) -> 3 20/01/23 18:54:03.635 Executor task launch worker for task 14 DEBUG TaskMemoryManager: Task 14 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@13c3494b 20/01/23 18:54:03.636 Executor task launch worker for task 14 INFO MemoryStore: Block rdd_21_3 stored as values in memory (estimated size 120.0 B, free 366.3 MiB) 20/01/23 18:54:03.636 Executor task launch worker for task 13 DEBUG TaskMemoryManager: Task 13 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@be3d404 20/01/23 18:54:03.636 Executor task launch worker for task 13 INFO MemoryStore: Block rdd_21_2 stored as values in memory (estimated size 120.0 B, free 366.3 MiB) 20/01/23 18:54:03.637 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_21_3 in memory on 192.168.1.225:41233 (size: 120.0 B, free: 366.3 MiB) 20/01/23 18:54:03.637 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_21_2 in memory on 192.168.1.225:41233 (size: 120.0 B, free: 366.3 MiB) 20/01/23 18:54:03.638 Executor task launch worker for task 13 DEBUG BlockManagerMaster: Updated info of block rdd_21_2 20/01/23 18:54:03.638 Executor task launch worker for task 13 DEBUG BlockManager: Told master about block rdd_21_2 20/01/23 18:54:03.638 Executor task launch worker for task 13 DEBUG BlockManager: Put block rdd_21_2 locally took 93 ms 20/01/23 18:54:03.638 Executor task launch worker for task 13 DEBUG BlockManager: Putting block rdd_21_2 without replication took 93 ms 20/01/23 18:54:03.638 Executor task launch worker for task 13 DEBUG BlockManager: Getting local block rdd_21_2 20/01/23 18:54:03.638 Executor task launch worker for task 13 DEBUG BlockManager: Level for block rdd_21_2 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:03.640 dag-scheduler-event-loop INFO DAGScheduler: Registering RDD 14 (collect at ClusteringEvaluator.scala:179) as input to shuffle 1 20/01/23 18:54:03.641 dag-scheduler-event-loop INFO DAGScheduler: Got job 2 (collect at ClusteringEvaluator.scala:179) with 1 output partitions 20/01/23 18:54:03.641 dag-scheduler-event-loop INFO DAGScheduler: Final stage: ResultStage 4 (collect at ClusteringEvaluator.scala:179) 20/01/23 18:54:03.641 dag-scheduler-event-loop INFO DAGScheduler: Parents of final stage: List(ShuffleMapStage 3) 20/01/23 18:54:03.642 dag-scheduler-event-loop INFO DAGScheduler: Missing parents: List(ShuffleMapStage 3) 20/01/23 18:54:03.642 dag-scheduler-event-loop DEBUG DAGScheduler: submitStage(ResultStage 4 (name=collect at ClusteringEvaluator.scala:179;jobs=2)) 20/01/23 18:54:03.642 Executor task launch worker for task 12 DEBUG TaskMemoryManager: Task 12 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@6c3f2d72 20/01/23 18:54:03.643 Executor task launch worker for task 12 INFO MemoryStore: Block rdd_21_1 stored as values in memory (estimated size 120.0 B, free 366.3 MiB) 20/01/23 18:54:03.643 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_21_1 in memory on 192.168.1.225:41233 (size: 120.0 B, free: 366.3 MiB) 20/01/23 18:54:03.644 Executor task launch worker for task 12 DEBUG BlockManagerMaster: Updated info of block rdd_21_1 20/01/23 18:54:03.644 Executor task launch worker for task 12 DEBUG BlockManager: Told master about block rdd_21_1 20/01/23 18:54:03.644 Executor task launch worker for task 12 DEBUG BlockManager: Put block rdd_21_1 locally took 90 ms 20/01/23 18:54:03.644 Executor task launch worker for task 12 DEBUG BlockManager: Putting block rdd_21_1 without replication took 91 ms 20/01/23 18:54:03.644 Executor task launch worker for task 12 DEBUG BlockManager: Getting local block rdd_21_1 20/01/23 18:54:03.644 Executor task launch worker for task 12 DEBUG BlockManager: Level for block rdd_21_1 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:03.645 dag-scheduler-event-loop DEBUG DAGScheduler: missing: List(ShuffleMapStage 3) 20/01/23 18:54:03.645 dag-scheduler-event-loop DEBUG DAGScheduler: submitStage(ShuffleMapStage 3 (name=collect at ClusteringEvaluator.scala:179;jobs=2)) 20/01/23 18:54:03.645 dag-scheduler-event-loop DEBUG DAGScheduler: missing: List() 20/01/23 18:54:03.646 dag-scheduler-event-loop INFO DAGScheduler: Submitting ShuffleMapStage 3 (MapPartitionsRDD[14] at collect at ClusteringEvaluator.scala:179), which has no missing parents 20/01/23 18:54:03.646 dag-scheduler-event-loop DEBUG DAGScheduler: submitMissingTasks(ShuffleMapStage 3) 20/01/23 18:54:03.648 Executor task launch worker for task 35 DEBUG BlockManager: Getting local block rdd_20_2 20/01/23 18:54:03.648 Executor task launch worker for task 35 DEBUG BlockManager: Level for block rdd_20_2 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:03.648 Executor task launch worker for task 35 INFO BlockManager: Found block rdd_20_2 locally 20/01/23 18:54:03.648 Executor task launch worker for task 35 DEBUG BlockManager: Getting local block rdd_19_2 20/01/23 18:54:03.648 Executor task launch worker for task 35 DEBUG BlockManager: Level for block rdd_19_2 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:03.648 Executor task launch worker for task 35 INFO BlockManager: Found block rdd_19_2 locally 20/01/23 18:54:03.654 Executor task launch worker for task 36 DEBUG BlockManager: Getting local block rdd_20_3 20/01/23 18:54:03.654 Executor task launch worker for task 33 DEBUG BlockManager: Getting local block rdd_20_0 20/01/23 18:54:03.655 Executor task launch worker for task 36 DEBUG BlockManager: Level for block rdd_20_3 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:03.655 Executor task launch worker for task 36 INFO BlockManager: Found block rdd_20_3 locally 20/01/23 18:54:03.655 Executor task launch worker for task 33 DEBUG BlockManager: Level for block rdd_20_0 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:03.655 Executor task launch worker for task 33 INFO BlockManager: Found block rdd_20_0 locally 20/01/23 18:54:03.655 Executor task launch worker for task 33 DEBUG BlockManager: Getting local block rdd_19_0 20/01/23 18:54:03.655 Executor task launch worker for task 33 DEBUG BlockManager: Level for block rdd_19_0 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:03.655 Executor task launch worker for task 33 INFO BlockManager: Found block rdd_19_0 locally 20/01/23 18:54:03.655 Executor task launch worker for task 36 DEBUG BlockManager: Getting local block rdd_19_3 20/01/23 18:54:03.657 Executor task launch worker for task 36 DEBUG BlockManager: Level for block rdd_19_3 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:03.657 Executor task launch worker for task 36 INFO BlockManager: Found block rdd_19_3 locally 20/01/23 18:54:03.658 Executor task launch worker for task 34 DEBUG BlockManager: Getting local block rdd_20_1 20/01/23 18:54:03.658 Executor task launch worker for task 34 DEBUG BlockManager: Level for block rdd_20_1 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:03.658 Executor task launch worker for task 34 INFO BlockManager: Found block rdd_20_1 locally 20/01/23 18:54:03.658 Executor task launch worker for task 34 DEBUG BlockManager: Getting local block rdd_19_1 20/01/23 18:54:03.658 Executor task launch worker for task 34 DEBUG BlockManager: Level for block rdd_19_1 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:03.658 Executor task launch worker for task 34 INFO BlockManager: Found block rdd_19_1 locally 20/01/23 18:54:03.661 Executor task launch worker for task 14 DEBUG BlockManagerMaster: Updated info of block rdd_21_3 20/01/23 18:54:03.661 Executor task launch worker for task 14 DEBUG BlockManager: Told master about block rdd_21_3 20/01/23 18:54:03.661 Executor task launch worker for task 14 DEBUG BlockManager: Put block rdd_21_3 locally took 101 ms 20/01/23 18:54:03.661 Executor task launch worker for task 14 DEBUG BlockManager: Putting block rdd_21_3 without replication took 101 ms 20/01/23 18:54:03.661 Executor task launch worker for task 14 DEBUG BlockManager: Getting local block rdd_21_3 20/01/23 18:54:03.661 Executor task launch worker for task 14 DEBUG BlockManager: Level for block rdd_21_3 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:03.661 Executor task launch worker for task 13 INFO Executor: Finished task 2.0 in stage 3.0 (TID 13). 1305 bytes result sent to driver 20/01/23 18:54:03.662 Executor task launch worker for task 35 WARN BLAS: Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS 20/01/23 18:54:03.662 dispatcher-event-loop-2 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_3.0, runningTasks: 2 20/01/23 18:54:03.662 Executor task launch worker for task 13 DEBUG ExecutorMetricsPoller: stageTCMP: (3, 0) -> 2 20/01/23 18:54:03.662 Executor task launch worker for task 35 WARN BLAS: Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS 20/01/23 18:54:03.662 task-result-getter-0 INFO TaskSetManager: Finished task 2.0 in stage 3.0 (TID 13) in 138 ms on 192.168.1.225 (executor driver) (2/4) 20/01/23 18:54:03.664 Executor task launch worker for task 36 DEBUG TaskMemoryManager: Task 36 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@53e34235 20/01/23 18:54:03.666 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_4 stored as values in memory (estimated size 24.6 KiB, free 366.2 MiB) 20/01/23 18:54:03.667 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_4 locally took 1 ms 20/01/23 18:54:03.667 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_4 without replication took 1 ms 20/01/23 18:54:03.668 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_4_piece0 stored as bytes in memory (estimated size 11.1 KiB, free 366.2 MiB) 20/01/23 18:54:03.669 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added broadcast_4_piece0 in memory on 192.168.1.225:45461 (size: 11.1 KiB, free: 366.3 MiB) 20/01/23 18:54:03.671 Executor task launch worker for task 36 INFO Executor: Finished task 3.0 in stage 9.0 (TID 36). 2002 bytes result sent to driver 20/01/23 18:54:03.671 Executor task launch worker for task 36 DEBUG ExecutorMetricsPoller: stageTCMP: (9, 0) -> 3 20/01/23 18:54:03.671 dispatcher-event-loop-2 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_9.0, runningTasks: 3 20/01/23 18:54:03.672 dispatcher-event-loop-2 INFO TaskSetManager: Starting task 4.0 in stage 9.0 (TID 37, 192.168.1.225, executor driver, partition 4, PROCESS_LOCAL, 7362 bytes) 20/01/23 18:54:03.672 task-result-getter-2 INFO TaskSetManager: Finished task 3.0 in stage 9.0 (TID 36) in 60 ms on 192.168.1.225 (executor driver) (1/10) 20/01/23 18:54:03.672 Executor task launch worker for task 12 INFO Executor: Finished task 1.0 in stage 3.0 (TID 12). 1305 bytes result sent to driver 20/01/23 18:54:03.673 Executor task launch worker for task 12 DEBUG ExecutorMetricsPoller: stageTCMP: (3, 0) -> 1 20/01/23 18:54:03.673 Executor task launch worker for task 37 INFO Executor: Running task 4.0 in stage 9.0 (TID 37) 20/01/23 18:54:03.673 dispatcher-event-loop-0 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_3.0, runningTasks: 1 20/01/23 18:54:03.674 dag-scheduler-event-loop DEBUG DAGScheduler: ShuffleMapTask finished on driver 20/01/23 18:54:03.674 task-result-getter-2 INFO TaskSetManager: Finished task 1.0 in stage 3.0 (TID 12) in 153 ms on 192.168.1.225 (executor driver) (3/4) 20/01/23 18:54:03.675 Executor task launch worker for task 37 DEBUG ExecutorMetricsPoller: stageTCMP: (9, 0) -> 4 20/01/23 18:54:03.676 Executor task launch worker for task 14 INFO Executor: Finished task 3.0 in stage 3.0 (TID 14). 1305 bytes result sent to driver 20/01/23 18:54:03.676 Executor task launch worker for task 14 DEBUG ExecutorMetricsPoller: removing (3, 0) from stageTCMP 20/01/23 18:54:03.677 dispatcher-event-loop-1 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_3.0, runningTasks: 0 20/01/23 18:54:03.678 task-result-getter-3 INFO TaskSetManager: Finished task 3.0 in stage 3.0 (TID 14) in 154 ms on 192.168.1.225 (executor driver) (4/4) 20/01/23 18:54:03.678 task-result-getter-3 INFO TaskSchedulerImpl: Removed TaskSet 3.0, whose tasks have all completed, from pool 20/01/23 18:54:03.679 dag-scheduler-event-loop INFO DAGScheduler: ResultStage 3 (count at CountVectorizer.scala:233) finished in 0.206 s 20/01/23 18:54:03.679 dag-scheduler-event-loop DEBUG DAGScheduler: After removal of stage 2, remaining stages = 1 20/01/23 18:54:03.679 dag-scheduler-event-loop DEBUG DAGScheduler: After removal of stage 3, remaining stages = 0 20/01/23 18:54:03.679 dag-scheduler-event-loop INFO DAGScheduler: Job 2 is finished. Cancelling potential speculative or zombie tasks for this job 20/01/23 18:54:03.679 dag-scheduler-event-loop INFO TaskSchedulerImpl: Killing all running tasks in stage 3: Stage finished 20/01/23 18:54:03.680 Thread-4 INFO DAGScheduler: Job 2 finished: count at CountVectorizer.scala:233, took 1.350427 s 20/01/23 18:54:03.680 Executor task launch worker for task 37 DEBUG BlockManager: Getting local block rdd_20_4 20/01/23 18:54:03.680 Executor task launch worker for task 37 DEBUG BlockManager: Level for block rdd_20_4 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:03.680 Executor task launch worker for task 37 INFO BlockManager: Found block rdd_20_4 locally 20/01/23 18:54:03.680 Executor task launch worker for task 37 DEBUG BlockManager: Getting local block rdd_19_4 20/01/23 18:54:03.680 Executor task launch worker for task 37 DEBUG BlockManager: Level for block rdd_19_4 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:03.680 Executor task launch worker for task 37 INFO BlockManager: Found block rdd_19_4 locally 20/01/23 18:54:03.681 Executor task launch worker for task 37 DEBUG TaskMemoryManager: Task 37 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@2390a504 20/01/23 18:54:03.682 dag-scheduler-event-loop DEBUG BlockManagerMaster: Updated info of block broadcast_4_piece0 20/01/23 18:54:03.682 dag-scheduler-event-loop DEBUG BlockManager: Told master about block broadcast_4_piece0 20/01/23 18:54:03.682 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_4_piece0 locally took 14 ms 20/01/23 18:54:03.682 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_4_piece0 without replication took 14 ms 20/01/23 18:54:03.683 dag-scheduler-event-loop INFO SparkContext: Created broadcast 4 from broadcast at DAGScheduler.scala:1200 20/01/23 18:54:03.684 dag-scheduler-event-loop INFO DAGScheduler: Submitting 4 missing tasks from ShuffleMapStage 3 (MapPartitionsRDD[14] at collect at ClusteringEvaluator.scala:179) (first 15 tasks are for partitions Vector(0, 1, 2, 3)) 20/01/23 18:54:03.684 dag-scheduler-event-loop INFO TaskSchedulerImpl: Adding task set 3.0 with 4 tasks 20/01/23 18:54:03.684 dag-scheduler-event-loop DEBUG TaskSetManager: Epoch for TaskSet 3.0: 1 20/01/23 18:54:03.685 dag-scheduler-event-loop DEBUG TaskSetManager: Adding pending tasks took 0 ms 20/01/23 18:54:03.685 dag-scheduler-event-loop DEBUG TaskSetManager: Valid locality levels for TaskSet 3.0: NO_PREF, ANY 20/01/23 18:54:03.686 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$takeOrdered$2 20/01/23 18:54:03.689 Executor task launch worker for task 37 INFO Executor: Finished task 4.0 in stage 9.0 (TID 37). 2002 bytes result sent to driver 20/01/23 18:54:03.689 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$takeOrdered$2) is now cleaned +++ 20/01/23 18:54:03.689 dispatcher-event-loop-3 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_9.0, runningTasks: 3 20/01/23 18:54:03.690 Executor task launch worker for task 34 DEBUG TaskMemoryManager: Task 34 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@18101ba7 20/01/23 18:54:03.690 dispatcher-event-loop-3 INFO TaskSetManager: Starting task 5.0 in stage 9.0 (TID 38, 192.168.1.225, executor driver, partition 5, PROCESS_LOCAL, 7362 bytes) 20/01/23 18:54:03.690 Executor task launch worker for task 38 INFO Executor: Running task 5.0 in stage 9.0 (TID 38) 20/01/23 18:54:03.691 task-result-getter-1 INFO TaskSetManager: Finished task 4.0 in stage 9.0 (TID 37) in 19 ms on 192.168.1.225 (executor driver) (2/10) 20/01/23 18:54:03.692 dag-scheduler-event-loop DEBUG DAGScheduler: ShuffleMapTask finished on driver 20/01/23 18:54:03.693 Executor task launch worker for task 38 DEBUG ExecutorMetricsPoller: stageTCMP: (9, 0) -> 5 20/01/23 18:54:03.693 Executor task launch worker for task 33 DEBUG TaskMemoryManager: Task 33 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@7b4bfddb 20/01/23 18:54:03.693 dispatcher-event-loop-2 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_3.0, runningTasks: 0 20/01/23 18:54:03.694 dispatcher-event-loop-2 INFO TaskSetManager: Starting task 0.0 in stage 3.0 (TID 9, 192.168.1.225, executor driver, partition 0, PROCESS_LOCAL, 7387 bytes) 20/01/23 18:54:03.694 dispatcher-event-loop-2 INFO TaskSetManager: Starting task 1.0 in stage 3.0 (TID 10, 192.168.1.225, executor driver, partition 1, PROCESS_LOCAL, 7452 bytes) 20/01/23 18:54:03.694 dispatcher-event-loop-2 INFO TaskSetManager: Starting task 2.0 in stage 3.0 (TID 11, 192.168.1.225, executor driver, partition 2, PROCESS_LOCAL, 7387 bytes) 20/01/23 18:54:03.695 dispatcher-event-loop-2 INFO TaskSetManager: Starting task 3.0 in stage 3.0 (TID 12, 192.168.1.225, executor driver, partition 3, PROCESS_LOCAL, 7452 bytes) 20/01/23 18:54:03.695 Executor task launch worker for task 10 INFO Executor: Running task 1.0 in stage 3.0 (TID 10) 20/01/23 18:54:03.695 Executor task launch worker for task 12 INFO Executor: Running task 3.0 in stage 3.0 (TID 12) 20/01/23 18:54:03.695 Executor task launch worker for task 11 INFO Executor: Running task 2.0 in stage 3.0 (TID 11) 20/01/23 18:54:03.696 Executor task launch worker for task 12 DEBUG ExecutorMetricsPoller: stageTCMP: (3, 0) -> 1 20/01/23 18:54:03.701 Executor task launch worker for task 10 DEBUG ExecutorMetricsPoller: stageTCMP: (3, 0) -> 3 20/01/23 18:54:03.695 Executor task launch worker for task 9 INFO Executor: Running task 0.0 in stage 3.0 (TID 9) 20/01/23 18:54:03.701 Executor task launch worker for task 9 DEBUG ExecutorMetricsPoller: stageTCMP: (3, 0) -> 4 20/01/23 18:54:03.702 Executor task launch worker for task 10 DEBUG BlockManager: Getting local block broadcast_4 20/01/23 18:54:03.702 Executor task launch worker for task 10 DEBUG BlockManager: Level for block broadcast_4 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:03.696 Executor task launch worker for task 11 DEBUG ExecutorMetricsPoller: stageTCMP: (3, 0) -> 2 20/01/23 18:54:03.706 Executor task launch worker for task 34 INFO Executor: Finished task 1.0 in stage 9.0 (TID 34). 2174 bytes result sent to driver 20/01/23 18:54:03.706 Executor task launch worker for task 33 INFO Executor: Finished task 0.0 in stage 9.0 (TID 33). 2174 bytes result sent to driver 20/01/23 18:54:03.707 dispatcher-event-loop-2 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_9.0, runningTasks: 3 20/01/23 18:54:03.707 dispatcher-event-loop-2 INFO TaskSetManager: Starting task 6.0 in stage 9.0 (TID 39, 192.168.1.225, executor driver, partition 6, PROCESS_LOCAL, 7362 bytes) 20/01/23 18:54:03.708 task-result-getter-0 INFO TaskSetManager: Finished task 1.0 in stage 9.0 (TID 34) in 97 ms on 192.168.1.225 (executor driver) (3/10) 20/01/23 18:54:03.708 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$takeOrdered$3 20/01/23 18:54:03.709 Executor task launch worker for task 34 DEBUG ExecutorMetricsPoller: stageTCMP: (9, 0) -> 4 20/01/23 18:54:03.709 Executor task launch worker for task 33 DEBUG ExecutorMetricsPoller: stageTCMP: (9, 0) -> 3 20/01/23 18:54:03.709 dag-scheduler-event-loop DEBUG DAGScheduler: ShuffleMapTask finished on driver 20/01/23 18:54:03.711 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$takeOrdered$3) is now cleaned +++ 20/01/23 18:54:03.714 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$runJob$6 20/01/23 18:54:03.717 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$runJob$6) is now cleaned +++ 20/01/23 18:54:03.719 Thread-4 INFO SparkContext: Starting job: top at CountVectorizer.scala:236 20/01/23 18:54:03.721 dag-scheduler-event-loop INFO DAGScheduler: Got job 3 (top at CountVectorizer.scala:236) with 4 output partitions 20/01/23 18:54:03.721 dag-scheduler-event-loop INFO DAGScheduler: Final stage: ResultStage 5 (top at CountVectorizer.scala:236) 20/01/23 18:54:03.721 dag-scheduler-event-loop INFO DAGScheduler: Parents of final stage: List(ShuffleMapStage 4) 20/01/23 18:54:03.721 dispatcher-event-loop-2 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_9.0, runningTasks: 3 20/01/23 18:54:03.722 dag-scheduler-event-loop INFO DAGScheduler: Missing parents: List() 20/01/23 18:54:03.722 dag-scheduler-event-loop DEBUG DAGScheduler: submitStage(ResultStage 5 (name=top at CountVectorizer.scala:236;jobs=3)) 20/01/23 18:54:03.722 dispatcher-event-loop-2 INFO TaskSetManager: Starting task 7.0 in stage 9.0 (TID 40, 192.168.1.225, executor driver, partition 7, PROCESS_LOCAL, 7362 bytes) 20/01/23 18:54:03.723 dag-scheduler-event-loop DEBUG DAGScheduler: missing: List() 20/01/23 18:54:03.723 dag-scheduler-event-loop INFO DAGScheduler: Submitting ResultStage 5 (MapPartitionsRDD[22] at top at CountVectorizer.scala:236), which has no missing parents 20/01/23 18:54:03.723 dag-scheduler-event-loop DEBUG DAGScheduler: submitMissingTasks(ResultStage 5) 20/01/23 18:54:03.725 Executor task launch worker for task 39 INFO Executor: Running task 6.0 in stage 9.0 (TID 39) 20/01/23 18:54:03.725 Executor task launch worker for task 39 DEBUG ExecutorMetricsPoller: stageTCMP: (9, 0) -> 4 20/01/23 18:54:03.731 Executor task launch worker for task 39 DEBUG BlockManager: Getting local block rdd_20_6 20/01/23 18:54:03.731 Executor task launch worker for task 39 DEBUG BlockManager: Level for block rdd_20_6 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:03.731 Executor task launch worker for task 39 INFO BlockManager: Found block rdd_20_6 locally 20/01/23 18:54:03.731 Executor task launch worker for task 37 DEBUG ExecutorMetricsPoller: stageTCMP: (9, 0) -> 3 20/01/23 18:54:03.731 Executor task launch worker for task 39 DEBUG BlockManager: Getting local block rdd_19_6 20/01/23 18:54:03.731 Executor task launch worker for task 39 DEBUG BlockManager: Level for block rdd_19_6 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:03.731 Executor task launch worker for task 39 INFO BlockManager: Found block rdd_19_6 locally 20/01/23 18:54:03.732 Executor task launch worker for task 39 DEBUG TaskMemoryManager: Task 39 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@1a099bfc 20/01/23 18:54:03.732 Executor task launch worker for task 38 DEBUG BlockManager: Getting local block rdd_20_5 20/01/23 18:54:03.733 Executor task launch worker for task 35 DEBUG TaskMemoryManager: Task 35 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@3d73cde1 20/01/23 18:54:03.733 Executor task launch worker for task 38 DEBUG BlockManager: Level for block rdd_20_5 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:03.733 Executor task launch worker for task 38 INFO BlockManager: Found block rdd_20_5 locally 20/01/23 18:54:03.733 Executor task launch worker for task 38 DEBUG BlockManager: Getting local block rdd_19_5 20/01/23 18:54:03.733 Executor task launch worker for task 38 DEBUG BlockManager: Level for block rdd_19_5 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:03.733 Executor task launch worker for task 38 INFO BlockManager: Found block rdd_19_5 locally 20/01/23 18:54:03.733 Executor task launch worker for task 39 INFO Executor: Finished task 6.0 in stage 9.0 (TID 39). 2002 bytes result sent to driver 20/01/23 18:54:03.734 Executor task launch worker for task 38 DEBUG TaskMemoryManager: Task 38 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@7c8814ea 20/01/23 18:54:03.734 dispatcher-event-loop-1 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_9.0, runningTasks: 3 20/01/23 18:54:03.734 dispatcher-event-loop-1 INFO TaskSetManager: Starting task 8.0 in stage 9.0 (TID 41, 192.168.1.225, executor driver, partition 8, PROCESS_LOCAL, 7362 bytes) 20/01/23 18:54:03.735 Executor task launch worker for task 39 DEBUG ExecutorMetricsPoller: stageTCMP: (9, 0) -> 2 20/01/23 18:54:03.735 Executor task launch worker for task 38 INFO Executor: Finished task 5.0 in stage 9.0 (TID 38). 2002 bytes result sent to driver 20/01/23 18:54:03.735 Executor task launch worker for task 38 DEBUG ExecutorMetricsPoller: stageTCMP: (9, 0) -> 1 20/01/23 18:54:03.735 Executor task launch worker for task 40 INFO Executor: Running task 7.0 in stage 9.0 (TID 40) 20/01/23 18:54:03.735 dispatcher-event-loop-0 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_9.0, runningTasks: 3 20/01/23 18:54:03.736 Executor task launch worker for task 40 DEBUG ExecutorMetricsPoller: stageTCMP: (9, 0) -> 2 20/01/23 18:54:03.736 dispatcher-event-loop-0 INFO TaskSetManager: Starting task 9.0 in stage 9.0 (TID 42, 192.168.1.225, executor driver, partition 9, PROCESS_LOCAL, 7362 bytes) 20/01/23 18:54:03.737 task-result-getter-3 INFO TaskSetManager: Finished task 0.0 in stage 9.0 (TID 33) in 128 ms on 192.168.1.225 (executor driver) (4/10) 20/01/23 18:54:03.737 task-result-getter-3 INFO TaskSetManager: Finished task 5.0 in stage 9.0 (TID 38) in 47 ms on 192.168.1.225 (executor driver) (5/10) 20/01/23 18:54:03.738 dag-scheduler-event-loop DEBUG DAGScheduler: ShuffleMapTask finished on driver 20/01/23 18:54:03.739 task-result-getter-2 INFO TaskSetManager: Finished task 6.0 in stage 9.0 (TID 39) in 32 ms on 192.168.1.225 (executor driver) (6/10) 20/01/23 18:54:03.740 dag-scheduler-event-loop DEBUG DAGScheduler: ShuffleMapTask finished on driver 20/01/23 18:54:03.741 dag-scheduler-event-loop DEBUG DAGScheduler: ShuffleMapTask finished on driver 20/01/23 18:54:03.741 Executor task launch worker for task 40 DEBUG BlockManager: Getting local block rdd_20_7 20/01/23 18:54:03.741 Executor task launch worker for task 40 DEBUG BlockManager: Level for block rdd_20_7 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:03.741 Executor task launch worker for task 40 INFO BlockManager: Found block rdd_20_7 locally 20/01/23 18:54:03.741 Executor task launch worker for task 40 DEBUG BlockManager: Getting local block rdd_19_7 20/01/23 18:54:03.741 Executor task launch worker for task 40 DEBUG BlockManager: Level for block rdd_19_7 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:03.741 Executor task launch worker for task 40 INFO BlockManager: Found block rdd_19_7 locally 20/01/23 18:54:03.741 Executor task launch worker for task 35 INFO Executor: Finished task 2.0 in stage 9.0 (TID 35). 2217 bytes result sent to driver 20/01/23 18:54:03.744 Executor task launch worker for task 35 DEBUG ExecutorMetricsPoller: stageTCMP: (9, 0) -> 1 20/01/23 18:54:03.745 dispatcher-event-loop-3 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_9.0, runningTasks: 3 20/01/23 18:54:03.745 dispatcher-event-loop-3 DEBUG TaskSetManager: No tasks for locality level PROCESS_LOCAL, so moving to locality level NODE_LOCAL 20/01/23 18:54:03.745 dispatcher-event-loop-3 DEBUG TaskSetManager: No tasks for locality level NODE_LOCAL, so moving to locality level ANY 20/01/23 18:54:03.745 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_6 stored as values in memory (estimated size 5.9 KiB, free 366.3 MiB) 20/01/23 18:54:03.746 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_6 locally took 1 ms 20/01/23 18:54:03.746 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_6 without replication took 1 ms 20/01/23 18:54:03.747 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_6_piece0 stored as bytes in memory (estimated size 3.1 KiB, free 366.3 MiB) 20/01/23 18:54:03.748 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added broadcast_6_piece0 in memory on 192.168.1.225:41233 (size: 3.1 KiB, free: 366.3 MiB) 20/01/23 18:54:03.753 dag-scheduler-event-loop DEBUG BlockManagerMaster: Updated info of block broadcast_6_piece0 20/01/23 18:54:03.753 dag-scheduler-event-loop DEBUG BlockManager: Told master about block broadcast_6_piece0 20/01/23 18:54:03.753 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_6_piece0 locally took 6 ms 20/01/23 18:54:03.753 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_6_piece0 without replication took 6 ms 20/01/23 18:54:03.753 dag-scheduler-event-loop INFO SparkContext: Created broadcast 6 from broadcast at DAGScheduler.scala:1200 20/01/23 18:54:03.754 Executor task launch worker for task 40 DEBUG TaskMemoryManager: Task 40 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@13f027c3 20/01/23 18:54:03.755 Executor task launch worker for task 40 INFO Executor: Finished task 7.0 in stage 9.0 (TID 40). 2002 bytes result sent to driver 20/01/23 18:54:03.756 Executor task launch worker for task 40 DEBUG ExecutorMetricsPoller: removing (9, 0) from stageTCMP 20/01/23 18:54:03.756 Executor task launch worker for task 42 INFO Executor: Running task 9.0 in stage 9.0 (TID 42) 20/01/23 18:54:03.756 dispatcher-event-loop-1 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_9.0, runningTasks: 2 20/01/23 18:54:03.756 Executor task launch worker for task 42 DEBUG ExecutorMetricsPoller: stageTCMP: (9, 0) -> 1 20/01/23 18:54:03.759 Executor task launch worker for task 41 INFO Executor: Running task 8.0 in stage 9.0 (TID 41) 20/01/23 18:54:03.759 Executor task launch worker for task 41 DEBUG ExecutorMetricsPoller: stageTCMP: (9, 0) -> 2 20/01/23 18:54:03.764 Executor task launch worker for task 41 DEBUG BlockManager: Getting local block rdd_20_8 20/01/23 18:54:03.764 Executor task launch worker for task 41 DEBUG BlockManager: Level for block rdd_20_8 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:03.765 Executor task launch worker for task 41 INFO BlockManager: Found block rdd_20_8 locally 20/01/23 18:54:03.765 Executor task launch worker for task 41 DEBUG BlockManager: Getting local block rdd_19_8 20/01/23 18:54:03.765 Executor task launch worker for task 41 DEBUG BlockManager: Level for block rdd_19_8 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:03.765 Executor task launch worker for task 41 INFO BlockManager: Found block rdd_19_8 locally 20/01/23 18:54:03.765 dag-scheduler-event-loop INFO DAGScheduler: Submitting 4 missing tasks from ResultStage 5 (MapPartitionsRDD[22] at top at CountVectorizer.scala:236) (first 15 tasks are for partitions Vector(0, 1, 2, 3)) 20/01/23 18:54:03.765 dag-scheduler-event-loop INFO TaskSchedulerImpl: Adding task set 5.0 with 4 tasks 20/01/23 18:54:03.765 task-result-getter-1 INFO TaskSetManager: Finished task 2.0 in stage 9.0 (TID 35) in 153 ms on 192.168.1.225 (executor driver) (7/10) 20/01/23 18:54:03.766 task-result-getter-1 INFO TaskSetManager: Finished task 7.0 in stage 9.0 (TID 40) in 44 ms on 192.168.1.225 (executor driver) (8/10) 20/01/23 18:54:03.766 dag-scheduler-event-loop DEBUG TaskSetManager: Epoch for TaskSet 5.0: 1 20/01/23 18:54:03.766 dag-scheduler-event-loop DEBUG DAGScheduler: ShuffleMapTask finished on driver 20/01/23 18:54:03.766 Executor task launch worker for task 41 DEBUG TaskMemoryManager: Task 41 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@321b2f30 20/01/23 18:54:03.767 dag-scheduler-event-loop DEBUG TaskSetManager: Adding pending tasks took 0 ms 20/01/23 18:54:03.767 dag-scheduler-event-loop DEBUG TaskSetManager: Valid locality levels for TaskSet 5.0: PROCESS_LOCAL, NODE_LOCAL, ANY 20/01/23 18:54:03.768 dag-scheduler-event-loop DEBUG DAGScheduler: ShuffleMapTask finished on driver 20/01/23 18:54:03.768 Executor task launch worker for task 41 INFO Executor: Finished task 8.0 in stage 9.0 (TID 41). 2002 bytes result sent to driver 20/01/23 18:54:03.768 dispatcher-event-loop-3 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_5.0, runningTasks: 0 20/01/23 18:54:03.769 dispatcher-event-loop-3 INFO TaskSetManager: Starting task 0.0 in stage 5.0 (TID 16, 192.168.1.225, executor driver, partition 0, PROCESS_LOCAL, 7143 bytes) 20/01/23 18:54:03.769 dispatcher-event-loop-3 INFO TaskSetManager: Starting task 1.0 in stage 5.0 (TID 17, 192.168.1.225, executor driver, partition 1, PROCESS_LOCAL, 7143 bytes) 20/01/23 18:54:03.769 dispatcher-event-loop-3 INFO TaskSetManager: Starting task 2.0 in stage 5.0 (TID 18, 192.168.1.225, executor driver, partition 2, PROCESS_LOCAL, 7143 bytes) 20/01/23 18:54:03.770 dispatcher-event-loop-3 INFO TaskSetManager: Starting task 3.0 in stage 5.0 (TID 19, 192.168.1.225, executor driver, partition 3, PROCESS_LOCAL, 7143 bytes) 20/01/23 18:54:03.770 Executor task launch worker for task 17 INFO Executor: Running task 1.0 in stage 5.0 (TID 17) 20/01/23 18:54:03.770 Executor task launch worker for task 18 INFO Executor: Running task 2.0 in stage 5.0 (TID 18) 20/01/23 18:54:03.770 Executor task launch worker for task 42 DEBUG BlockManager: Getting local block rdd_20_9 20/01/23 18:54:03.771 Executor task launch worker for task 42 DEBUG BlockManager: Level for block rdd_20_9 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:03.771 Executor task launch worker for task 17 DEBUG ExecutorMetricsPoller: stageTCMP: (5, 0) -> 1 20/01/23 18:54:03.771 Executor task launch worker for task 42 INFO BlockManager: Found block rdd_20_9 locally 20/01/23 18:54:03.771 Executor task launch worker for task 42 DEBUG BlockManager: Getting local block rdd_19_9 20/01/23 18:54:03.771 Executor task launch worker for task 42 DEBUG BlockManager: Level for block rdd_19_9 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:03.771 Executor task launch worker for task 42 INFO BlockManager: Found block rdd_19_9 locally 20/01/23 18:54:03.770 Executor task launch worker for task 16 INFO Executor: Running task 0.0 in stage 5.0 (TID 16) 20/01/23 18:54:03.771 Executor task launch worker for task 42 DEBUG TaskMemoryManager: Task 42 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@2fa53f0f 20/01/23 18:54:03.771 Executor task launch worker for task 17 DEBUG BlockManager: Getting local block broadcast_6 20/01/23 18:54:03.772 Executor task launch worker for task 16 DEBUG ExecutorMetricsPoller: stageTCMP: (5, 0) -> 3 20/01/23 18:54:03.770 Executor task launch worker for task 19 INFO Executor: Running task 3.0 in stage 5.0 (TID 19) 20/01/23 18:54:03.772 Executor task launch worker for task 19 DEBUG ExecutorMetricsPoller: stageTCMP: (5, 0) -> 4 20/01/23 18:54:03.773 Executor task launch worker for task 42 INFO Executor: Finished task 9.0 in stage 9.0 (TID 42). 2002 bytes result sent to driver 20/01/23 18:54:03.773 Executor task launch worker for task 17 DEBUG BlockManager: Level for block broadcast_6 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:03.773 dispatcher-event-loop-3 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_9.0, runningTasks: 1 20/01/23 18:54:03.773 Executor task launch worker for task 41 DEBUG ExecutorMetricsPoller: stageTCMP: (9, 0) -> 1 20/01/23 18:54:03.774 Executor task launch worker for task 42 DEBUG ExecutorMetricsPoller: removing (9, 0) from stageTCMP 20/01/23 18:54:03.774 task-result-getter-2 INFO TaskSetManager: Finished task 8.0 in stage 9.0 (TID 41) in 39 ms on 192.168.1.225 (executor driver) (9/10) 20/01/23 18:54:03.771 Executor task launch worker for task 18 DEBUG ExecutorMetricsPoller: stageTCMP: (5, 0) -> 2 20/01/23 18:54:03.774 dispatcher-event-loop-0 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_9.0, runningTasks: 0 20/01/23 18:54:03.774 dag-scheduler-event-loop DEBUG DAGScheduler: ShuffleMapTask finished on driver 20/01/23 18:54:03.775 task-result-getter-1 INFO TaskSetManager: Finished task 9.0 in stage 9.0 (TID 42) in 39 ms on 192.168.1.225 (executor driver) (10/10) 20/01/23 18:54:03.775 task-result-getter-1 INFO TaskSchedulerImpl: Removed TaskSet 9.0, whose tasks have all completed, from pool 20/01/23 18:54:03.776 dag-scheduler-event-loop DEBUG DAGScheduler: ShuffleMapTask finished on driver 20/01/23 18:54:03.776 dag-scheduler-event-loop INFO DAGScheduler: ShuffleMapStage 9 (flatMap at ALS.scala:1674) finished in 0.196 s 20/01/23 18:54:03.776 dag-scheduler-event-loop INFO DAGScheduler: looking for newly runnable stages 20/01/23 18:54:03.776 dag-scheduler-event-loop INFO DAGScheduler: running: Set() 20/01/23 18:54:03.776 dag-scheduler-event-loop INFO DAGScheduler: waiting: Set(ResultStage 12, ShuffleMapStage 11) 20/01/23 18:54:03.776 dag-scheduler-event-loop INFO DAGScheduler: failed: Set() 20/01/23 18:54:03.776 dag-scheduler-event-loop DEBUG MapOutputTrackerMaster: Increasing epoch to 4 20/01/23 18:54:03.776 dag-scheduler-event-loop DEBUG DAGScheduler: submitStage(ShuffleMapStage 11 (name=flatMap at ALS.scala:1674;jobs=3)) 20/01/23 18:54:03.779 dag-scheduler-event-loop DEBUG DAGScheduler: missing: List() 20/01/23 18:54:03.779 dag-scheduler-event-loop INFO DAGScheduler: Submitting ShuffleMapStage 11 (MapPartitionsRDD[40] at flatMap at ALS.scala:1674), which has no missing parents 20/01/23 18:54:03.779 dag-scheduler-event-loop DEBUG DAGScheduler: submitMissingTasks(ShuffleMapStage 11) 20/01/23 18:54:03.779 Executor task launch worker for task 16 DEBUG BlockManager: Getting local block rdd_21_0 20/01/23 18:54:03.779 Executor task launch worker for task 16 DEBUG BlockManager: Level for block rdd_21_0 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:03.780 Executor task launch worker for task 16 INFO BlockManager: Found block rdd_21_0 locally 20/01/23 18:54:03.781 Executor task launch worker for task 18 DEBUG BlockManager: Getting local block rdd_21_2 20/01/23 18:54:03.781 Executor task launch worker for task 18 DEBUG BlockManager: Level for block rdd_21_2 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:03.781 Executor task launch worker for task 18 INFO BlockManager: Found block rdd_21_2 locally 20/01/23 18:54:03.783 Executor task launch worker for task 19 DEBUG BlockManager: Getting local block rdd_21_3 20/01/23 18:54:03.783 Executor task launch worker for task 17 DEBUG BlockManager: Getting local block rdd_21_1 20/01/23 18:54:03.783 Executor task launch worker for task 19 DEBUG BlockManager: Level for block rdd_21_3 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:03.783 Executor task launch worker for task 17 DEBUG BlockManager: Level for block rdd_21_1 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:03.783 Executor task launch worker for task 19 INFO BlockManager: Found block rdd_21_3 locally 20/01/23 18:54:03.783 Executor task launch worker for task 17 INFO BlockManager: Found block rdd_21_1 locally 20/01/23 18:54:03.785 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_7 stored as values in memory (estimated size 34.2 KiB, free 366.0 MiB) 20/01/23 18:54:03.785 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_7 locally took 0 ms 20/01/23 18:54:03.786 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_7 without replication took 0 ms 20/01/23 18:54:03.787 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_7_piece0 stored as bytes in memory (estimated size 15.7 KiB, free 366.0 MiB) 20/01/23 18:54:03.787 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added broadcast_7_piece0 in memory on 192.168.1.225:35863 (size: 15.7 KiB, free: 366.2 MiB) 20/01/23 18:54:03.788 dag-scheduler-event-loop DEBUG BlockManagerMaster: Updated info of block broadcast_7_piece0 20/01/23 18:54:03.788 dag-scheduler-event-loop DEBUG BlockManager: Told master about block broadcast_7_piece0 20/01/23 18:54:03.788 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_7_piece0 locally took 1 ms 20/01/23 18:54:03.788 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_7_piece0 without replication took 1 ms 20/01/23 18:54:03.788 dag-scheduler-event-loop INFO SparkContext: Created broadcast 7 from broadcast at DAGScheduler.scala:1200 20/01/23 18:54:03.789 dag-scheduler-event-loop INFO DAGScheduler: Submitting 10 missing tasks from ShuffleMapStage 11 (MapPartitionsRDD[40] at flatMap at ALS.scala:1674) (first 15 tasks are for partitions Vector(0, 1, 2, 3, 4, 5, 6, 7, 8, 9)) 20/01/23 18:54:03.789 dag-scheduler-event-loop INFO TaskSchedulerImpl: Adding task set 11.0 with 10 tasks 20/01/23 18:54:03.789 dag-scheduler-event-loop DEBUG TaskSetManager: Epoch for TaskSet 11.0: 4 20/01/23 18:54:03.789 dag-scheduler-event-loop DEBUG TaskSetManager: Adding pending tasks took 0 ms 20/01/23 18:54:03.789 dag-scheduler-event-loop DEBUG TaskSetManager: Valid locality levels for TaskSet 11.0: PROCESS_LOCAL, NODE_LOCAL, ANY 20/01/23 18:54:03.790 dispatcher-event-loop-1 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_11.0, runningTasks: 0 20/01/23 18:54:03.790 dispatcher-event-loop-1 INFO TaskSetManager: Starting task 0.0 in stage 11.0 (TID 43, 192.168.1.225, executor driver, partition 0, PROCESS_LOCAL, 7422 bytes) 20/01/23 18:54:03.791 dispatcher-event-loop-1 INFO TaskSetManager: Starting task 1.0 in stage 11.0 (TID 44, 192.168.1.225, executor driver, partition 1, PROCESS_LOCAL, 7422 bytes) 20/01/23 18:54:03.791 dispatcher-event-loop-1 INFO TaskSetManager: Starting task 2.0 in stage 11.0 (TID 45, 192.168.1.225, executor driver, partition 2, PROCESS_LOCAL, 7422 bytes) 20/01/23 18:54:03.791 dispatcher-event-loop-1 INFO TaskSetManager: Starting task 3.0 in stage 11.0 (TID 46, 192.168.1.225, executor driver, partition 3, PROCESS_LOCAL, 7422 bytes) 20/01/23 18:54:03.792 Executor task launch worker for task 43 INFO Executor: Running task 0.0 in stage 11.0 (TID 43) 20/01/23 18:54:03.792 Executor task launch worker for task 46 INFO Executor: Running task 3.0 in stage 11.0 (TID 46) 20/01/23 18:54:03.792 Executor task launch worker for task 43 DEBUG ExecutorMetricsPoller: stageTCMP: (11, 0) -> 1 20/01/23 18:54:03.792 Executor task launch worker for task 46 DEBUG ExecutorMetricsPoller: stageTCMP: (11, 0) -> 2 20/01/23 18:54:03.793 Executor task launch worker for task 43 DEBUG BlockManager: Getting local block broadcast_7 20/01/23 18:54:03.793 Executor task launch worker for task 43 DEBUG BlockManager: Level for block broadcast_7 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:03.797 Executor task launch worker for task 44 INFO Executor: Running task 1.0 in stage 11.0 (TID 44) 20/01/23 18:54:03.797 Executor task launch worker for task 44 DEBUG ExecutorMetricsPoller: stageTCMP: (11, 0) -> 3 20/01/23 18:54:03.803 Executor task launch worker for task 45 INFO Executor: Running task 2.0 in stage 11.0 (TID 45) 20/01/23 18:54:03.803 Executor task launch worker for task 45 DEBUG ExecutorMetricsPoller: stageTCMP: (11, 0) -> 4 20/01/23 18:54:03.819 Executor task launch worker for task 45 DEBUG BlockManager: Getting local block rdd_25_2 20/01/23 18:54:03.819 Executor task launch worker for task 45 DEBUG BlockManager: Level for block rdd_25_2 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:03.819 Executor task launch worker for task 45 INFO BlockManager: Found block rdd_25_2 locally 20/01/23 18:54:03.819 Executor task launch worker for task 45 DEBUG BlockManager: Getting local block rdd_24_2 20/01/23 18:54:03.819 Executor task launch worker for task 45 DEBUG BlockManager: Level for block rdd_24_2 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:03.820 Executor task launch worker for task 45 INFO BlockManager: Found block rdd_24_2 locally 20/01/23 18:54:03.820 Executor task launch worker for task 45 DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 4, partitions 2-3 20/01/23 18:54:03.820 Executor task launch worker for task 45 DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 20/01/23 18:54:03.821 Executor task launch worker for task 45 INFO ShuffleBlockFetcherIterator: Getting 3 (707.0 B) non-empty blocks including 3 (707.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 20/01/23 18:54:03.821 Executor task launch worker for task 45 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 20/01/23 18:54:03.821 Executor task launch worker for task 45 DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_4_33_2,0), (shuffle_4_34_2,1), (shuffle_4_35_2,2) 20/01/23 18:54:03.821 Executor task launch worker for task 45 DEBUG ShuffleBlockFetcherIterator: Got local blocks in 1 ms 20/01/23 18:54:03.823 Executor task launch worker for task 45 DEBUG TaskMemoryManager: Task 45 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@1c4d6ae0 20/01/23 18:54:03.827 Executor task launch worker for task 44 DEBUG BlockManager: Getting local block rdd_25_1 20/01/23 18:54:03.827 Executor task launch worker for task 44 DEBUG BlockManager: Level for block rdd_25_1 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:03.827 Executor task launch worker for task 44 INFO BlockManager: Found block rdd_25_1 locally 20/01/23 18:54:03.827 Executor task launch worker for task 44 DEBUG BlockManager: Getting local block rdd_24_1 20/01/23 18:54:03.827 Executor task launch worker for task 46 DEBUG BlockManager: Getting local block rdd_25_3 20/01/23 18:54:03.827 Executor task launch worker for task 46 DEBUG BlockManager: Level for block rdd_25_3 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:03.827 Executor task launch worker for task 46 INFO BlockManager: Found block rdd_25_3 locally 20/01/23 18:54:03.827 Executor task launch worker for task 46 DEBUG BlockManager: Getting local block rdd_24_3 20/01/23 18:54:03.827 Executor task launch worker for task 46 DEBUG BlockManager: Level for block rdd_24_3 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:03.827 Executor task launch worker for task 46 INFO BlockManager: Found block rdd_24_3 locally 20/01/23 18:54:03.828 Executor task launch worker for task 46 DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 4, partitions 3-4 20/01/23 18:54:03.828 Executor task launch worker for task 46 DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 20/01/23 18:54:03.828 Executor task launch worker for task 46 INFO ShuffleBlockFetcherIterator: Getting 3 (684.0 B) non-empty blocks including 3 (684.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 20/01/23 18:54:03.829 Executor task launch worker for task 46 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 20/01/23 18:54:03.829 Executor task launch worker for task 46 DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_4_33_3,0), (shuffle_4_34_3,1), (shuffle_4_35_3,2) 20/01/23 18:54:03.829 Executor task launch worker for task 43 DEBUG BlockManager: Getting local block rdd_25_0 20/01/23 18:54:03.829 Executor task launch worker for task 43 DEBUG BlockManager: Level for block rdd_25_0 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:03.829 Executor task launch worker for task 43 INFO BlockManager: Found block rdd_25_0 locally 20/01/23 18:54:03.829 Executor task launch worker for task 43 DEBUG BlockManager: Getting local block rdd_24_0 20/01/23 18:54:03.829 Executor task launch worker for task 43 DEBUG BlockManager: Level for block rdd_24_0 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:03.829 Executor task launch worker for task 46 DEBUG ShuffleBlockFetcherIterator: Got local blocks in 1 ms 20/01/23 18:54:03.827 Executor task launch worker for task 44 DEBUG BlockManager: Level for block rdd_24_1 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:03.831 Executor task launch worker for task 44 INFO BlockManager: Found block rdd_24_1 locally 20/01/23 18:54:03.831 Executor task launch worker for task 44 DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 4, partitions 1-2 20/01/23 18:54:03.829 Executor task launch worker for task 43 INFO BlockManager: Found block rdd_24_0 locally 20/01/23 18:54:03.831 Executor task launch worker for task 43 DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 4, partitions 0-1 20/01/23 18:54:03.832 Executor task launch worker for task 44 DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 20/01/23 18:54:03.832 Executor task launch worker for task 43 DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 20/01/23 18:54:03.832 Executor task launch worker for task 44 INFO ShuffleBlockFetcherIterator: Getting 3 (730.0 B) non-empty blocks including 3 (730.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 20/01/23 18:54:03.832 Executor task launch worker for task 44 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 20/01/23 18:54:03.832 Executor task launch worker for task 44 DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_4_33_1,0), (shuffle_4_34_1,1), (shuffle_4_35_1,2) 20/01/23 18:54:03.833 Executor task launch worker for task 46 DEBUG TaskMemoryManager: Task 46 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@737791f0 20/01/23 18:54:03.832 Executor task launch worker for task 43 INFO ShuffleBlockFetcherIterator: Getting 3 (684.0 B) non-empty blocks including 3 (684.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 20/01/23 18:54:03.834 Executor task launch worker for task 43 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 2 ms 20/01/23 18:54:03.834 Executor task launch worker for task 46 DEBUG TaskMemoryManager: Task 46 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@65528da2 20/01/23 18:54:03.834 Executor task launch worker for task 46 DEBUG TaskMemoryManager: Task 46 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@2ad7d35b 20/01/23 18:54:03.834 Executor task launch worker for task 43 DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_4_33_0,0), (shuffle_4_34_0,1), (shuffle_4_35_0,2) 20/01/23 18:54:03.835 Executor task launch worker for task 46 INFO Executor: Finished task 3.0 in stage 11.0 (TID 46). 2303 bytes result sent to driver 20/01/23 18:54:03.836 Executor task launch worker for task 46 DEBUG ExecutorMetricsPoller: stageTCMP: (11, 0) -> 3 20/01/23 18:54:03.836 Executor task launch worker for task 44 DEBUG ShuffleBlockFetcherIterator: Got local blocks in 4 ms 20/01/23 18:54:03.836 dispatcher-event-loop-2 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_11.0, runningTasks: 3 20/01/23 18:54:03.836 Executor task launch worker for task 43 DEBUG ShuffleBlockFetcherIterator: Got local blocks in 4 ms 20/01/23 18:54:03.836 dispatcher-event-loop-2 INFO TaskSetManager: Starting task 4.0 in stage 11.0 (TID 47, 192.168.1.225, executor driver, partition 4, PROCESS_LOCAL, 7422 bytes) 20/01/23 18:54:03.837 task-result-getter-0 INFO TaskSetManager: Finished task 3.0 in stage 11.0 (TID 46) in 46 ms on 192.168.1.225 (executor driver) (1/10) 20/01/23 18:54:03.838 dag-scheduler-event-loop DEBUG DAGScheduler: ShuffleMapTask finished on driver 20/01/23 18:54:03.838 Executor task launch worker for task 44 DEBUG TaskMemoryManager: Task 44 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@5993bd0a 20/01/23 18:54:03.838 Executor task launch worker for task 43 DEBUG TaskMemoryManager: Task 43 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@2e15854a 20/01/23 18:54:03.839 Executor task launch worker for task 47 INFO Executor: Running task 4.0 in stage 11.0 (TID 47) 20/01/23 18:54:03.839 Executor task launch worker for task 47 DEBUG ExecutorMetricsPoller: stageTCMP: (11, 0) -> 4 20/01/23 18:54:03.841 Executor task launch worker for task 19 INFO Executor: Finished task 3.0 in stage 5.0 (TID 19). 2643 bytes result sent to driver 20/01/23 18:54:03.841 Executor task launch worker for task 19 DEBUG ExecutorMetricsPoller: stageTCMP: (5, 0) -> 3 20/01/23 18:54:03.842 Executor task launch worker for task 18 INFO Executor: Finished task 2.0 in stage 5.0 (TID 18). 2643 bytes result sent to driver 20/01/23 18:54:03.842 Executor task launch worker for task 18 DEBUG ExecutorMetricsPoller: stageTCMP: (5, 0) -> 2 20/01/23 18:54:03.842 dispatcher-event-loop-2 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_5.0, runningTasks: 3 20/01/23 18:54:03.841 Executor task launch worker for task 17 INFO Executor: Finished task 1.0 in stage 5.0 (TID 17). 2643 bytes result sent to driver 20/01/23 18:54:03.842 Executor task launch worker for task 17 DEBUG ExecutorMetricsPoller: stageTCMP: (5, 0) -> 1 20/01/23 18:54:03.842 dispatcher-event-loop-2 DEBUG TaskSetManager: No tasks for locality level PROCESS_LOCAL, so moving to locality level NODE_LOCAL 20/01/23 18:54:03.842 dispatcher-event-loop-2 DEBUG TaskSetManager: No tasks for locality level NODE_LOCAL, so moving to locality level ANY 20/01/23 18:54:03.843 dispatcher-event-loop-2 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_5.0, runningTasks: 2 20/01/23 18:54:03.843 dispatcher-event-loop-2 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_5.0, runningTasks: 1 20/01/23 18:54:03.844 task-result-getter-1 INFO TaskSetManager: Finished task 3.0 in stage 5.0 (TID 19) in 74 ms on 192.168.1.225 (executor driver) (1/4) 20/01/23 18:54:03.846 task-result-getter-0 INFO TaskSetManager: Finished task 2.0 in stage 5.0 (TID 18) in 77 ms on 192.168.1.225 (executor driver) (2/4) 20/01/23 18:54:03.847 task-result-getter-2 INFO TaskSetManager: Finished task 1.0 in stage 5.0 (TID 17) in 78 ms on 192.168.1.225 (executor driver) (3/4) 20/01/23 18:54:03.850 Executor task launch worker for task 16 INFO Executor: Finished task 0.0 in stage 5.0 (TID 16). 2426 bytes result sent to driver 20/01/23 18:54:03.852 Executor task launch worker for task 16 DEBUG ExecutorMetricsPoller: removing (5, 0) from stageTCMP 20/01/23 18:54:03.852 dispatcher-event-loop-3 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_5.0, runningTasks: 0 20/01/23 18:54:03.854 task-result-getter-3 INFO TaskSetManager: Finished task 0.0 in stage 5.0 (TID 16) in 86 ms on 192.168.1.225 (executor driver) (4/4) 20/01/23 18:54:03.854 task-result-getter-3 INFO TaskSchedulerImpl: Removed TaskSet 5.0, whose tasks have all completed, from pool 20/01/23 18:54:03.854 dag-scheduler-event-loop INFO DAGScheduler: ResultStage 5 (top at CountVectorizer.scala:236) finished in 0.130 s 20/01/23 18:54:03.855 dag-scheduler-event-loop DEBUG DAGScheduler: After removal of stage 5, remaining stages = 1 20/01/23 18:54:03.855 dag-scheduler-event-loop DEBUG DAGScheduler: After removal of stage 4, remaining stages = 0 20/01/23 18:54:03.855 dag-scheduler-event-loop INFO DAGScheduler: Job 3 is finished. Cancelling potential speculative or zombie tasks for this job 20/01/23 18:54:03.855 dag-scheduler-event-loop INFO TaskSchedulerImpl: Killing all running tasks in stage 5: Stage finished 20/01/23 18:54:03.856 Thread-4 INFO DAGScheduler: Job 3 finished: top at CountVectorizer.scala:236, took 0.136314 s 20/01/23 18:54:03.856 Executor task launch worker for task 47 DEBUG BlockManager: Getting local block rdd_25_4 20/01/23 18:54:03.856 Executor task launch worker for task 47 DEBUG BlockManager: Level for block rdd_25_4 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:03.856 Executor task launch worker for task 47 INFO BlockManager: Found block rdd_25_4 locally 20/01/23 18:54:03.856 Executor task launch worker for task 47 DEBUG BlockManager: Getting local block rdd_24_4 20/01/23 18:54:03.856 Executor task launch worker for task 47 DEBUG BlockManager: Level for block rdd_24_4 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:03.856 Executor task launch worker for task 47 INFO BlockManager: Found block rdd_24_4 locally 20/01/23 18:54:03.856 Executor task launch worker for task 47 DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 4, partitions 4-5 20/01/23 18:54:03.857 Executor task launch worker for task 47 DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 20/01/23 18:54:03.857 Executor task launch worker for task 47 INFO ShuffleBlockFetcherIterator: Getting 3 (684.0 B) non-empty blocks including 3 (684.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 20/01/23 18:54:03.857 Executor task launch worker for task 47 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 20/01/23 18:54:03.857 Executor task launch worker for task 47 DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_4_33_4,0), (shuffle_4_34_4,1), (shuffle_4_35_4,2) 20/01/23 18:54:03.857 Thread-4 INFO MapPartitionsRDD: Removing RDD 21 from persistence list 20/01/23 18:54:03.858 Executor task launch worker for task 47 DEBUG ShuffleBlockFetcherIterator: Got local blocks in 1 ms 20/01/23 18:54:03.869 Executor task launch worker for task 47 DEBUG TaskMemoryManager: Task 47 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@2cd6a07f 20/01/23 18:54:03.869 Executor task launch worker for task 47 DEBUG TaskMemoryManager: Task 47 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@618aa1a5 20/01/23 18:54:03.871 Executor task launch worker for task 12 DEBUG BlockManager: Getting local block broadcast_3 20/01/23 18:54:03.871 Executor task launch worker for task 12 DEBUG BlockManager: Level for block broadcast_3 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:03.881 Executor task launch worker for task 9 INFO PythonRunner: Times: total = 71, boot = -1606, init = 1677, finish = 0 20/01/23 18:54:03.882 Executor task launch worker for task 47 DEBUG TaskMemoryManager: Task 47 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@267041e6 20/01/23 18:54:03.881 Executor task launch worker for task 11 INFO PythonRunner: Times: total = 18, boot = -1595, init = 1613, finish = 0 20/01/23 18:54:03.884 Executor task launch worker for task 12 INFO PythonRunner: Times: total = 56, boot = -1580, init = 1635, finish = 1 20/01/23 18:54:03.885 block-manager-slave-async-thread-pool-6 DEBUG BlockManagerSlaveEndpoint: removing RDD 21 20/01/23 18:54:03.885 Executor task launch worker for task 10 INFO PythonRunner: Times: total = 106, boot = -1607, init = 1713, finish = 0 20/01/23 18:54:03.886 block-manager-slave-async-thread-pool-6 INFO BlockManager: Removing RDD 21 20/01/23 18:54:03.887 block-manager-slave-async-thread-pool-6 DEBUG BlockManager: Removing block rdd_21_1 20/01/23 18:54:03.887 block-manager-slave-async-thread-pool-6 DEBUG MemoryStore: Block rdd_21_1 of size 120 dropped from memory (free 384042168) 20/01/23 18:54:03.888 block-manager-slave-async-thread-pool-6 DEBUG BlockManager: Removing block rdd_21_0 20/01/23 18:54:03.888 block-manager-slave-async-thread-pool-6 DEBUG MemoryStore: Block rdd_21_0 of size 16 dropped from memory (free 384042184) 20/01/23 18:54:03.888 block-manager-slave-async-thread-pool-6 DEBUG BlockManager: Removing block rdd_21_3 20/01/23 18:54:03.888 block-manager-slave-async-thread-pool-6 DEBUG MemoryStore: Block rdd_21_3 of size 120 dropped from memory (free 384042304) 20/01/23 18:54:03.888 block-manager-slave-async-thread-pool-6 DEBUG BlockManager: Removing block rdd_21_2 20/01/23 18:54:03.888 block-manager-slave-async-thread-pool-6 DEBUG MemoryStore: Block rdd_21_2 of size 120 dropped from memory (free 384042424) 20/01/23 18:54:03.895 Thread-4 DEBUG CountVectorizerModel: Input schema: {"type":"struct","fields":[{"name":"id","type":"long","nullable":true,"metadata":{}},{"name":"words","type":{"type":"array","elementType":"string","containsNull":true},"nullable":true,"metadata":{}},{"name":"expected","type":{"type":"udt","class":"org.apache.spark.ml.linalg.VectorUDT","pyClass":"pyspark.ml.linalg.VectorUDT","sqlType":{"type":"struct","fields":[{"name":"type","type":"byte","nullable":false,"metadata":{}},{"name":"size","type":"integer","nullable":true,"metadata":{}},{"name":"indices","type":{"type":"array","elementType":"integer","containsNull":false},"nullable":true,"metadata":{}},{"name":"values","type":{"type":"array","elementType":"double","containsNull":false},"nullable":true,"metadata":{}}]}},"nullable":true,"metadata":{}}]} 20/01/23 18:54:03.896 Thread-4 DEBUG CountVectorizerModel: Expected output schema: {"type":"struct","fields":[{"name":"id","type":"long","nullable":true,"metadata":{}},{"name":"words","type":{"type":"array","elementType":"string","containsNull":true},"nullable":true,"metadata":{}},{"name":"expected","type":{"type":"udt","class":"org.apache.spark.ml.linalg.VectorUDT","pyClass":"pyspark.ml.linalg.VectorUDT","sqlType":{"type":"struct","fields":[{"name":"type","type":"byte","nullable":false,"metadata":{}},{"name":"size","type":"integer","nullable":true,"metadata":{}},{"name":"indices","type":{"type":"array","elementType":"integer","containsNull":false},"nullable":true,"metadata":{}},{"name":"values","type":{"type":"array","elementType":"double","containsNull":false},"nullable":true,"metadata":{}}]}},"nullable":true,"metadata":{}},{"name":"features","type":{"type":"udt","class":"org.apache.spark.ml.linalg.VectorUDT","pyClass":"pyspark.ml.linalg.VectorUDT","sqlType":{"type":"struct","fields":[{"name":"type","type":"byte","nullable":false,"metadata":{}},{"name":"size","type":"integer","nullable":true,"metadata":{}},{"name":"indices","type":{"type":"array","elementType":"integer","containsNull":false},"nullable":true,"metadata":{}},{"name":"values","type":{"type":"array","elementType":"double","containsNull":false},"nullable":true,"metadata":{}}]}},"nullable":false,"metadata":{"ml_attr":{"attrs":{},"num_attrs":3}}}]} 20/01/23 18:54:03.897 Thread-4 INFO MemoryStore: Block broadcast_7 stored as values in memory (estimated size 256.0 B, free 366.3 MiB) 20/01/23 18:54:03.898 Thread-4 DEBUG BlockManager: Put block broadcast_7 locally took 1 ms 20/01/23 18:54:03.898 Thread-4 DEBUG BlockManager: Putting block broadcast_7 without replication took 1 ms 20/01/23 18:54:03.899 Thread-4 INFO MemoryStore: Block broadcast_7_piece0 stored as bytes in memory (estimated size 264.0 B, free 366.3 MiB) 20/01/23 18:54:03.903 Executor task launch worker for task 9 INFO Executor: Finished task 0.0 in stage 3.0 (TID 9). 2710 bytes result sent to driver 20/01/23 18:54:03.903 Executor task launch worker for task 10 INFO Executor: Finished task 1.0 in stage 3.0 (TID 10). 2710 bytes result sent to driver 20/01/23 18:54:03.903 Executor task launch worker for task 9 DEBUG ExecutorMetricsPoller: stageTCMP: (3, 0) -> 3 20/01/23 18:54:03.904 Executor task launch worker for task 10 DEBUG ExecutorMetricsPoller: stageTCMP: (3, 0) -> 2 20/01/23 18:54:03.904 dispatcher-event-loop-1 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_3.0, runningTasks: 3 20/01/23 18:54:03.904 dispatcher-event-loop-1 DEBUG TaskSetManager: No tasks for locality level NO_PREF, so moving to locality level ANY 20/01/23 18:54:03.905 dispatcher-event-loop-1 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_3.0, runningTasks: 2 20/01/23 18:54:03.905 task-result-getter-1 INFO TaskSetManager: Finished task 0.0 in stage 3.0 (TID 9) in 212 ms on 192.168.1.225 (executor driver) (1/4) 20/01/23 18:54:03.906 Executor task launch worker for task 11 INFO Executor: Finished task 2.0 in stage 3.0 (TID 11). 2710 bytes result sent to driver 20/01/23 18:54:03.906 task-result-getter-0 INFO TaskSetManager: Finished task 1.0 in stage 3.0 (TID 10) in 211 ms on 192.168.1.225 (executor driver) (2/4) 20/01/23 18:54:03.906 dispatcher-event-loop-0 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_3.0, runningTasks: 1 20/01/23 18:54:03.907 Executor task launch worker for task 11 DEBUG ExecutorMetricsPoller: stageTCMP: (3, 0) -> 1 20/01/23 18:54:03.907 task-result-getter-3 INFO TaskSetManager: Finished task 2.0 in stage 3.0 (TID 11) in 213 ms on 192.168.1.225 (executor driver) (3/4) 20/01/23 18:54:03.908 Executor task launch worker for task 12 INFO Executor: Finished task 3.0 in stage 3.0 (TID 12). 2710 bytes result sent to driver 20/01/23 18:54:03.908 dispatcher-event-loop-2 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_3.0, runningTasks: 0 20/01/23 18:54:03.909 task-result-getter-2 INFO TaskSetManager: Finished task 3.0 in stage 3.0 (TID 12) in 214 ms on 192.168.1.225 (executor driver) (4/4) 20/01/23 18:54:03.909 task-result-getter-2 INFO TaskSchedulerImpl: Removed TaskSet 3.0, whose tasks have all completed, from pool 20/01/23 18:54:03.909 Executor task launch worker for task 12 DEBUG ExecutorMetricsPoller: removing (3, 0) from stageTCMP 20/01/23 18:54:03.913 Executor task launch worker for task 45 WARN LAPACK: Failed to load implementation from: com.github.fommil.netlib.NativeSystemLAPACK 20/01/23 18:54:03.914 Executor task launch worker for task 47 INFO Executor: Finished task 4.0 in stage 11.0 (TID 47). 2303 bytes result sent to driver 20/01/23 18:54:03.914 Executor task launch worker for task 47 DEBUG ExecutorMetricsPoller: stageTCMP: (11, 0) -> 3 20/01/23 18:54:03.914 Executor task launch worker for task 45 WARN LAPACK: Failed to load implementation from: com.github.fommil.netlib.NativeRefLAPACK 20/01/23 18:54:03.900 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added broadcast_7_piece0 in memory on 192.168.1.225:41233 (size: 264.0 B, free: 366.3 MiB) 20/01/23 18:54:03.914 dispatcher-event-loop-0 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_11.0, runningTasks: 3 20/01/23 18:54:03.915 Thread-4 DEBUG BlockManagerMaster: Updated info of block broadcast_7_piece0 20/01/23 18:54:03.915 Thread-4 DEBUG BlockManager: Told master about block broadcast_7_piece0 20/01/23 18:54:03.915 Thread-4 DEBUG BlockManager: Put block broadcast_7_piece0 locally took 15 ms 20/01/23 18:54:03.915 Thread-4 DEBUG BlockManager: Putting block broadcast_7_piece0 without replication took 16 ms 20/01/23 18:54:03.915 dispatcher-event-loop-0 INFO TaskSetManager: Starting task 5.0 in stage 11.0 (TID 48, 192.168.1.225, executor driver, partition 5, PROCESS_LOCAL, 7422 bytes) 20/01/23 18:54:03.915 Thread-4 INFO SparkContext: Created broadcast 7 from broadcast at CountVectorizer.scala:306 20/01/23 18:54:03.915 Executor task launch worker for task 48 INFO Executor: Running task 5.0 in stage 11.0 (TID 48) 20/01/23 18:54:03.916 Executor task launch worker for task 48 DEBUG ExecutorMetricsPoller: stageTCMP: (11, 0) -> 4 20/01/23 18:54:03.920 block-manager-slave-async-thread-pool-8 DEBUG BlockManagerSlaveEndpoint: Done removing RDD 21, response is 0 20/01/23 18:54:03.921 task-result-getter-3 INFO TaskSetManager: Finished task 4.0 in stage 11.0 (TID 47) in 85 ms on 192.168.1.225 (executor driver) (2/10) 20/01/23 18:54:03.922 dag-scheduler-event-loop DEBUG DAGScheduler: ShuffleMapTask finished on driver 20/01/23 18:54:03.923 Thread-4 DEBUG Analyzer$ResolveReferences: Resolving 'words to words#22 20/01/23 18:54:03.925 Executor task launch worker for task 48 DEBUG BlockManager: Getting local block rdd_25_5 20/01/23 18:54:03.926 Executor task launch worker for task 48 DEBUG BlockManager: Level for block rdd_25_5 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:03.926 Executor task launch worker for task 48 INFO BlockManager: Found block rdd_25_5 locally 20/01/23 18:54:03.926 Executor task launch worker for task 48 DEBUG BlockManager: Getting local block rdd_24_5 20/01/23 18:54:03.926 Executor task launch worker for task 48 DEBUG BlockManager: Level for block rdd_24_5 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:03.926 Executor task launch worker for task 48 INFO BlockManager: Found block rdd_24_5 locally 20/01/23 18:54:03.926 Executor task launch worker for task 48 DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 4, partitions 5-6 20/01/23 18:54:03.926 Executor task launch worker for task 48 DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 20/01/23 18:54:03.927 Executor task launch worker for task 48 INFO ShuffleBlockFetcherIterator: Getting 3 (684.0 B) non-empty blocks including 3 (684.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 20/01/23 18:54:03.927 Executor task launch worker for task 48 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 20/01/23 18:54:03.927 Executor task launch worker for task 48 DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_4_33_5,0), (shuffle_4_34_5,1), (shuffle_4_35_5,2) 20/01/23 18:54:03.927 Executor task launch worker for task 48 DEBUG ShuffleBlockFetcherIterator: Got local blocks in 1 ms 20/01/23 18:54:03.931 Executor task launch worker for task 48 DEBUG TaskMemoryManager: Task 48 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@a0552fa 20/01/23 18:54:03.931 Executor task launch worker for task 48 DEBUG TaskMemoryManager: Task 48 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@7b4bbc88 20/01/23 18:54:03.932 Executor task launch worker for task 48 DEBUG TaskMemoryManager: Task 48 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@2a98206f 20/01/23 18:54:03.933 Executor task launch worker for task 48 INFO Executor: Finished task 5.0 in stage 11.0 (TID 48). 2303 bytes result sent to driver 20/01/23 18:54:03.933 dag-scheduler-event-loop DEBUG DAGScheduler: ShuffleMapTask finished on driver 20/01/23 18:54:03.935 dag-scheduler-event-loop DEBUG DAGScheduler: ShuffleMapTask finished on driver 20/01/23 18:54:03.935 dispatcher-event-loop-2 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_11.0, runningTasks: 3 20/01/23 18:54:03.936 dispatcher-event-loop-2 INFO TaskSetManager: Starting task 6.0 in stage 11.0 (TID 49, 192.168.1.225, executor driver, partition 6, PROCESS_LOCAL, 7422 bytes) 20/01/23 18:54:03.936 Executor task launch worker for task 49 INFO Executor: Running task 6.0 in stage 11.0 (TID 49) 20/01/23 18:54:03.936 dag-scheduler-event-loop DEBUG DAGScheduler: ShuffleMapTask finished on driver 20/01/23 18:54:03.937 task-result-getter-2 INFO TaskSetManager: Finished task 5.0 in stage 11.0 (TID 48) in 22 ms on 192.168.1.225 (executor driver) (3/10) 20/01/23 18:54:03.937 Executor task launch worker for task 49 DEBUG ExecutorMetricsPoller: stageTCMP: (11, 0) -> 5 20/01/23 18:54:03.937 block-manager-slave-async-thread-pool-8 DEBUG BlockManagerSlaveEndpoint: Sent response: 0 to 192.168.1.225:42003 20/01/23 18:54:03.937 dag-scheduler-event-loop DEBUG DAGScheduler: ShuffleMapTask finished on driver 20/01/23 18:54:03.938 dag-scheduler-event-loop INFO DAGScheduler: ShuffleMapStage 3 (collect at ClusteringEvaluator.scala:179) finished in 0.290 s 20/01/23 18:54:03.938 dag-scheduler-event-loop DEBUG DAGScheduler: ShuffleMapTask finished on driver 20/01/23 18:54:03.938 dag-scheduler-event-loop INFO DAGScheduler: looking for newly runnable stages 20/01/23 18:54:03.938 dag-scheduler-event-loop INFO DAGScheduler: running: Set() 20/01/23 18:54:03.938 dag-scheduler-event-loop INFO DAGScheduler: waiting: Set(ResultStage 4) 20/01/23 18:54:03.938 dag-scheduler-event-loop INFO DAGScheduler: failed: Set() 20/01/23 18:54:03.938 dag-scheduler-event-loop DEBUG MapOutputTrackerMaster: Increasing epoch to 2 20/01/23 18:54:03.938 dag-scheduler-event-loop DEBUG DAGScheduler: submitStage(ResultStage 4 (name=collect at ClusteringEvaluator.scala:179;jobs=2)) 20/01/23 18:54:03.938 dag-scheduler-event-loop DEBUG DAGScheduler: missing: List() 20/01/23 18:54:03.939 dag-scheduler-event-loop INFO DAGScheduler: Submitting ResultStage 4 (MapPartitionsRDD[17] at collect at ClusteringEvaluator.scala:179), which has no missing parents 20/01/23 18:54:03.939 dag-scheduler-event-loop DEBUG DAGScheduler: submitMissingTasks(ResultStage 4) 20/01/23 18:54:03.943 Executor task launch worker for task 49 DEBUG BlockManager: Getting local block rdd_25_6 20/01/23 18:54:03.943 Executor task launch worker for task 49 DEBUG BlockManager: Level for block rdd_25_6 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:03.943 Executor task launch worker for task 49 INFO BlockManager: Found block rdd_25_6 locally 20/01/23 18:54:03.943 Executor task launch worker for task 49 DEBUG BlockManager: Getting local block rdd_24_6 20/01/23 18:54:03.943 Executor task launch worker for task 49 DEBUG BlockManager: Level for block rdd_24_6 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:03.943 Executor task launch worker for task 49 INFO BlockManager: Found block rdd_24_6 locally 20/01/23 18:54:03.943 Executor task launch worker for task 49 DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 4, partitions 6-7 20/01/23 18:54:03.944 Executor task launch worker for task 49 DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 20/01/23 18:54:03.944 Executor task launch worker for task 49 INFO ShuffleBlockFetcherIterator: Getting 3 (684.0 B) non-empty blocks including 3 (684.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 20/01/23 18:54:03.944 Executor task launch worker for task 49 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 20/01/23 18:54:03.944 Executor task launch worker for task 49 DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_4_33_6,0), (shuffle_4_34_6,1), (shuffle_4_35_6,2) 20/01/23 18:54:03.945 Executor task launch worker for task 49 DEBUG ShuffleBlockFetcherIterator: Got local blocks in 1 ms 20/01/23 18:54:03.946 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_5 stored as values in memory (estimated size 11.8 KiB, free 366.2 MiB) 20/01/23 18:54:03.947 Executor task launch worker for task 49 DEBUG TaskMemoryManager: Task 49 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@6fd05413 20/01/23 18:54:03.948 Executor task launch worker for task 49 DEBUG TaskMemoryManager: Task 49 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@77bb8151 20/01/23 18:54:03.948 Executor task launch worker for task 49 DEBUG TaskMemoryManager: Task 49 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@4a286e8c 20/01/23 18:54:03.949 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_5 locally took 1 ms 20/01/23 18:54:03.949 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_5 without replication took 4 ms 20/01/23 18:54:03.949 Executor task launch worker for task 49 INFO Executor: Finished task 6.0 in stage 11.0 (TID 49). 2303 bytes result sent to driver 20/01/23 18:54:03.950 Executor task launch worker for task 48 DEBUG ExecutorMetricsPoller: stageTCMP: (11, 0) -> 4 20/01/23 18:54:03.950 dispatcher-event-loop-0 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_11.0, runningTasks: 3 20/01/23 18:54:03.950 dispatcher-event-loop-0 INFO TaskSetManager: Starting task 7.0 in stage 11.0 (TID 50, 192.168.1.225, executor driver, partition 7, PROCESS_LOCAL, 7422 bytes) 20/01/23 18:54:03.951 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_5_piece0 stored as bytes in memory (estimated size 5.6 KiB, free 366.2 MiB) 20/01/23 18:54:03.951 task-result-getter-1 INFO TaskSetManager: Finished task 6.0 in stage 11.0 (TID 49) in 15 ms on 192.168.1.225 (executor driver) (4/10) 20/01/23 18:54:03.951 Executor task launch worker for task 50 INFO Executor: Running task 7.0 in stage 11.0 (TID 50) 20/01/23 18:54:03.952 Executor task launch worker for task 49 DEBUG ExecutorMetricsPoller: stageTCMP: (11, 0) -> 3 20/01/23 18:54:03.952 Executor task launch worker for task 50 DEBUG ExecutorMetricsPoller: stageTCMP: (11, 0) -> 4 20/01/23 18:54:03.952 dag-scheduler-event-loop DEBUG DAGScheduler: ShuffleMapTask finished on driver 20/01/23 18:54:03.954 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added broadcast_5_piece0 in memory on 192.168.1.225:45461 (size: 5.6 KiB, free: 366.3 MiB) 20/01/23 18:54:03.954 dag-scheduler-event-loop DEBUG BlockManagerMaster: Updated info of block broadcast_5_piece0 20/01/23 18:54:03.954 dag-scheduler-event-loop DEBUG BlockManager: Told master about block broadcast_5_piece0 20/01/23 18:54:03.955 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_5_piece0 locally took 4 ms 20/01/23 18:54:03.955 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_5_piece0 without replication took 4 ms 20/01/23 18:54:03.955 dag-scheduler-event-loop INFO SparkContext: Created broadcast 5 from broadcast at DAGScheduler.scala:1200 20/01/23 18:54:03.956 dag-scheduler-event-loop INFO DAGScheduler: Submitting 1 missing tasks from ResultStage 4 (MapPartitionsRDD[17] at collect at ClusteringEvaluator.scala:179) (first 15 tasks are for partitions Vector(0)) 20/01/23 18:54:03.956 dag-scheduler-event-loop INFO TaskSchedulerImpl: Adding task set 4.0 with 1 tasks 20/01/23 18:54:03.956 dag-scheduler-event-loop DEBUG TaskSetManager: Epoch for TaskSet 4.0: 2 20/01/23 18:54:03.956 dag-scheduler-event-loop DEBUG TaskSetManager: Adding pending tasks took 0 ms 20/01/23 18:54:03.956 dag-scheduler-event-loop DEBUG TaskSetManager: Valid locality levels for TaskSet 4.0: NODE_LOCAL, ANY 20/01/23 18:54:03.957 dispatcher-event-loop-3 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_4.0, runningTasks: 0 20/01/23 18:54:03.959 Executor task launch worker for task 50 DEBUG BlockManager: Getting local block rdd_25_7 20/01/23 18:54:03.959 dispatcher-event-loop-3 INFO TaskSetManager: Starting task 0.0 in stage 4.0 (TID 13, 192.168.1.225, executor driver, partition 0, NODE_LOCAL, 7248 bytes) 20/01/23 18:54:03.959 Executor task launch worker for task 50 DEBUG BlockManager: Level for block rdd_25_7 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:03.959 Executor task launch worker for task 50 INFO BlockManager: Found block rdd_25_7 locally 20/01/23 18:54:03.959 Executor task launch worker for task 50 DEBUG BlockManager: Getting local block rdd_24_7 20/01/23 18:54:03.959 Executor task launch worker for task 50 DEBUG BlockManager: Level for block rdd_24_7 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:03.959 Executor task launch worker for task 50 INFO BlockManager: Found block rdd_24_7 locally 20/01/23 18:54:03.959 dispatcher-event-loop-3 DEBUG TaskSetManager: No tasks for locality level NODE_LOCAL, so moving to locality level ANY 20/01/23 18:54:03.959 Executor task launch worker for task 50 DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 4, partitions 7-8 20/01/23 18:54:03.959 Executor task launch worker for task 13 INFO Executor: Running task 0.0 in stage 4.0 (TID 13) 20/01/23 18:54:03.959 Executor task launch worker for task 50 DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 20/01/23 18:54:03.960 Executor task launch worker for task 13 DEBUG ExecutorMetricsPoller: stageTCMP: (4, 0) -> 1 20/01/23 18:54:03.961 Executor task launch worker for task 13 DEBUG BlockManager: Getting local block broadcast_5 20/01/23 18:54:03.961 Executor task launch worker for task 13 DEBUG BlockManager: Level for block broadcast_5 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:03.963 Executor task launch worker for task 13 DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 1, partitions 0-1 20/01/23 18:54:03.964 Executor task launch worker for task 13 DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 20/01/23 18:54:03.964 Executor task launch worker for task 13 INFO ShuffleBlockFetcherIterator: Getting 4 (282.0 B) non-empty blocks including 4 (282.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 20/01/23 18:54:03.965 Executor task launch worker for task 13 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 20/01/23 18:54:03.965 Executor task launch worker for task 13 DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_1_9_0,0), (shuffle_1_10_0,1), (shuffle_1_11_0,2), (shuffle_1_12_0,3) 20/01/23 18:54:03.965 Executor task launch worker for task 13 DEBUG ShuffleBlockFetcherIterator: Got local blocks in 1 ms 20/01/23 18:54:03.969 Executor task launch worker for task 50 INFO ShuffleBlockFetcherIterator: Getting 3 (684.0 B) non-empty blocks including 3 (684.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 20/01/23 18:54:03.969 Executor task launch worker for task 50 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 10 ms 20/01/23 18:54:03.970 Executor task launch worker for task 50 DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_4_33_7,0), (shuffle_4_34_7,1), (shuffle_4_35_7,2) 20/01/23 18:54:03.970 Executor task launch worker for task 50 DEBUG ShuffleBlockFetcherIterator: Got local blocks in 10 ms 20/01/23 18:54:03.972 Executor task launch worker for task 50 DEBUG TaskMemoryManager: Task 50 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@60b15c8a 20/01/23 18:54:03.972 Executor task launch worker for task 50 DEBUG TaskMemoryManager: Task 50 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@1b697517 20/01/23 18:54:03.973 Executor task launch worker for task 50 DEBUG TaskMemoryManager: Task 50 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@26fe6671 20/01/23 18:54:03.975 Executor task launch worker for task 13 INFO Executor: Finished task 0.0 in stage 4.0 (TID 13). 3116 bytes result sent to driver 20/01/23 18:54:03.976 Executor task launch worker for task 13 DEBUG ExecutorMetricsPoller: removing (4, 0) from stageTCMP 20/01/23 18:54:03.976 dispatcher-event-loop-0 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_4.0, runningTasks: 0 20/01/23 18:54:03.977 task-result-getter-1 INFO TaskSetManager: Finished task 0.0 in stage 4.0 (TID 13) in 19 ms on 192.168.1.225 (executor driver) (1/1) 20/01/23 18:54:03.977 task-result-getter-1 INFO TaskSchedulerImpl: Removed TaskSet 4.0, whose tasks have all completed, from pool 20/01/23 18:54:03.979 dag-scheduler-event-loop INFO DAGScheduler: ResultStage 4 (collect at ClusteringEvaluator.scala:179) finished in 0.038 s 20/01/23 18:54:03.979 dag-scheduler-event-loop DEBUG DAGScheduler: After removal of stage 4, remaining stages = 1 20/01/23 18:54:03.979 dag-scheduler-event-loop DEBUG DAGScheduler: After removal of stage 3, remaining stages = 0 20/01/23 18:54:03.979 dag-scheduler-event-loop INFO DAGScheduler: Job 2 is finished. Cancelling potential speculative or zombie tasks for this job 20/01/23 18:54:03.979 dag-scheduler-event-loop INFO TaskSchedulerImpl: Killing all running tasks in stage 4: Stage finished 20/01/23 18:54:03.980 Thread-4 INFO DAGScheduler: Job 2 finished: collect at ClusteringEvaluator.scala:179, took 0.350938 s 20/01/23 18:54:03.984 Thread-4 DEBUG GenerateSafeProjection: code for createexternalrow(input[0, double, true], StructField(avg(UDF(normalizedFeatures, cast(prediction as double))),DoubleType,true)): /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificSafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificSafeProjection extends org.apache.spark.sql.catalyst.expressions.codegen.BaseProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private InternalRow mutableRow; /* 009 */ /* 010 */ /* 011 */ public SpecificSafeProjection(Object[] references) { /* 012 */ this.references = references; /* 013 */ mutableRow = (InternalRow) references[references.length - 1]; /* 014 */ /* 015 */ } /* 016 */ /* 017 */ public void initialize(int partitionIndex) { /* 018 */ /* 019 */ } /* 020 */ /* 021 */ public java.lang.Object apply(java.lang.Object _i) { /* 022 */ InternalRow i = (InternalRow) _i; /* 023 */ Object[] values_0 = new Object[1]; /* 024 */ /* 025 */ boolean isNull_1 = i.isNullAt(0); /* 026 */ double value_1 = isNull_1 ? /* 027 */ -1.0 : (i.getDouble(0)); /* 028 */ if (isNull_1) { /* 029 */ values_0[0] = null; /* 030 */ } else { /* 031 */ values_0[0] = value_1; /* 032 */ } /* 033 */ /* 034 */ final org.apache.spark.sql.Row value_0 = new org.apache.spark.sql.catalyst.expressions.GenericRowWithSchema(values_0, ((org.apache.spark.sql.types.StructType) references[0] /* schema */)); /* 035 */ if (false) { /* 036 */ mutableRow.setNullAt(0); /* 037 */ } else { /* 038 */ /* 039 */ mutableRow.update(0, value_0); /* 040 */ } /* 041 */ /* 042 */ return mutableRow; /* 043 */ } /* 044 */ /* 045 */ /* 046 */ } 20/01/23 18:54:03.985 Thread-4 DEBUG CodeGenerator: /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificSafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificSafeProjection extends org.apache.spark.sql.catalyst.expressions.codegen.BaseProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private InternalRow mutableRow; /* 009 */ /* 010 */ /* 011 */ public SpecificSafeProjection(Object[] references) { /* 012 */ this.references = references; /* 013 */ mutableRow = (InternalRow) references[references.length - 1]; /* 014 */ /* 015 */ } /* 016 */ /* 017 */ public void initialize(int partitionIndex) { /* 018 */ /* 019 */ } /* 020 */ /* 021 */ public java.lang.Object apply(java.lang.Object _i) { /* 022 */ InternalRow i = (InternalRow) _i; /* 023 */ Object[] values_0 = new Object[1]; /* 024 */ /* 025 */ boolean isNull_1 = i.isNullAt(0); /* 026 */ double value_1 = isNull_1 ? /* 027 */ -1.0 : (i.getDouble(0)); /* 028 */ if (isNull_1) { /* 029 */ values_0[0] = null; /* 030 */ } else { /* 031 */ values_0[0] = value_1; /* 032 */ } /* 033 */ /* 034 */ final org.apache.spark.sql.Row value_0 = new org.apache.spark.sql.catalyst.expressions.GenericRowWithSchema(values_0, ((org.apache.spark.sql.types.StructType) references[0] /* schema */)); /* 035 */ if (false) { /* 036 */ mutableRow.setNullAt(0); /* 037 */ } else { /* 038 */ /* 039 */ mutableRow.update(0, value_0); /* 040 */ } /* 041 */ /* 042 */ return mutableRow; /* 043 */ } /* 044 */ /* 045 */ /* 046 */ } 20/01/23 18:54:03.986 Thread-4 DEBUG Analyzer$ResolveReferences: Resolving 'features to features#30 20/01/23 18:54:03.988 Thread-4 DEBUG Analyzer$ResolveReferences: Resolving 'expected to expected#23 20/01/23 18:54:03.994 Executor task launch worker for task 50 INFO Executor: Finished task 7.0 in stage 11.0 (TID 50). 2303 bytes result sent to driver 20/01/23 18:54:03.995 dispatcher-event-loop-2 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_11.0, runningTasks: 3 20/01/23 18:54:03.995 dispatcher-event-loop-2 INFO TaskSetManager: Starting task 8.0 in stage 11.0 (TID 51, 192.168.1.225, executor driver, partition 8, PROCESS_LOCAL, 7422 bytes) 20/01/23 18:54:03.995 Executor task launch worker for task 51 INFO Executor: Running task 8.0 in stage 11.0 (TID 51) 20/01/23 18:54:03.996 task-result-getter-0 INFO TaskSetManager: Finished task 7.0 in stage 11.0 (TID 50) in 46 ms on 192.168.1.225 (executor driver) (5/10) 20/01/23 18:54:03.996 Executor task launch worker for task 51 DEBUG ExecutorMetricsPoller: stageTCMP: (11, 0) -> 5 20/01/23 18:54:03.996 Thread-4 INFO CodeGenerator: Code generated in 11.696543 ms 20/01/23 18:54:03.998 dag-scheduler-event-loop DEBUG DAGScheduler: ShuffleMapTask finished on driver 20/01/23 18:54:03.999 Executor task launch worker for task 50 DEBUG ExecutorMetricsPoller: stageTCMP: (11, 0) -> 4 20/01/23 18:54:04.003 Executor task launch worker for task 51 DEBUG BlockManager: Getting local block rdd_25_8 20/01/23 18:54:04.004 Executor task launch worker for task 51 DEBUG BlockManager: Level for block rdd_25_8 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:04.004 Executor task launch worker for task 51 INFO BlockManager: Found block rdd_25_8 locally 20/01/23 18:54:04.004 Executor task launch worker for task 51 DEBUG BlockManager: Getting local block rdd_24_8 20/01/23 18:54:04.004 Executor task launch worker for task 51 DEBUG BlockManager: Level for block rdd_24_8 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:04.004 Executor task launch worker for task 51 INFO BlockManager: Found block rdd_24_8 locally 20/01/23 18:54:04.004 Executor task launch worker for task 51 DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 4, partitions 8-9 20/01/23 18:54:04.004 Executor task launch worker for task 51 DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 20/01/23 18:54:04.005 Executor task launch worker for task 51 INFO ShuffleBlockFetcherIterator: Getting 3 (684.0 B) non-empty blocks including 3 (684.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 20/01/23 18:54:04.005 Executor task launch worker for task 51 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 20/01/23 18:54:04.005 Executor task launch worker for task 51 DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_4_33_8,0), (shuffle_4_34_8,1), (shuffle_4_35_8,2) 20/01/23 18:54:04.005 Executor task launch worker for task 51 DEBUG ShuffleBlockFetcherIterator: Got local blocks in 1 ms 20/01/23 18:54:04.011 Executor task launch worker for task 51 DEBUG TaskMemoryManager: Task 51 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@4c7781f0 20/01/23 18:54:04.011 Executor task launch worker for task 51 DEBUG TaskMemoryManager: Task 51 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@e04ebca 20/01/23 18:54:04.011 Executor task launch worker for task 51 DEBUG TaskMemoryManager: Task 51 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@d3b4400 20/01/23 18:54:04.014 Executor task launch worker for task 51 INFO Executor: Finished task 8.0 in stage 11.0 (TID 51). 2303 bytes result sent to driver 20/01/23 18:54:04.014 dispatcher-event-loop-0 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_11.0, runningTasks: 3 20/01/23 18:54:04.015 dispatcher-event-loop-0 INFO TaskSetManager: Starting task 9.0 in stage 11.0 (TID 52, 192.168.1.225, executor driver, partition 9, PROCESS_LOCAL, 7422 bytes) 20/01/23 18:54:04.016 task-result-getter-3 INFO TaskSetManager: Finished task 8.0 in stage 11.0 (TID 51) in 21 ms on 192.168.1.225 (executor driver) (6/10) 20/01/23 18:54:04.016 Executor task launch worker for task 52 INFO Executor: Running task 9.0 in stage 11.0 (TID 52) 20/01/23 18:54:04.016 Executor task launch worker for task 51 DEBUG ExecutorMetricsPoller: stageTCMP: (11, 0) -> 3 20/01/23 18:54:04.016 Executor task launch worker for task 52 DEBUG ExecutorMetricsPoller: stageTCMP: (11, 0) -> 4 20/01/23 18:54:04.016 dag-scheduler-event-loop DEBUG DAGScheduler: ShuffleMapTask finished on driver 20/01/23 18:54:04.018 Thread-4 INFO TorrentBroadcast: Destroying Broadcast(3) (from destroy at ClusteringEvaluator.scala:654) 20/01/23 18:54:04.018 Thread-4 DEBUG TorrentBroadcast: Unpersisting TorrentBroadcast 3 20/01/23 18:54:04.024 Executor task launch worker for task 52 DEBUG BlockManager: Getting local block rdd_25_9 20/01/23 18:54:04.024 Executor task launch worker for task 52 DEBUG BlockManager: Level for block rdd_25_9 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:04.024 Executor task launch worker for task 52 INFO BlockManager: Found block rdd_25_9 locally 20/01/23 18:54:04.024 Executor task launch worker for task 52 DEBUG BlockManager: Getting local block rdd_24_9 20/01/23 18:54:04.024 Executor task launch worker for task 52 DEBUG BlockManager: Level for block rdd_24_9 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:04.024 Executor task launch worker for task 52 INFO BlockManager: Found block rdd_24_9 locally 20/01/23 18:54:04.024 Executor task launch worker for task 52 DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 4, partitions 9-10 20/01/23 18:54:04.025 Executor task launch worker for task 52 DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 20/01/23 18:54:04.025 Executor task launch worker for task 52 INFO ShuffleBlockFetcherIterator: Getting 3 (684.0 B) non-empty blocks including 3 (684.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 20/01/23 18:54:04.025 Executor task launch worker for task 52 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 20/01/23 18:54:04.025 Executor task launch worker for task 52 DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_4_33_9,0), (shuffle_4_34_9,1), (shuffle_4_35_9,2) 20/01/23 18:54:04.026 Executor task launch worker for task 52 DEBUG ShuffleBlockFetcherIterator: Got local blocks in 1 ms 20/01/23 18:54:04.028 Executor task launch worker for task 52 DEBUG TaskMemoryManager: Task 52 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@51369da2 20/01/23 18:54:04.028 Executor task launch worker for task 52 DEBUG TaskMemoryManager: Task 52 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@452c937 20/01/23 18:54:04.028 Executor task launch worker for task 52 DEBUG TaskMemoryManager: Task 52 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@6a88aa72 20/01/23 18:54:04.030 Executor task launch worker for task 52 INFO Executor: Finished task 9.0 in stage 11.0 (TID 52). 2303 bytes result sent to driver 20/01/23 18:54:04.033 block-manager-slave-async-thread-pool-3 DEBUG BlockManagerSlaveEndpoint: removing broadcast 3 20/01/23 18:54:04.034 block-manager-slave-async-thread-pool-3 DEBUG BlockManager: Removing broadcast 3 20/01/23 18:54:04.034 block-manager-slave-async-thread-pool-3 DEBUG BlockManager: Removing block broadcast_3_piece0 20/01/23 18:54:04.049 block-manager-slave-async-thread-pool-3 DEBUG MemoryStore: Block broadcast_3_piece0 of size 484 dropped from memory (free 383993723) 20/01/23 18:54:04.050 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Removed broadcast_3_piece0 on 192.168.1.225:45461 in memory (size: 484.0 B, free: 366.3 MiB) 20/01/23 18:54:04.050 block-manager-slave-async-thread-pool-3 DEBUG BlockManagerMaster: Updated info of block broadcast_3_piece0 20/01/23 18:54:04.050 block-manager-slave-async-thread-pool-3 DEBUG BlockManager: Told master about block broadcast_3_piece0 20/01/23 18:54:04.050 block-manager-slave-async-thread-pool-3 DEBUG BlockManager: Removing block broadcast_3 20/01/23 18:54:04.050 block-manager-slave-async-thread-pool-3 DEBUG MemoryStore: Block broadcast_3 of size 400 dropped from memory (free 383994123) 20/01/23 18:54:04.056 dispatcher-event-loop-2 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_11.0, runningTasks: 3 20/01/23 18:54:04.056 dispatcher-event-loop-2 DEBUG TaskSetManager: No tasks for locality level PROCESS_LOCAL, so moving to locality level NODE_LOCAL 20/01/23 18:54:04.056 dispatcher-event-loop-2 DEBUG TaskSetManager: No tasks for locality level NODE_LOCAL, so moving to locality level ANY 20/01/23 18:54:04.057 task-result-getter-2 INFO TaskSetManager: Finished task 9.0 in stage 11.0 (TID 52) in 42 ms on 192.168.1.225 (executor driver) (7/10) 20/01/23 18:54:04.058 dag-scheduler-event-loop DEBUG DAGScheduler: ShuffleMapTask finished on driver 20/01/23 18:54:04.059 Executor task launch worker for task 52 DEBUG ExecutorMetricsPoller: stageTCMP: (11, 0) -> 3 20/01/23 18:54:04.078 block-manager-slave-async-thread-pool-5 DEBUG BlockManagerSlaveEndpoint: Done removing broadcast 3, response is 0 20/01/23 18:54:04.084 Executor task launch worker for task 43 DEBUG TaskMemoryManager: Task 43 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@591800aa 20/01/23 18:54:04.085 Executor task launch worker for task 44 DEBUG TaskMemoryManager: Task 44 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@26e50ddb 20/01/23 18:54:04.087 Executor task launch worker for task 44 DEBUG TaskMemoryManager: Task 44 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@7e03e5d0 20/01/23 18:54:04.090 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$pythonToJava$1 20/01/23 18:54:04.090 Executor task launch worker for task 44 INFO Executor: Finished task 1.0 in stage 11.0 (TID 44). 2475 bytes result sent to driver 20/01/23 18:54:04.090 Executor task launch worker for task 44 DEBUG ExecutorMetricsPoller: stageTCMP: (11, 0) -> 2 20/01/23 18:54:04.091 dispatcher-event-loop-3 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_11.0, runningTasks: 2 20/01/23 18:54:04.091 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$pythonToJava$1) is now cleaned +++ 20/01/23 18:54:04.092 task-result-getter-1 INFO TaskSetManager: Finished task 1.0 in stage 11.0 (TID 44) in 300 ms on 192.168.1.225 (executor driver) (8/10) 20/01/23 18:54:04.093 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$toJavaArray$1 20/01/23 18:54:04.093 dag-scheduler-event-loop DEBUG DAGScheduler: ShuffleMapTask finished on driver 20/01/23 18:54:04.096 Executor task launch worker for task 43 DEBUG TaskMemoryManager: Task 43 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@7fffd568 20/01/23 18:54:04.099 Executor task launch worker for task 43 INFO Executor: Finished task 0.0 in stage 11.0 (TID 43). 2475 bytes result sent to driver 20/01/23 18:54:04.099 Executor task launch worker for task 43 DEBUG ExecutorMetricsPoller: stageTCMP: (11, 0) -> 1 20/01/23 18:54:04.099 dispatcher-event-loop-0 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_11.0, runningTasks: 1 20/01/23 18:54:04.100 task-result-getter-0 INFO TaskSetManager: Finished task 0.0 in stage 11.0 (TID 43) in 310 ms on 192.168.1.225 (executor driver) (9/10) 20/01/23 18:54:04.101 dag-scheduler-event-loop DEBUG DAGScheduler: ShuffleMapTask finished on driver 20/01/23 18:54:04.104 Executor task launch worker for task 45 DEBUG TaskMemoryManager: Task 45 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@73670fcb 20/01/23 18:54:04.107 Executor task launch worker for task 45 DEBUG TaskMemoryManager: Task 45 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@4457587e 20/01/23 18:54:04.109 block-manager-slave-async-thread-pool-5 DEBUG BlockManagerSlaveEndpoint: Sent response: 0 to 192.168.1.225:33699 20/01/23 18:54:04.125 Executor task launch worker for task 45 INFO Executor: Finished task 2.0 in stage 11.0 (TID 45). 2475 bytes result sent to driver 20/01/23 18:54:04.126 Executor task launch worker for task 45 DEBUG ExecutorMetricsPoller: removing (11, 0) from stageTCMP 20/01/23 18:54:04.126 dispatcher-event-loop-1 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_11.0, runningTasks: 0 20/01/23 18:54:04.127 task-result-getter-3 INFO TaskSetManager: Finished task 2.0 in stage 11.0 (TID 45) in 336 ms on 192.168.1.225 (executor driver) (10/10) 20/01/23 18:54:04.127 task-result-getter-3 INFO TaskSchedulerImpl: Removed TaskSet 11.0, whose tasks have all completed, from pool 20/01/23 18:54:04.128 dag-scheduler-event-loop DEBUG DAGScheduler: ShuffleMapTask finished on driver 20/01/23 18:54:04.128 dag-scheduler-event-loop INFO DAGScheduler: ShuffleMapStage 11 (flatMap at ALS.scala:1674) finished in 0.347 s 20/01/23 18:54:04.128 dag-scheduler-event-loop INFO DAGScheduler: looking for newly runnable stages 20/01/23 18:54:04.128 dag-scheduler-event-loop INFO DAGScheduler: running: Set() 20/01/23 18:54:04.128 dag-scheduler-event-loop INFO DAGScheduler: waiting: Set(ResultStage 12) 20/01/23 18:54:04.128 dag-scheduler-event-loop INFO DAGScheduler: failed: Set() 20/01/23 18:54:04.128 dag-scheduler-event-loop DEBUG MapOutputTrackerMaster: Increasing epoch to 5 20/01/23 18:54:04.128 dag-scheduler-event-loop DEBUG DAGScheduler: submitStage(ResultStage 12 (name=count at ALS.scala:1048;jobs=3)) 20/01/23 18:54:04.132 dag-scheduler-event-loop DEBUG DAGScheduler: missing: List() 20/01/23 18:54:04.132 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$toJavaArray$1) is now cleaned +++ 20/01/23 18:54:04.132 dag-scheduler-event-loop INFO DAGScheduler: Submitting ResultStage 12 (userFactors MapPartitionsRDD[50] at mapPartitions at ALS.scala:1028), which has no missing parents 20/01/23 18:54:04.132 dag-scheduler-event-loop DEBUG DAGScheduler: submitMissingTasks(ResultStage 12) 20/01/23 18:54:04.143 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$applySchemaToPythonRDD$1 20/01/23 18:54:04.145 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$applySchemaToPythonRDD$1) is now cleaned +++ 20/01/23 18:54:04.146 Thread-4 DEBUG WholeStageCodegenExec: /* 001 */ public Object generate(Object[] references) { /* 002 */ return new GeneratedIteratorForCodegenStage1(references); /* 003 */ } /* 004 */ /* 005 */ // codegenStageId=1 /* 006 */ final class GeneratedIteratorForCodegenStage1 extends org.apache.spark.sql.execution.BufferedRowIterator { /* 007 */ private Object[] references; /* 008 */ private scala.collection.Iterator[] inputs; /* 009 */ private scala.collection.Iterator rdd_input_0; /* 010 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[] rdd_mutableStateArray_0 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[5]; /* 011 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[] rdd_mutableStateArray_1 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[7]; /* 012 */ /* 013 */ public GeneratedIteratorForCodegenStage1(Object[] references) { /* 014 */ this.references = references; /* 015 */ } /* 016 */ /* 017 */ public void init(int index, scala.collection.Iterator[] inputs) { /* 018 */ partitionIndex = index; /* 019 */ this.inputs = inputs; /* 020 */ wholestagecodegen_init_0_0(); /* 021 */ wholestagecodegen_init_0_1(); /* 022 */ /* 023 */ } /* 024 */ /* 025 */ private void project_writeFields_1_1(InternalRow project_tmpInput_3) { /* 026 */ if ((project_tmpInput_3.isNullAt(3))) { /* 027 */ rdd_mutableStateArray_0[4].setNullAt(3); /* 028 */ } else { /* 029 */ // Remember the current cursor so that we can calculate how many bytes are /* 030 */ // written later. /* 031 */ final int project_previousCursor_5 = rdd_mutableStateArray_0[4].cursor(); /* 032 */ /* 033 */ final ArrayData project_tmpInput_5 = (project_tmpInput_3.getArray(3)); /* 034 */ if (project_tmpInput_5 instanceof UnsafeArrayData) { /* 035 */ rdd_mutableStateArray_0[4].write((UnsafeArrayData) project_tmpInput_5); /* 036 */ } else { /* 037 */ final int project_numElements_3 = project_tmpInput_5.numElements(); /* 038 */ rdd_mutableStateArray_1[6].initialize(project_numElements_3); /* 039 */ /* 040 */ for (int project_index_3 = 0; project_index_3 < project_numElements_3; project_index_3++) { /* 041 */ rdd_mutableStateArray_1[6].write(project_index_3, project_tmpInput_5.getDouble(project_index_3)); /* 042 */ } /* 043 */ } /* 044 */ /* 045 */ rdd_mutableStateArray_0[4].setOffsetAndSizeFromPreviousCursor(3, project_previousCursor_5); /* 046 */ } /* 047 */ /* 048 */ } /* 049 */ /* 050 */ private void project_writeFields_1_0(InternalRow project_tmpInput_3) { /* 051 */ rdd_mutableStateArray_0[4].write(0, (project_tmpInput_3.getByte(0))); /* 052 */ /* 053 */ if ((project_tmpInput_3.isNullAt(1))) { /* 054 */ rdd_mutableStateArray_0[4].setNullAt(1); /* 055 */ } else { /* 056 */ rdd_mutableStateArray_0[4].write(1, (project_tmpInput_3.getInt(1))); /* 057 */ } /* 058 */ /* 059 */ if ((project_tmpInput_3.isNullAt(2))) { /* 060 */ rdd_mutableStateArray_0[4].setNullAt(2); /* 061 */ } else { /* 062 */ // Remember the current cursor so that we can calculate how many bytes are /* 063 */ // written later. /* 064 */ final int project_previousCursor_4 = rdd_mutableStateArray_0[4].cursor(); /* 065 */ /* 066 */ final ArrayData project_tmpInput_4 = (project_tmpInput_3.getArray(2)); /* 067 */ if (project_tmpInput_4 instanceof UnsafeArrayData) { /* 068 */ rdd_mutableStateArray_0[4].write((UnsafeArrayData) project_tmpInput_4); /* 069 */ } else { /* 070 */ final int project_numElements_2 = project_tmpInput_4.numElements(); /* 071 */ rdd_mutableStateArray_1[5].initialize(project_numElements_2); /* 072 */ /* 073 */ for (int project_index_2 = 0; project_index_2 < project_numElements_2; project_index_2++) { /* 074 */ rdd_mutableStateArray_1[5].write(project_index_2, project_tmpInput_4.getInt(project_index_2)); /* 075 */ } /* 076 */ } /* 077 */ /* 078 */ rdd_mutableStateArray_0[4].setOffsetAndSizeFromPreviousCursor(2, project_previousCursor_4); /* 079 */ } /* 080 */ /* 081 */ } /* 082 */ /* 083 */ private void project_writeFields_0_1(InternalRow project_tmpInput_0) { /* 084 */ if ((project_tmpInput_0.isNullAt(3))) { /* 085 */ rdd_mutableStateArray_0[3].setNullAt(3); /* 086 */ } else { /* 087 */ // Remember the current cursor so that we can calculate how many bytes are /* 088 */ // written later. /* 089 */ final int project_previousCursor_2 = rdd_mutableStateArray_0[3].cursor(); /* 090 */ /* 091 */ final ArrayData project_tmpInput_2 = (project_tmpInput_0.getArray(3)); /* 092 */ if (project_tmpInput_2 instanceof UnsafeArrayData) { /* 093 */ rdd_mutableStateArray_0[3].write((UnsafeArrayData) project_tmpInput_2); /* 094 */ } else { /* 095 */ final int project_numElements_1 = project_tmpInput_2.numElements(); /* 096 */ rdd_mutableStateArray_1[4].initialize(project_numElements_1); /* 097 */ /* 098 */ for (int project_index_1 = 0; project_index_1 < project_numElements_1; project_index_1++) { /* 099 */ rdd_mutableStateArray_1[4].write(project_index_1, project_tmpInput_2.getDouble(project_index_1)); /* 100 */ } /* 101 */ } /* 102 */ /* 103 */ rdd_mutableStateArray_0[3].setOffsetAndSizeFromPreviousCursor(3, project_previousCursor_2); /* 104 */ } /* 105 */ /* 106 */ } /* 107 */ /* 108 */ private void wholestagecodegen_init_0_1() { /* 109 */ rdd_mutableStateArray_1[4] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[3], 8); /* 110 */ rdd_mutableStateArray_0[4] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(rdd_mutableStateArray_0[2], 4); /* 111 */ rdd_mutableStateArray_1[5] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[4], 4); /* 112 */ rdd_mutableStateArray_1[6] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[4], 8); /* 113 */ /* 114 */ } /* 115 */ /* 116 */ private void project_writeFields_0_0(InternalRow project_tmpInput_0) { /* 117 */ rdd_mutableStateArray_0[3].write(0, (project_tmpInput_0.getByte(0))); /* 118 */ /* 119 */ if ((project_tmpInput_0.isNullAt(1))) { /* 120 */ rdd_mutableStateArray_0[3].setNullAt(1); /* 121 */ } else { /* 122 */ rdd_mutableStateArray_0[3].write(1, (project_tmpInput_0.getInt(1))); /* 123 */ } /* 124 */ /* 125 */ if ((project_tmpInput_0.isNullAt(2))) { /* 126 */ rdd_mutableStateArray_0[3].setNullAt(2); /* 127 */ } else { /* 128 */ // Remember the current cursor so that we can calculate how many bytes are /* 129 */ // written later. /* 130 */ final int project_previousCursor_1 = rdd_mutableStateArray_0[3].cursor(); /* 131 */ /* 132 */ final ArrayData project_tmpInput_1 = (project_tmpInput_0.getArray(2)); /* 133 */ if (project_tmpInput_1 instanceof UnsafeArrayData) { /* 134 */ rdd_mutableStateArray_0[3].write((UnsafeArrayData) project_tmpInput_1); /* 135 */ } else { /* 136 */ final int project_numElements_0 = project_tmpInput_1.numElements(); /* 137 */ rdd_mutableStateArray_1[3].initialize(project_numElements_0); /* 138 */ /* 139 */ for (int project_index_0 = 0; project_index_0 < project_numElements_0; project_index_0++) { /* 140 */ rdd_mutableStateArray_1[3].write(project_index_0, project_tmpInput_1.getInt(project_index_0)); /* 141 */ } /* 142 */ } /* 143 */ /* 144 */ rdd_mutableStateArray_0[3].setOffsetAndSizeFromPreviousCursor(2, project_previousCursor_1); /* 145 */ } /* 146 */ /* 147 */ } /* 148 */ /* 149 */ protected void processNext() throws java.io.IOException { /* 150 */ while ( rdd_input_0.hasNext()) { /* 151 */ InternalRow rdd_row_0 = (InternalRow) rdd_input_0.next(); /* 152 */ ((org.apache.spark.sql.execution.metric.SQLMetric) references[0] /* numOutputRows */).add(1); /* 153 */ boolean rdd_isNull_1 = rdd_row_0.isNullAt(1); /* 154 */ ArrayData rdd_value_1 = rdd_isNull_1 ? /* 155 */ null : (rdd_row_0.getArray(1)); /* 156 */ Object project_arg_0 = rdd_isNull_1 ? null : ((scala.Function1[]) references[1] /* converters */)[0].apply(rdd_value_1); /* 157 */ /* 158 */ InternalRow project_result_0 = null; /* 159 */ try { /* 160 */ project_result_0 = (InternalRow)((scala.Function1[]) references[1] /* converters */)[1].apply(((scala.Function1) references[3] /* udf */).apply(project_arg_0)); /* 161 */ } catch (Exception e) { /* 162 */ throw new org.apache.spark.SparkException(((java.lang.String) references[2] /* errMsg */), e); /* 163 */ } /* 164 */ /* 165 */ boolean project_isNull_0 = project_result_0 == null; /* 166 */ InternalRow project_value_0 = null; /* 167 */ if (!project_isNull_0) { /* 168 */ project_value_0 = project_result_0; /* 169 */ } /* 170 */ boolean rdd_isNull_2 = rdd_row_0.isNullAt(2); /* 171 */ InternalRow rdd_value_2 = rdd_isNull_2 ? /* 172 */ null : (rdd_row_0.getStruct(2, 4)); /* 173 */ rdd_mutableStateArray_0[2].reset(); /* 174 */ /* 175 */ rdd_mutableStateArray_0[2].zeroOutNullBytes(); /* 176 */ /* 177 */ if (project_isNull_0) { /* 178 */ rdd_mutableStateArray_0[2].setNullAt(0); /* 179 */ } else { /* 180 */ final InternalRow project_tmpInput_0 = project_value_0; /* 181 */ if (project_tmpInput_0 instanceof UnsafeRow) { /* 182 */ rdd_mutableStateArray_0[2].write(0, (UnsafeRow) project_tmpInput_0); /* 183 */ } else { /* 184 */ // Remember the current cursor so that we can calculate how many bytes are /* 185 */ // written later. /* 186 */ final int project_previousCursor_0 = rdd_mutableStateArray_0[2].cursor(); /* 187 */ /* 188 */ rdd_mutableStateArray_0[3].resetRowWriter(); /* 189 */ project_writeFields_0_0(project_tmpInput_0); /* 190 */ project_writeFields_0_1(project_tmpInput_0); /* 191 */ /* 192 */ rdd_mutableStateArray_0[2].setOffsetAndSizeFromPreviousCursor(0, project_previousCursor_0); /* 193 */ } /* 194 */ } /* 195 */ /* 196 */ if (rdd_isNull_2) { /* 197 */ rdd_mutableStateArray_0[2].setNullAt(1); /* 198 */ } else { /* 199 */ final InternalRow project_tmpInput_3 = rdd_value_2; /* 200 */ if (project_tmpInput_3 instanceof UnsafeRow) { /* 201 */ rdd_mutableStateArray_0[2].write(1, (UnsafeRow) project_tmpInput_3); /* 202 */ } else { /* 203 */ // Remember the current cursor so that we can calculate how many bytes are /* 204 */ // written later. /* 205 */ final int project_previousCursor_3 = rdd_mutableStateArray_0[2].cursor(); /* 206 */ /* 207 */ rdd_mutableStateArray_0[4].resetRowWriter(); /* 208 */ project_writeFields_1_0(project_tmpInput_3); /* 209 */ project_writeFields_1_1(project_tmpInput_3); /* 210 */ /* 211 */ rdd_mutableStateArray_0[2].setOffsetAndSizeFromPreviousCursor(1, project_previousCursor_3); /* 212 */ } /* 213 */ } /* 214 */ append((rdd_mutableStateArray_0[2].getRow())); /* 215 */ if (shouldStop()) return; /* 216 */ } /* 217 */ } /* 218 */ /* 219 */ private void wholestagecodegen_init_0_0() { /* 220 */ rdd_input_0 = inputs[0]; /* 221 */ rdd_mutableStateArray_0[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(3, 64); /* 222 */ rdd_mutableStateArray_1[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[0], 8); /* 223 */ rdd_mutableStateArray_0[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(rdd_mutableStateArray_0[0], 4); /* 224 */ rdd_mutableStateArray_1[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[1], 4); /* 225 */ rdd_mutableStateArray_1[2] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[1], 8); /* 226 */ rdd_mutableStateArray_0[2] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(2, 64); /* 227 */ rdd_mutableStateArray_0[3] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(rdd_mutableStateArray_0[2], 4); /* 228 */ rdd_mutableStateArray_1[3] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[3], 4); /* 229 */ /* 230 */ } /* 231 */ /* 232 */ } 20/01/23 18:54:04.154 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_8 stored as values in memory (estimated size 36.2 KiB, free 366.0 MiB) 20/01/23 18:54:04.154 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_8 locally took 0 ms 20/01/23 18:54:04.155 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_8 without replication took 0 ms 20/01/23 18:54:04.156 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_8_piece0 stored as bytes in memory (estimated size 16.2 KiB, free 366.0 MiB) 20/01/23 18:54:04.156 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added broadcast_8_piece0 in memory on 192.168.1.225:35863 (size: 16.2 KiB, free: 366.2 MiB) 20/01/23 18:54:04.157 dag-scheduler-event-loop DEBUG BlockManagerMaster: Updated info of block broadcast_8_piece0 20/01/23 18:54:04.157 dag-scheduler-event-loop DEBUG BlockManager: Told master about block broadcast_8_piece0 20/01/23 18:54:04.157 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_8_piece0 locally took 1 ms 20/01/23 18:54:04.157 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_8_piece0 without replication took 1 ms 20/01/23 18:54:04.157 dag-scheduler-event-loop INFO SparkContext: Created broadcast 8 from broadcast at DAGScheduler.scala:1200 20/01/23 18:54:04.158 dag-scheduler-event-loop INFO DAGScheduler: Submitting 10 missing tasks from ResultStage 12 (userFactors MapPartitionsRDD[50] at mapPartitions at ALS.scala:1028) (first 15 tasks are for partitions Vector(0, 1, 2, 3, 4, 5, 6, 7, 8, 9)) 20/01/23 18:54:04.158 dag-scheduler-event-loop INFO TaskSchedulerImpl: Adding task set 12.0 with 10 tasks 20/01/23 18:54:04.159 dag-scheduler-event-loop DEBUG TaskSetManager: Epoch for TaskSet 12.0: 5 20/01/23 18:54:04.159 dag-scheduler-event-loop DEBUG TaskSetManager: Adding pending tasks took 0 ms 20/01/23 18:54:04.159 dag-scheduler-event-loop DEBUG TaskSetManager: Valid locality levels for TaskSet 12.0: PROCESS_LOCAL, NODE_LOCAL, ANY 20/01/23 18:54:04.160 dispatcher-event-loop-2 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_12.0, runningTasks: 0 20/01/23 18:54:04.160 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$doExecute$4$adapted 20/01/23 18:54:04.160 dispatcher-event-loop-2 INFO TaskSetManager: Starting task 0.0 in stage 12.0 (TID 53, 192.168.1.225, executor driver, partition 0, PROCESS_LOCAL, 7433 bytes) 20/01/23 18:54:04.160 dispatcher-event-loop-2 INFO TaskSetManager: Starting task 1.0 in stage 12.0 (TID 54, 192.168.1.225, executor driver, partition 1, PROCESS_LOCAL, 7433 bytes) 20/01/23 18:54:04.161 dispatcher-event-loop-2 INFO TaskSetManager: Starting task 2.0 in stage 12.0 (TID 55, 192.168.1.225, executor driver, partition 2, PROCESS_LOCAL, 7433 bytes) 20/01/23 18:54:04.161 dispatcher-event-loop-2 INFO TaskSetManager: Starting task 3.0 in stage 12.0 (TID 56, 192.168.1.225, executor driver, partition 3, PROCESS_LOCAL, 7433 bytes) 20/01/23 18:54:04.161 Executor task launch worker for task 56 INFO Executor: Running task 3.0 in stage 12.0 (TID 56) 20/01/23 18:54:04.161 Executor task launch worker for task 53 INFO Executor: Running task 0.0 in stage 12.0 (TID 53) 20/01/23 18:54:04.161 Executor task launch worker for task 54 INFO Executor: Running task 1.0 in stage 12.0 (TID 54) 20/01/23 18:54:04.162 Executor task launch worker for task 56 DEBUG ExecutorMetricsPoller: stageTCMP: (12, 0) -> 1 20/01/23 18:54:04.162 Executor task launch worker for task 54 DEBUG ExecutorMetricsPoller: stageTCMP: (12, 0) -> 2 20/01/23 18:54:04.162 Executor task launch worker for task 56 DEBUG BlockManager: Getting local block broadcast_8 20/01/23 18:54:04.162 Executor task launch worker for task 56 DEBUG BlockManager: Level for block broadcast_8 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:04.162 Executor task launch worker for task 55 INFO Executor: Running task 2.0 in stage 12.0 (TID 55) 20/01/23 18:54:04.163 Executor task launch worker for task 55 DEBUG ExecutorMetricsPoller: stageTCMP: (12, 0) -> 3 20/01/23 18:54:04.163 Executor task launch worker for task 53 DEBUG ExecutorMetricsPoller: stageTCMP: (12, 0) -> 4 20/01/23 18:54:04.168 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$doExecute$4$adapted) is now cleaned +++ 20/01/23 18:54:04.170 Executor task launch worker for task 56 DEBUG BlockManager: Getting local block rdd_50_3 20/01/23 18:54:04.170 Executor task launch worker for task 56 DEBUG BlockManager: Block rdd_50_3 was not found 20/01/23 18:54:04.170 Executor task launch worker for task 56 DEBUG BlockManager: Getting remote block rdd_50_3 20/01/23 18:54:04.170 Executor task launch worker for task 56 DEBUG BlockManager: Block rdd_50_3 is unknown by block manager master 20/01/23 18:54:04.170 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$collect$2 20/01/23 18:54:04.170 Executor task launch worker for task 56 DEBUG BlockManager: Getting local block rdd_19_3 20/01/23 18:54:04.171 Executor task launch worker for task 56 DEBUG BlockManager: Level for block rdd_19_3 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:04.171 Executor task launch worker for task 56 INFO BlockManager: Found block rdd_19_3 locally 20/01/23 18:54:04.171 Executor task launch worker for task 56 DEBUG BlockManager: Getting local block rdd_19_3 20/01/23 18:54:04.171 Executor task launch worker for task 56 DEBUG BlockManager: Level for block rdd_19_3 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:04.171 Executor task launch worker for task 56 INFO BlockManager: Found block rdd_19_3 locally 20/01/23 18:54:04.171 Executor task launch worker for task 56 DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 3, partitions 3-4 20/01/23 18:54:04.171 Executor task launch worker for task 56 DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 20/01/23 18:54:04.172 Executor task launch worker for task 56 INFO ShuffleBlockFetcherIterator: Getting 3 (684.0 B) non-empty blocks including 3 (684.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 20/01/23 18:54:04.172 Executor task launch worker for task 56 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 20/01/23 18:54:04.172 Executor task launch worker for task 56 DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_3_43_3,0), (shuffle_3_44_3,1), (shuffle_3_45_3,2) 20/01/23 18:54:04.172 Executor task launch worker for task 53 DEBUG BlockManager: Getting local block rdd_50_0 20/01/23 18:54:04.172 Executor task launch worker for task 53 DEBUG BlockManager: Block rdd_50_0 was not found 20/01/23 18:54:04.172 Executor task launch worker for task 53 DEBUG BlockManager: Getting remote block rdd_50_0 20/01/23 18:54:04.172 Executor task launch worker for task 56 DEBUG ShuffleBlockFetcherIterator: Got local blocks in 1 ms 20/01/23 18:54:04.173 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$collect$2) is now cleaned +++ 20/01/23 18:54:04.173 Executor task launch worker for task 55 DEBUG BlockManager: Getting local block rdd_50_2 20/01/23 18:54:04.173 Executor task launch worker for task 55 DEBUG BlockManager: Block rdd_50_2 was not found 20/01/23 18:54:04.173 Executor task launch worker for task 55 DEBUG BlockManager: Getting remote block rdd_50_2 20/01/23 18:54:04.174 Executor task launch worker for task 56 DEBUG TaskMemoryManager: Task 56 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@74238cce 20/01/23 18:54:04.175 Executor task launch worker for task 56 DEBUG TaskMemoryManager: Task 56 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@107229d1 20/01/23 18:54:04.175 Executor task launch worker for task 56 DEBUG TaskMemoryManager: Task 56 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@74800de3 20/01/23 18:54:04.176 Executor task launch worker for task 54 DEBUG BlockManager: Getting local block rdd_50_1 20/01/23 18:54:04.176 Executor task launch worker for task 54 DEBUG BlockManager: Block rdd_50_1 was not found 20/01/23 18:54:04.176 Executor task launch worker for task 54 DEBUG BlockManager: Getting remote block rdd_50_1 20/01/23 18:54:04.176 Executor task launch worker for task 53 DEBUG BlockManager: Block rdd_50_0 is unknown by block manager master 20/01/23 18:54:04.176 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$runJob$5 20/01/23 18:54:04.177 Executor task launch worker for task 54 DEBUG BlockManager: Block rdd_50_1 is unknown by block manager master 20/01/23 18:54:04.177 Executor task launch worker for task 53 DEBUG BlockManager: Getting local block rdd_19_0 20/01/23 18:54:04.177 Executor task launch worker for task 54 DEBUG BlockManager: Getting local block rdd_19_1 20/01/23 18:54:04.177 Executor task launch worker for task 53 DEBUG BlockManager: Level for block rdd_19_0 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:04.177 Executor task launch worker for task 53 INFO BlockManager: Found block rdd_19_0 locally 20/01/23 18:54:04.177 Executor task launch worker for task 53 DEBUG BlockManager: Getting local block rdd_19_0 20/01/23 18:54:04.177 Executor task launch worker for task 53 DEBUG BlockManager: Level for block rdd_19_0 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:04.177 Executor task launch worker for task 53 INFO BlockManager: Found block rdd_19_0 locally 20/01/23 18:54:04.177 Executor task launch worker for task 55 DEBUG BlockManager: Block rdd_50_2 is unknown by block manager master 20/01/23 18:54:04.177 Executor task launch worker for task 55 DEBUG BlockManager: Getting local block rdd_19_2 20/01/23 18:54:04.177 Executor task launch worker for task 55 DEBUG BlockManager: Level for block rdd_19_2 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:04.177 Executor task launch worker for task 55 INFO BlockManager: Found block rdd_19_2 locally 20/01/23 18:54:04.177 Executor task launch worker for task 55 DEBUG BlockManager: Getting local block rdd_19_2 20/01/23 18:54:04.178 Executor task launch worker for task 55 DEBUG BlockManager: Level for block rdd_19_2 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:04.178 Executor task launch worker for task 55 INFO BlockManager: Found block rdd_19_2 locally 20/01/23 18:54:04.178 Executor task launch worker for task 55 DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 3, partitions 2-3 20/01/23 18:54:04.178 Executor task launch worker for task 55 DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 20/01/23 18:54:04.179 Executor task launch worker for task 55 INFO ShuffleBlockFetcherIterator: Getting 3 (707.0 B) non-empty blocks including 3 (707.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 20/01/23 18:54:04.179 Executor task launch worker for task 55 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 20/01/23 18:54:04.179 Executor task launch worker for task 55 DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_3_43_2,0), (shuffle_3_44_2,1), (shuffle_3_45_2,2) 20/01/23 18:54:04.179 Executor task launch worker for task 55 DEBUG ShuffleBlockFetcherIterator: Got local blocks in 1 ms 20/01/23 18:54:04.181 Executor task launch worker for task 55 DEBUG TaskMemoryManager: Task 55 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@5a7a5566 20/01/23 18:54:04.182 Executor task launch worker for task 55 DEBUG TaskMemoryManager: Task 55 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@2104916e 20/01/23 18:54:04.182 Executor task launch worker for task 55 DEBUG TaskMemoryManager: Task 55 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@31177527 20/01/23 18:54:04.183 Executor task launch worker for task 55 INFO MemoryStore: Block rdd_50_2 stored as values in memory (estimated size 88.0 B, free 366.0 MiB) 20/01/23 18:54:04.183 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_50_2 in memory on 192.168.1.225:35863 (size: 88.0 B, free: 366.2 MiB) 20/01/23 18:54:04.176 Executor task launch worker for task 56 INFO MemoryStore: Block rdd_50_3 stored as values in memory (estimated size 16.0 B, free 366.0 MiB) 20/01/23 18:54:04.177 Executor task launch worker for task 53 DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 3, partitions 0-1 20/01/23 18:54:04.177 Executor task launch worker for task 54 DEBUG BlockManager: Level for block rdd_19_1 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:04.184 Executor task launch worker for task 54 INFO BlockManager: Found block rdd_19_1 locally 20/01/23 18:54:04.184 Executor task launch worker for task 54 DEBUG BlockManager: Getting local block rdd_19_1 20/01/23 18:54:04.184 Executor task launch worker for task 54 DEBUG BlockManager: Level for block rdd_19_1 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:04.184 Executor task launch worker for task 54 INFO BlockManager: Found block rdd_19_1 locally 20/01/23 18:54:04.184 Executor task launch worker for task 54 DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 3, partitions 1-2 20/01/23 18:54:04.185 Executor task launch worker for task 54 DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 20/01/23 18:54:04.184 Executor task launch worker for task 53 DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 20/01/23 18:54:04.185 Executor task launch worker for task 54 INFO ShuffleBlockFetcherIterator: Getting 3 (707.0 B) non-empty blocks including 3 (707.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 20/01/23 18:54:04.185 Executor task launch worker for task 54 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 20/01/23 18:54:04.185 Executor task launch worker for task 53 INFO ShuffleBlockFetcherIterator: Getting 3 (707.0 B) non-empty blocks including 3 (707.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 20/01/23 18:54:04.185 Executor task launch worker for task 54 DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_3_43_1,0), (shuffle_3_44_1,1), (shuffle_3_45_1,2) 20/01/23 18:54:04.185 Executor task launch worker for task 53 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 1 ms 20/01/23 18:54:04.185 Executor task launch worker for task 53 DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_3_43_0,0), (shuffle_3_44_0,1), (shuffle_3_45_0,2) 20/01/23 18:54:04.186 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_50_3 in memory on 192.168.1.225:35863 (size: 16.0 B, free: 366.2 MiB) 20/01/23 18:54:04.186 Executor task launch worker for task 53 DEBUG ShuffleBlockFetcherIterator: Got local blocks in 1 ms 20/01/23 18:54:04.186 Executor task launch worker for task 56 DEBUG BlockManagerMaster: Updated info of block rdd_50_3 20/01/23 18:54:04.186 Executor task launch worker for task 56 DEBUG BlockManager: Told master about block rdd_50_3 20/01/23 18:54:04.186 Executor task launch worker for task 56 DEBUG BlockManager: Put block rdd_50_3 locally took 15 ms 20/01/23 18:54:04.186 Executor task launch worker for task 54 DEBUG ShuffleBlockFetcherIterator: Got local blocks in 1 ms 20/01/23 18:54:04.186 Executor task launch worker for task 56 DEBUG BlockManager: Putting block rdd_50_3 without replication took 15 ms 20/01/23 18:54:04.186 Executor task launch worker for task 56 DEBUG BlockManager: Getting local block rdd_50_3 20/01/23 18:54:04.186 Executor task launch worker for task 56 DEBUG BlockManager: Level for block rdd_50_3 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:04.187 Executor task launch worker for task 56 INFO Executor: Finished task 3.0 in stage 12.0 (TID 56). 2265 bytes result sent to driver 20/01/23 18:54:04.187 Executor task launch worker for task 56 DEBUG ExecutorMetricsPoller: stageTCMP: (12, 0) -> 3 20/01/23 18:54:04.188 dispatcher-event-loop-3 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_12.0, runningTasks: 3 20/01/23 18:54:04.188 Executor task launch worker for task 53 DEBUG TaskMemoryManager: Task 53 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@6b925ccd 20/01/23 18:54:04.188 Executor task launch worker for task 54 DEBUG TaskMemoryManager: Task 54 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@2683b96f 20/01/23 18:54:04.188 dispatcher-event-loop-3 INFO TaskSetManager: Starting task 4.0 in stage 12.0 (TID 57, 192.168.1.225, executor driver, partition 4, PROCESS_LOCAL, 7433 bytes) 20/01/23 18:54:04.188 task-result-getter-2 INFO TaskSetManager: Finished task 3.0 in stage 12.0 (TID 56) in 27 ms on 192.168.1.225 (executor driver) (1/10) 20/01/23 18:54:04.189 Executor task launch worker for task 53 DEBUG TaskMemoryManager: Task 53 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@40b4c8e9 20/01/23 18:54:04.189 Executor task launch worker for task 54 DEBUG TaskMemoryManager: Task 54 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@438a10b9 20/01/23 18:54:04.189 Executor task launch worker for task 53 DEBUG TaskMemoryManager: Task 53 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@3b4dd12b 20/01/23 18:54:04.190 Executor task launch worker for task 54 DEBUG TaskMemoryManager: Task 54 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@421dea4c 20/01/23 18:54:04.190 Executor task launch worker for task 53 INFO MemoryStore: Block rdd_50_0 stored as values in memory (estimated size 88.0 B, free 366.0 MiB) 20/01/23 18:54:04.190 Executor task launch worker for task 57 INFO Executor: Running task 4.0 in stage 12.0 (TID 57) 20/01/23 18:54:04.190 Executor task launch worker for task 54 INFO MemoryStore: Block rdd_50_1 stored as values in memory (estimated size 88.0 B, free 366.0 MiB) 20/01/23 18:54:04.190 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_50_0 in memory on 192.168.1.225:35863 (size: 88.0 B, free: 366.2 MiB) 20/01/23 18:54:04.190 Executor task launch worker for task 55 DEBUG BlockManagerMaster: Updated info of block rdd_50_2 20/01/23 18:54:04.190 Executor task launch worker for task 55 DEBUG BlockManager: Told master about block rdd_50_2 20/01/23 18:54:04.190 Executor task launch worker for task 55 DEBUG BlockManager: Put block rdd_50_2 locally took 13 ms 20/01/23 18:54:04.191 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_50_1 in memory on 192.168.1.225:35863 (size: 88.0 B, free: 366.2 MiB) 20/01/23 18:54:04.191 Executor task launch worker for task 55 DEBUG BlockManager: Putting block rdd_50_2 without replication took 13 ms 20/01/23 18:54:04.191 Executor task launch worker for task 55 DEBUG BlockManager: Getting local block rdd_50_2 20/01/23 18:54:04.191 Executor task launch worker for task 55 DEBUG BlockManager: Level for block rdd_50_2 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:04.192 Executor task launch worker for task 57 DEBUG ExecutorMetricsPoller: stageTCMP: (12, 0) -> 4 20/01/23 18:54:04.192 Executor task launch worker for task 55 INFO Executor: Finished task 2.0 in stage 12.0 (TID 55). 2308 bytes result sent to driver 20/01/23 18:54:04.192 Executor task launch worker for task 54 DEBUG BlockManagerMaster: Updated info of block rdd_50_1 20/01/23 18:54:04.192 Executor task launch worker for task 54 DEBUG BlockManager: Told master about block rdd_50_1 20/01/23 18:54:04.192 Executor task launch worker for task 54 DEBUG BlockManager: Put block rdd_50_1 locally took 15 ms 20/01/23 18:54:04.192 Executor task launch worker for task 54 DEBUG BlockManager: Putting block rdd_50_1 without replication took 15 ms 20/01/23 18:54:04.192 Executor task launch worker for task 54 DEBUG BlockManager: Getting local block rdd_50_1 20/01/23 18:54:04.192 Executor task launch worker for task 54 DEBUG BlockManager: Level for block rdd_50_1 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:04.193 Executor task launch worker for task 54 INFO Executor: Finished task 1.0 in stage 12.0 (TID 54). 2308 bytes result sent to driver 20/01/23 18:54:04.197 Executor task launch worker for task 57 DEBUG BlockManager: Getting local block rdd_50_4 20/01/23 18:54:04.197 Executor task launch worker for task 57 DEBUG BlockManager: Block rdd_50_4 was not found 20/01/23 18:54:04.197 Executor task launch worker for task 57 DEBUG BlockManager: Getting remote block rdd_50_4 20/01/23 18:54:04.203 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$runJob$5) is now cleaned +++ 20/01/23 18:54:04.203 Executor task launch worker for task 53 DEBUG BlockManagerMaster: Updated info of block rdd_50_0 20/01/23 18:54:04.203 Executor task launch worker for task 53 DEBUG BlockManager: Told master about block rdd_50_0 20/01/23 18:54:04.203 Executor task launch worker for task 53 DEBUG BlockManager: Put block rdd_50_0 locally took 26 ms 20/01/23 18:54:04.204 Executor task launch worker for task 53 DEBUG BlockManager: Putting block rdd_50_0 without replication took 27 ms 20/01/23 18:54:04.204 Executor task launch worker for task 53 DEBUG BlockManager: Getting local block rdd_50_0 20/01/23 18:54:04.204 Executor task launch worker for task 53 DEBUG BlockManager: Level for block rdd_50_0 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:04.204 Thread-4 INFO SparkContext: Starting job: at /usr/lib/python3.6/runpy.py:85 20/01/23 18:54:04.205 Executor task launch worker for task 53 INFO Executor: Finished task 0.0 in stage 12.0 (TID 53). 2308 bytes result sent to driver 20/01/23 18:54:04.205 dispatcher-event-loop-1 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_12.0, runningTasks: 3 20/01/23 18:54:04.206 dispatcher-event-loop-1 INFO TaskSetManager: Starting task 5.0 in stage 12.0 (TID 58, 192.168.1.225, executor driver, partition 5, PROCESS_LOCAL, 7433 bytes) 20/01/23 18:54:04.205 dag-scheduler-event-loop INFO DAGScheduler: Got job 4 ( at /usr/lib/python3.6/runpy.py:85) with 4 output partitions 20/01/23 18:54:04.206 dag-scheduler-event-loop INFO DAGScheduler: Final stage: ResultStage 6 ( at /usr/lib/python3.6/runpy.py:85) 20/01/23 18:54:04.206 Executor task launch worker for task 55 DEBUG ExecutorMetricsPoller: stageTCMP: (12, 0) -> 3 20/01/23 18:54:04.206 dag-scheduler-event-loop INFO DAGScheduler: Parents of final stage: List() 20/01/23 18:54:04.206 task-result-getter-1 INFO TaskSetManager: Finished task 2.0 in stage 12.0 (TID 55) in 46 ms on 192.168.1.225 (executor driver) (2/10) 20/01/23 18:54:04.206 Executor task launch worker for task 58 INFO Executor: Running task 5.0 in stage 12.0 (TID 58) 20/01/23 18:54:04.206 dag-scheduler-event-loop INFO DAGScheduler: Missing parents: List() 20/01/23 18:54:04.207 dispatcher-event-loop-2 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_12.0, runningTasks: 3 20/01/23 18:54:04.207 dag-scheduler-event-loop DEBUG DAGScheduler: submitStage(ResultStage 6 (name= at /usr/lib/python3.6/runpy.py:85;jobs=4)) 20/01/23 18:54:04.207 dispatcher-event-loop-2 INFO TaskSetManager: Starting task 6.0 in stage 12.0 (TID 59, 192.168.1.225, executor driver, partition 6, PROCESS_LOCAL, 7433 bytes) 20/01/23 18:54:04.207 dag-scheduler-event-loop DEBUG DAGScheduler: missing: List() 20/01/23 18:54:04.207 dag-scheduler-event-loop INFO DAGScheduler: Submitting ResultStage 6 (MapPartitionsRDD[24] at at /usr/lib/python3.6/runpy.py:85), which has no missing parents 20/01/23 18:54:04.207 dag-scheduler-event-loop DEBUG DAGScheduler: submitMissingTasks(ResultStage 6) 20/01/23 18:54:04.207 Executor task launch worker for task 59 INFO Executor: Running task 6.0 in stage 12.0 (TID 59) 20/01/23 18:54:04.207 task-result-getter-0 INFO TaskSetManager: Finished task 0.0 in stage 12.0 (TID 53) in 47 ms on 192.168.1.225 (executor driver) (3/10) 20/01/23 18:54:04.207 Executor task launch worker for task 57 DEBUG BlockManager: Block rdd_50_4 is unknown by block manager master 20/01/23 18:54:04.208 Executor task launch worker for task 57 DEBUG BlockManager: Getting local block rdd_19_4 20/01/23 18:54:04.208 Executor task launch worker for task 59 DEBUG ExecutorMetricsPoller: stageTCMP: (12, 0) -> 4 20/01/23 18:54:04.208 Executor task launch worker for task 57 DEBUG BlockManager: Level for block rdd_19_4 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:04.208 Executor task launch worker for task 57 INFO BlockManager: Found block rdd_19_4 locally 20/01/23 18:54:04.208 Executor task launch worker for task 57 DEBUG BlockManager: Getting local block rdd_19_4 20/01/23 18:54:04.208 Executor task launch worker for task 57 DEBUG BlockManager: Level for block rdd_19_4 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:04.208 Executor task launch worker for task 57 INFO BlockManager: Found block rdd_19_4 locally 20/01/23 18:54:04.208 Executor task launch worker for task 57 DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 3, partitions 4-5 20/01/23 18:54:04.208 Executor task launch worker for task 57 DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 20/01/23 18:54:04.209 Executor task launch worker for task 57 INFO ShuffleBlockFetcherIterator: Getting 3 (684.0 B) non-empty blocks including 3 (684.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 20/01/23 18:54:04.209 Executor task launch worker for task 57 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 20/01/23 18:54:04.209 Executor task launch worker for task 57 DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_3_43_4,0), (shuffle_3_44_4,1), (shuffle_3_45_4,2) 20/01/23 18:54:04.210 Executor task launch worker for task 57 DEBUG ShuffleBlockFetcherIterator: Got local blocks in 1 ms 20/01/23 18:54:04.211 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_8 stored as values in memory (estimated size 25.6 KiB, free 366.2 MiB) 20/01/23 18:54:04.211 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_8 locally took 1 ms 20/01/23 18:54:04.211 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_8 without replication took 1 ms 20/01/23 18:54:04.211 Executor task launch worker for task 57 DEBUG TaskMemoryManager: Task 57 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@4451c904 20/01/23 18:54:04.212 Executor task launch worker for task 57 DEBUG TaskMemoryManager: Task 57 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@1959d65c 20/01/23 18:54:04.212 Executor task launch worker for task 57 DEBUG TaskMemoryManager: Task 57 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@380aba86 20/01/23 18:54:04.212 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_8_piece0 stored as bytes in memory (estimated size 10.9 KiB, free 366.2 MiB) 20/01/23 18:54:04.213 Executor task launch worker for task 57 INFO MemoryStore: Block rdd_50_4 stored as values in memory (estimated size 16.0 B, free 366.0 MiB) 20/01/23 18:54:04.213 Executor task launch worker for task 53 DEBUG ExecutorMetricsPoller: stageTCMP: (12, 0) -> 3 20/01/23 18:54:04.213 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_50_4 in memory on 192.168.1.225:35863 (size: 16.0 B, free: 366.2 MiB) 20/01/23 18:54:04.213 Executor task launch worker for task 54 DEBUG ExecutorMetricsPoller: stageTCMP: (12, 0) -> 2 20/01/23 18:54:04.213 Executor task launch worker for task 57 DEBUG BlockManagerMaster: Updated info of block rdd_50_4 20/01/23 18:54:04.214 Executor task launch worker for task 57 DEBUG BlockManager: Told master about block rdd_50_4 20/01/23 18:54:04.214 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added broadcast_8_piece0 in memory on 192.168.1.225:41233 (size: 10.9 KiB, free: 366.3 MiB) 20/01/23 18:54:04.214 Executor task launch worker for task 57 DEBUG BlockManager: Put block rdd_50_4 locally took 6 ms 20/01/23 18:54:04.214 Executor task launch worker for task 57 DEBUG BlockManager: Putting block rdd_50_4 without replication took 6 ms 20/01/23 18:54:04.214 Executor task launch worker for task 57 DEBUG BlockManager: Getting local block rdd_50_4 20/01/23 18:54:04.214 Executor task launch worker for task 58 DEBUG ExecutorMetricsPoller: stageTCMP: (12, 0) -> 3 20/01/23 18:54:04.214 dispatcher-event-loop-0 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_12.0, runningTasks: 3 20/01/23 18:54:04.214 dispatcher-event-loop-0 INFO TaskSetManager: Starting task 7.0 in stage 12.0 (TID 60, 192.168.1.225, executor driver, partition 7, PROCESS_LOCAL, 7433 bytes) 20/01/23 18:54:04.215 task-result-getter-3 INFO TaskSetManager: Finished task 1.0 in stage 12.0 (TID 54) in 55 ms on 192.168.1.225 (executor driver) (4/10) 20/01/23 18:54:04.215 dag-scheduler-event-loop DEBUG BlockManagerMaster: Updated info of block broadcast_8_piece0 20/01/23 18:54:04.216 dag-scheduler-event-loop DEBUG BlockManager: Told master about block broadcast_8_piece0 20/01/23 18:54:04.216 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_8_piece0 locally took 3 ms 20/01/23 18:54:04.216 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_8_piece0 without replication took 3 ms 20/01/23 18:54:04.216 dag-scheduler-event-loop INFO SparkContext: Created broadcast 8 from broadcast at DAGScheduler.scala:1200 20/01/23 18:54:04.214 Executor task launch worker for task 57 DEBUG BlockManager: Level for block rdd_50_4 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:04.217 dag-scheduler-event-loop INFO DAGScheduler: Submitting 4 missing tasks from ResultStage 6 (MapPartitionsRDD[24] at at /usr/lib/python3.6/runpy.py:85) (first 15 tasks are for partitions Vector(0, 1, 2, 3)) 20/01/23 18:54:04.217 dag-scheduler-event-loop INFO TaskSchedulerImpl: Adding task set 6.0 with 4 tasks 20/01/23 18:54:04.217 dag-scheduler-event-loop DEBUG TaskSetManager: Epoch for TaskSet 6.0: 1 20/01/23 18:54:04.217 dag-scheduler-event-loop DEBUG TaskSetManager: Adding pending tasks took 0 ms 20/01/23 18:54:04.217 dag-scheduler-event-loop DEBUG TaskSetManager: Valid locality levels for TaskSet 6.0: NO_PREF, ANY 20/01/23 18:54:04.217 Executor task launch worker for task 57 INFO Executor: Finished task 4.0 in stage 12.0 (TID 57). 2265 bytes result sent to driver 20/01/23 18:54:04.218 Executor task launch worker for task 57 DEBUG ExecutorMetricsPoller: stageTCMP: (12, 0) -> 2 20/01/23 18:54:04.218 dispatcher-event-loop-0 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_6.0, runningTasks: 0 20/01/23 18:54:04.218 dispatcher-event-loop-0 INFO TaskSetManager: Starting task 0.0 in stage 6.0 (TID 20, 192.168.1.225, executor driver, partition 0, PROCESS_LOCAL, 7432 bytes) 20/01/23 18:54:04.218 dispatcher-event-loop-0 INFO TaskSetManager: Starting task 1.0 in stage 6.0 (TID 21, 192.168.1.225, executor driver, partition 1, PROCESS_LOCAL, 7396 bytes) 20/01/23 18:54:04.219 dispatcher-event-loop-0 INFO TaskSetManager: Starting task 2.0 in stage 6.0 (TID 22, 192.168.1.225, executor driver, partition 2, PROCESS_LOCAL, 7411 bytes) 20/01/23 18:54:04.219 Executor task launch worker for task 58 DEBUG BlockManager: Getting local block rdd_50_5 20/01/23 18:54:04.219 Executor task launch worker for task 58 DEBUG BlockManager: Block rdd_50_5 was not found 20/01/23 18:54:04.219 Executor task launch worker for task 58 DEBUG BlockManager: Getting remote block rdd_50_5 20/01/23 18:54:04.220 dispatcher-event-loop-1 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_12.0, runningTasks: 3 20/01/23 18:54:04.220 dispatcher-event-loop-1 INFO TaskSetManager: Starting task 8.0 in stage 12.0 (TID 61, 192.168.1.225, executor driver, partition 8, PROCESS_LOCAL, 7433 bytes) 20/01/23 18:54:04.221 Executor task launch worker for task 60 INFO Executor: Running task 7.0 in stage 12.0 (TID 60) 20/01/23 18:54:04.221 Executor task launch worker for task 58 DEBUG BlockManager: Block rdd_50_5 is unknown by block manager master 20/01/23 18:54:04.221 Executor task launch worker for task 59 DEBUG BlockManager: Getting local block rdd_50_6 20/01/23 18:54:04.221 Executor task launch worker for task 59 DEBUG BlockManager: Block rdd_50_6 was not found 20/01/23 18:54:04.221 Executor task launch worker for task 59 DEBUG BlockManager: Getting remote block rdd_50_6 20/01/23 18:54:04.221 Executor task launch worker for task 59 DEBUG BlockManager: Block rdd_50_6 is unknown by block manager master 20/01/23 18:54:04.221 Executor task launch worker for task 59 DEBUG BlockManager: Getting local block rdd_19_6 20/01/23 18:54:04.221 Executor task launch worker for task 59 DEBUG BlockManager: Level for block rdd_19_6 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:04.221 Executor task launch worker for task 59 INFO BlockManager: Found block rdd_19_6 locally 20/01/23 18:54:04.221 Executor task launch worker for task 59 DEBUG BlockManager: Getting local block rdd_19_6 20/01/23 18:54:04.221 Executor task launch worker for task 59 DEBUG BlockManager: Level for block rdd_19_6 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:04.222 Executor task launch worker for task 59 INFO BlockManager: Found block rdd_19_6 locally 20/01/23 18:54:04.222 Executor task launch worker for task 59 DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 3, partitions 6-7 20/01/23 18:54:04.222 Executor task launch worker for task 59 DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 20/01/23 18:54:04.223 Executor task launch worker for task 59 INFO ShuffleBlockFetcherIterator: Getting 3 (684.0 B) non-empty blocks including 3 (684.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 20/01/23 18:54:04.223 Executor task launch worker for task 59 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 1 ms 20/01/23 18:54:04.223 Executor task launch worker for task 59 DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_3_43_6,0), (shuffle_3_44_6,1), (shuffle_3_45_6,2) 20/01/23 18:54:04.224 Executor task launch worker for task 59 DEBUG ShuffleBlockFetcherIterator: Got local blocks in 1 ms 20/01/23 18:54:04.226 Executor task launch worker for task 59 DEBUG TaskMemoryManager: Task 59 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@7bab9a9 20/01/23 18:54:04.226 Executor task launch worker for task 59 DEBUG TaskMemoryManager: Task 59 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@149a1128 20/01/23 18:54:04.227 Executor task launch worker for task 59 DEBUG TaskMemoryManager: Task 59 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@3ba77898 20/01/23 18:54:04.227 Executor task launch worker for task 59 INFO MemoryStore: Block rdd_50_6 stored as values in memory (estimated size 16.0 B, free 366.0 MiB) 20/01/23 18:54:04.227 task-result-getter-2 INFO TaskSetManager: Finished task 4.0 in stage 12.0 (TID 57) in 32 ms on 192.168.1.225 (executor driver) (5/10) 20/01/23 18:54:04.227 Executor task launch worker for task 61 INFO Executor: Running task 8.0 in stage 12.0 (TID 61) 20/01/23 18:54:04.227 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_50_6 in memory on 192.168.1.225:35863 (size: 16.0 B, free: 366.2 MiB) 20/01/23 18:54:04.227 dispatcher-event-loop-0 INFO TaskSetManager: Starting task 3.0 in stage 6.0 (TID 23, 192.168.1.225, executor driver, partition 3, PROCESS_LOCAL, 7393 bytes) 20/01/23 18:54:04.227 Executor task launch worker for task 58 DEBUG BlockManager: Getting local block rdd_19_5 20/01/23 18:54:04.228 Executor task launch worker for task 58 DEBUG BlockManager: Level for block rdd_19_5 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:04.228 Executor task launch worker for task 58 INFO BlockManager: Found block rdd_19_5 locally 20/01/23 18:54:04.228 Executor task launch worker for task 58 DEBUG BlockManager: Getting local block rdd_19_5 20/01/23 18:54:04.228 Executor task launch worker for task 58 DEBUG BlockManager: Level for block rdd_19_5 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:04.228 Executor task launch worker for task 20 INFO Executor: Running task 0.0 in stage 6.0 (TID 20) 20/01/23 18:54:04.228 Executor task launch worker for task 58 INFO BlockManager: Found block rdd_19_5 locally 20/01/23 18:54:04.228 Executor task launch worker for task 58 DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 3, partitions 5-6 20/01/23 18:54:04.228 Executor task launch worker for task 23 INFO Executor: Running task 3.0 in stage 6.0 (TID 23) 20/01/23 18:54:04.228 Executor task launch worker for task 21 INFO Executor: Running task 1.0 in stage 6.0 (TID 21) 20/01/23 18:54:04.228 Executor task launch worker for task 58 DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 20/01/23 18:54:04.229 Executor task launch worker for task 22 INFO Executor: Running task 2.0 in stage 6.0 (TID 22) 20/01/23 18:54:04.229 Executor task launch worker for task 23 DEBUG ExecutorMetricsPoller: stageTCMP: (6, 0) -> 1 20/01/23 18:54:04.229 Executor task launch worker for task 58 INFO ShuffleBlockFetcherIterator: Getting 3 (684.0 B) non-empty blocks including 3 (684.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 20/01/23 18:54:04.229 Executor task launch worker for task 58 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 20/01/23 18:54:04.229 Executor task launch worker for task 22 DEBUG ExecutorMetricsPoller: stageTCMP: (6, 0) -> 2 20/01/23 18:54:04.229 Executor task launch worker for task 58 DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_3_43_5,0), (shuffle_3_44_5,1), (shuffle_3_45_5,2) 20/01/23 18:54:04.229 Executor task launch worker for task 23 DEBUG BlockManager: Getting local block broadcast_8 20/01/23 18:54:04.230 Executor task launch worker for task 58 DEBUG ShuffleBlockFetcherIterator: Got local blocks in 1 ms 20/01/23 18:54:04.230 Executor task launch worker for task 23 DEBUG BlockManager: Level for block broadcast_8 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:04.231 Executor task launch worker for task 58 DEBUG TaskMemoryManager: Task 58 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@139b9d2 20/01/23 18:54:04.232 Executor task launch worker for task 58 DEBUG TaskMemoryManager: Task 58 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@5d7c147e 20/01/23 18:54:04.232 Executor task launch worker for task 58 DEBUG TaskMemoryManager: Task 58 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@382900a1 20/01/23 18:54:04.232 Executor task launch worker for task 58 INFO MemoryStore: Block rdd_50_5 stored as values in memory (estimated size 16.0 B, free 366.0 MiB) 20/01/23 18:54:04.233 Executor task launch worker for task 60 DEBUG ExecutorMetricsPoller: stageTCMP: (12, 0) -> 3 20/01/23 18:54:04.233 Executor task launch worker for task 61 DEBUG ExecutorMetricsPoller: stageTCMP: (12, 0) -> 4 20/01/23 18:54:04.233 Thread-4 DEBUG Analyzer$ResolveReferences: Resolving 'prediction to prediction#30 20/01/23 18:54:04.234 Executor task launch worker for task 20 DEBUG ExecutorMetricsPoller: stageTCMP: (6, 0) -> 3 20/01/23 18:54:04.237 Executor task launch worker for task 21 DEBUG ExecutorMetricsPoller: stageTCMP: (6, 0) -> 4 20/01/23 18:54:04.237 Executor task launch worker for task 59 DEBUG BlockManagerMaster: Updated info of block rdd_50_6 20/01/23 18:54:04.237 Executor task launch worker for task 59 DEBUG BlockManager: Told master about block rdd_50_6 20/01/23 18:54:04.237 Executor task launch worker for task 59 DEBUG BlockManager: Put block rdd_50_6 locally took 16 ms 20/01/23 18:54:04.237 Executor task launch worker for task 59 DEBUG BlockManager: Putting block rdd_50_6 without replication took 16 ms 20/01/23 18:54:04.239 Executor task launch worker for task 61 DEBUG BlockManager: Getting local block rdd_50_8 20/01/23 18:54:04.239 Executor task launch worker for task 61 DEBUG BlockManager: Block rdd_50_8 was not found 20/01/23 18:54:04.239 Executor task launch worker for task 61 DEBUG BlockManager: Getting remote block rdd_50_8 20/01/23 18:54:04.242 Thread-4 DEBUG Analyzer$ResolveReferences: Resolving 'label to label#29 20/01/23 18:54:04.249 Executor task launch worker for task 59 DEBUG BlockManager: Getting local block rdd_50_6 20/01/23 18:54:04.249 Executor task launch worker for task 59 DEBUG BlockManager: Level for block rdd_50_6 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:04.250 Executor task launch worker for task 59 INFO Executor: Finished task 6.0 in stage 12.0 (TID 59). 2265 bytes result sent to driver 20/01/23 18:54:04.250 Executor task launch worker for task 59 DEBUG ExecutorMetricsPoller: stageTCMP: (12, 0) -> 3 20/01/23 18:54:04.252 Executor task launch worker for task 60 DEBUG BlockManager: Getting local block rdd_50_7 20/01/23 18:54:04.252 Executor task launch worker for task 60 DEBUG BlockManager: Block rdd_50_7 was not found 20/01/23 18:54:04.252 Executor task launch worker for task 60 DEBUG BlockManager: Getting remote block rdd_50_7 20/01/23 18:54:04.253 dispatcher-event-loop-3 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_12.0, runningTasks: 3 20/01/23 18:54:04.253 dispatcher-event-loop-3 INFO TaskSetManager: Starting task 9.0 in stage 12.0 (TID 62, 192.168.1.225, executor driver, partition 9, PROCESS_LOCAL, 7433 bytes) 20/01/23 18:54:04.253 task-result-getter-1 INFO TaskSetManager: Finished task 6.0 in stage 12.0 (TID 59) in 46 ms on 192.168.1.225 (executor driver) (6/10) 20/01/23 18:54:04.254 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_50_5 in memory on 192.168.1.225:35863 (size: 16.0 B, free: 366.2 MiB) 20/01/23 18:54:04.254 Executor task launch worker for task 61 DEBUG BlockManager: Block rdd_50_8 is unknown by block manager master 20/01/23 18:54:04.254 Executor task launch worker for task 58 DEBUG BlockManagerMaster: Updated info of block rdd_50_5 20/01/23 18:54:04.254 Executor task launch worker for task 58 DEBUG BlockManager: Told master about block rdd_50_5 20/01/23 18:54:04.254 Executor task launch worker for task 58 DEBUG BlockManager: Put block rdd_50_5 locally took 33 ms 20/01/23 18:54:04.255 Executor task launch worker for task 61 DEBUG BlockManager: Getting local block rdd_19_8 20/01/23 18:54:04.255 Executor task launch worker for task 58 DEBUG BlockManager: Putting block rdd_50_5 without replication took 33 ms 20/01/23 18:54:04.255 Executor task launch worker for task 58 DEBUG BlockManager: Getting local block rdd_50_5 20/01/23 18:54:04.255 Executor task launch worker for task 61 DEBUG BlockManager: Level for block rdd_19_8 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:04.255 Executor task launch worker for task 58 DEBUG BlockManager: Level for block rdd_50_5 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:04.255 Executor task launch worker for task 61 INFO BlockManager: Found block rdd_19_8 locally 20/01/23 18:54:04.255 Executor task launch worker for task 61 DEBUG BlockManager: Getting local block rdd_19_8 20/01/23 18:54:04.255 Executor task launch worker for task 60 DEBUG BlockManager: Block rdd_50_7 is unknown by block manager master 20/01/23 18:54:04.255 Executor task launch worker for task 61 DEBUG BlockManager: Level for block rdd_19_8 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:04.255 Executor task launch worker for task 61 INFO BlockManager: Found block rdd_19_8 locally 20/01/23 18:54:04.255 Executor task launch worker for task 60 DEBUG BlockManager: Getting local block rdd_19_7 20/01/23 18:54:04.255 Executor task launch worker for task 60 DEBUG BlockManager: Level for block rdd_19_7 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:04.255 Executor task launch worker for task 60 INFO BlockManager: Found block rdd_19_7 locally 20/01/23 18:54:04.255 Executor task launch worker for task 60 DEBUG BlockManager: Getting local block rdd_19_7 20/01/23 18:54:04.255 Executor task launch worker for task 60 DEBUG BlockManager: Level for block rdd_19_7 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:04.255 Executor task launch worker for task 60 INFO BlockManager: Found block rdd_19_7 locally 20/01/23 18:54:04.255 Executor task launch worker for task 60 DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 3, partitions 7-8 20/01/23 18:54:04.256 Executor task launch worker for task 60 DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 20/01/23 18:54:04.256 Executor task launch worker for task 60 INFO ShuffleBlockFetcherIterator: Getting 3 (684.0 B) non-empty blocks including 3 (684.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 20/01/23 18:54:04.256 Executor task launch worker for task 60 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 20/01/23 18:54:04.256 Executor task launch worker for task 60 DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_3_43_7,0), (shuffle_3_44_7,1), (shuffle_3_45_7,2) 20/01/23 18:54:04.257 Executor task launch worker for task 60 DEBUG ShuffleBlockFetcherIterator: Got local blocks in 1 ms 20/01/23 18:54:04.255 Executor task launch worker for task 62 INFO Executor: Running task 9.0 in stage 12.0 (TID 62) 20/01/23 18:54:04.259 Executor task launch worker for task 62 DEBUG ExecutorMetricsPoller: stageTCMP: (12, 0) -> 4 20/01/23 18:54:04.258 Executor task launch worker for task 58 INFO Executor: Finished task 5.0 in stage 12.0 (TID 58). 2265 bytes result sent to driver 20/01/23 18:54:04.255 Executor task launch worker for task 61 DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 3, partitions 8-9 20/01/23 18:54:04.261 dispatcher-event-loop-0 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_12.0, runningTasks: 3 20/01/23 18:54:04.261 Executor task launch worker for task 61 DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 20/01/23 18:54:04.262 Executor task launch worker for task 61 INFO ShuffleBlockFetcherIterator: Getting 3 (684.0 B) non-empty blocks including 3 (684.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 20/01/23 18:54:04.262 Executor task launch worker for task 61 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 20/01/23 18:54:04.262 Executor task launch worker for task 61 DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_3_43_8,0), (shuffle_3_44_8,1), (shuffle_3_45_8,2) 20/01/23 18:54:04.262 Executor task launch worker for task 60 DEBUG TaskMemoryManager: Task 60 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@502af986 20/01/23 18:54:04.263 Executor task launch worker for task 61 DEBUG ShuffleBlockFetcherIterator: Got local blocks in 1 ms 20/01/23 18:54:04.263 Executor task launch worker for task 60 DEBUG TaskMemoryManager: Task 60 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@47c2a503 20/01/23 18:54:04.263 Executor task launch worker for task 60 DEBUG TaskMemoryManager: Task 60 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@684f75ed 20/01/23 18:54:04.264 Executor task launch worker for task 60 INFO MemoryStore: Block rdd_50_7 stored as values in memory (estimated size 16.0 B, free 366.0 MiB) 20/01/23 18:54:04.261 Executor task launch worker for task 58 DEBUG ExecutorMetricsPoller: stageTCMP: (12, 0) -> 3 20/01/23 18:54:04.261 dispatcher-event-loop-0 DEBUG TaskSetManager: No tasks for locality level PROCESS_LOCAL, so moving to locality level NODE_LOCAL 20/01/23 18:54:04.264 dispatcher-event-loop-0 DEBUG TaskSetManager: No tasks for locality level NODE_LOCAL, so moving to locality level ANY 20/01/23 18:54:04.264 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_50_7 in memory on 192.168.1.225:35863 (size: 16.0 B, free: 366.2 MiB) 20/01/23 18:54:04.264 Executor task launch worker for task 61 DEBUG TaskMemoryManager: Task 61 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@6e95abe5 20/01/23 18:54:04.265 Executor task launch worker for task 61 DEBUG TaskMemoryManager: Task 61 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@2bc5c1e1 20/01/23 18:54:04.265 task-result-getter-0 INFO TaskSetManager: Finished task 5.0 in stage 12.0 (TID 58) in 60 ms on 192.168.1.225 (executor driver) (7/10) 20/01/23 18:54:04.265 Executor task launch worker for task 61 DEBUG TaskMemoryManager: Task 61 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@1e1727aa 20/01/23 18:54:04.266 Executor task launch worker for task 61 INFO MemoryStore: Block rdd_50_8 stored as values in memory (estimated size 16.0 B, free 366.0 MiB) 20/01/23 18:54:04.266 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_50_8 in memory on 192.168.1.225:35863 (size: 16.0 B, free: 366.2 MiB) 20/01/23 18:54:04.268 Executor task launch worker for task 61 DEBUG BlockManagerMaster: Updated info of block rdd_50_8 20/01/23 18:54:04.268 Executor task launch worker for task 61 DEBUG BlockManager: Told master about block rdd_50_8 20/01/23 18:54:04.268 Executor task launch worker for task 61 DEBUG BlockManager: Put block rdd_50_8 locally took 13 ms 20/01/23 18:54:04.268 Executor task launch worker for task 62 DEBUG BlockManager: Getting local block rdd_50_9 20/01/23 18:54:04.268 Executor task launch worker for task 61 DEBUG BlockManager: Putting block rdd_50_8 without replication took 13 ms 20/01/23 18:54:04.268 Executor task launch worker for task 61 DEBUG BlockManager: Getting local block rdd_50_8 20/01/23 18:54:04.268 Executor task launch worker for task 61 DEBUG BlockManager: Level for block rdd_50_8 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:04.268 Executor task launch worker for task 60 DEBUG BlockManagerMaster: Updated info of block rdd_50_7 20/01/23 18:54:04.268 Executor task launch worker for task 60 DEBUG BlockManager: Told master about block rdd_50_7 20/01/23 18:54:04.269 Executor task launch worker for task 60 DEBUG BlockManager: Put block rdd_50_7 locally took 13 ms 20/01/23 18:54:04.269 Executor task launch worker for task 60 DEBUG BlockManager: Putting block rdd_50_7 without replication took 13 ms 20/01/23 18:54:04.269 Executor task launch worker for task 60 DEBUG BlockManager: Getting local block rdd_50_7 20/01/23 18:54:04.269 Executor task launch worker for task 60 DEBUG BlockManager: Level for block rdd_50_7 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:04.269 Executor task launch worker for task 61 INFO Executor: Finished task 8.0 in stage 12.0 (TID 61). 2265 bytes result sent to driver 20/01/23 18:54:04.270 Executor task launch worker for task 60 INFO Executor: Finished task 7.0 in stage 12.0 (TID 60). 2265 bytes result sent to driver 20/01/23 18:54:04.268 Executor task launch worker for task 62 DEBUG BlockManager: Block rdd_50_9 was not found 20/01/23 18:54:04.270 Executor task launch worker for task 62 DEBUG BlockManager: Getting remote block rdd_50_9 20/01/23 18:54:04.270 dispatcher-event-loop-2 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_12.0, runningTasks: 2 20/01/23 18:54:04.270 Executor task launch worker for task 61 DEBUG ExecutorMetricsPoller: stageTCMP: (12, 0) -> 2 20/01/23 18:54:04.270 task-result-getter-3 INFO TaskSetManager: Finished task 8.0 in stage 12.0 (TID 61) in 50 ms on 192.168.1.225 (executor driver) (8/10) 20/01/23 18:54:04.270 Executor task launch worker for task 62 DEBUG BlockManager: Block rdd_50_9 is unknown by block manager master 20/01/23 18:54:04.270 Executor task launch worker for task 62 DEBUG BlockManager: Getting local block rdd_19_9 20/01/23 18:54:04.271 Executor task launch worker for task 62 DEBUG BlockManager: Level for block rdd_19_9 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:04.271 Executor task launch worker for task 62 INFO BlockManager: Found block rdd_19_9 locally 20/01/23 18:54:04.271 Executor task launch worker for task 62 DEBUG BlockManager: Getting local block rdd_19_9 20/01/23 18:54:04.271 Executor task launch worker for task 62 DEBUG BlockManager: Level for block rdd_19_9 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:04.271 Executor task launch worker for task 62 INFO BlockManager: Found block rdd_19_9 locally 20/01/23 18:54:04.271 Executor task launch worker for task 62 DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 3, partitions 9-10 20/01/23 18:54:04.271 Executor task launch worker for task 62 DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 20/01/23 18:54:04.272 Executor task launch worker for task 62 INFO ShuffleBlockFetcherIterator: Getting 3 (684.0 B) non-empty blocks including 3 (684.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 20/01/23 18:54:04.272 Executor task launch worker for task 62 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 20/01/23 18:54:04.272 Executor task launch worker for task 62 DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_3_43_9,0), (shuffle_3_44_9,1), (shuffle_3_45_9,2) 20/01/23 18:54:04.272 Executor task launch worker for task 62 DEBUG ShuffleBlockFetcherIterator: Got local blocks in 1 ms 20/01/23 18:54:04.274 Executor task launch worker for task 62 DEBUG TaskMemoryManager: Task 62 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@6238b51f 20/01/23 18:54:04.274 Executor task launch worker for task 62 DEBUG TaskMemoryManager: Task 62 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@42796427 20/01/23 18:54:04.275 Executor task launch worker for task 62 DEBUG TaskMemoryManager: Task 62 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@39b5e90a 20/01/23 18:54:04.275 Executor task launch worker for task 62 INFO MemoryStore: Block rdd_50_9 stored as values in memory (estimated size 16.0 B, free 366.0 MiB) 20/01/23 18:54:04.289 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_50_9 in memory on 192.168.1.225:35863 (size: 16.0 B, free: 366.2 MiB) 20/01/23 18:54:04.290 Executor task launch worker for task 60 DEBUG ExecutorMetricsPoller: stageTCMP: (12, 0) -> 1 20/01/23 18:54:04.290 Executor task launch worker for task 62 DEBUG BlockManagerMaster: Updated info of block rdd_50_9 20/01/23 18:54:04.290 Executor task launch worker for task 62 DEBUG BlockManager: Told master about block rdd_50_9 20/01/23 18:54:04.290 Executor task launch worker for task 62 DEBUG BlockManager: Put block rdd_50_9 locally took 19 ms 20/01/23 18:54:04.290 dispatcher-event-loop-3 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_12.0, runningTasks: 1 20/01/23 18:54:04.290 Executor task launch worker for task 62 DEBUG BlockManager: Putting block rdd_50_9 without replication took 19 ms 20/01/23 18:54:04.290 Executor task launch worker for task 62 DEBUG BlockManager: Getting local block rdd_50_9 20/01/23 18:54:04.290 Executor task launch worker for task 62 DEBUG BlockManager: Level for block rdd_50_9 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:04.291 task-result-getter-2 INFO TaskSetManager: Finished task 7.0 in stage 12.0 (TID 60) in 77 ms on 192.168.1.225 (executor driver) (9/10) 20/01/23 18:54:04.291 Executor task launch worker for task 62 INFO Executor: Finished task 9.0 in stage 12.0 (TID 62). 2265 bytes result sent to driver 20/01/23 18:54:04.291 Executor task launch worker for task 62 DEBUG ExecutorMetricsPoller: removing (12, 0) from stageTCMP 20/01/23 18:54:04.292 dispatcher-event-loop-1 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_12.0, runningTasks: 0 20/01/23 18:54:04.292 task-result-getter-1 INFO TaskSetManager: Finished task 9.0 in stage 12.0 (TID 62) in 39 ms on 192.168.1.225 (executor driver) (10/10) 20/01/23 18:54:04.292 task-result-getter-1 INFO TaskSchedulerImpl: Removed TaskSet 12.0, whose tasks have all completed, from pool 20/01/23 18:54:04.293 dag-scheduler-event-loop INFO DAGScheduler: ResultStage 12 (count at ALS.scala:1048) finished in 0.159 s 20/01/23 18:54:04.293 dag-scheduler-event-loop DEBUG DAGScheduler: After removal of stage 8, remaining stages = 5 20/01/23 18:54:04.293 dag-scheduler-event-loop DEBUG DAGScheduler: After removal of stage 11, remaining stages = 4 20/01/23 18:54:04.293 dag-scheduler-event-loop DEBUG DAGScheduler: After removal of stage 7, remaining stages = 3 20/01/23 18:54:04.294 dag-scheduler-event-loop DEBUG DAGScheduler: After removal of stage 10, remaining stages = 2 20/01/23 18:54:04.294 dag-scheduler-event-loop DEBUG DAGScheduler: After removal of stage 9, remaining stages = 1 20/01/23 18:54:04.294 dag-scheduler-event-loop DEBUG DAGScheduler: After removal of stage 12, remaining stages = 0 20/01/23 18:54:04.294 dag-scheduler-event-loop INFO DAGScheduler: Job 3 is finished. Cancelling potential speculative or zombie tasks for this job 20/01/23 18:54:04.294 dag-scheduler-event-loop INFO TaskSchedulerImpl: Killing all running tasks in stage 12: Stage finished 20/01/23 18:54:04.294 Thread-4 INFO DAGScheduler: Job 3 finished: count at ALS.scala:1048, took 0.736379 s 20/01/23 18:54:04.297 Thread-4 INFO MapPartitionsRDD: Removing RDD 19 from persistence list 20/01/23 18:54:04.306 Thread-4 INFO MapPartitionsRDD: Removing RDD 20 from persistence list 20/01/23 18:54:04.307 Executor task launch worker for task 23 DEBUG BlockManager: Getting local block broadcast_7 20/01/23 18:54:04.307 Executor task launch worker for task 23 DEBUG BlockManager: Level for block broadcast_7 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:04.307 Thread-4 INFO MapPartitionsRDD: Removing RDD 25 from persistence list 20/01/23 18:54:04.308 Thread-4 INFO MapPartitionsRDD: Removing RDD 16 from persistence list 20/01/23 18:54:04.309 Executor task launch worker for task 23 INFO PythonRunner: Times: total = 64, boot = -1094, init = 1158, finish = 0 20/01/23 18:54:04.322 Thread-4 DEBUG CoGroupedRDD: Adding one-to-one dependency with MapPartitionsRDD[51] at mapValues at ALS.scala:1038 20/01/23 18:54:04.322 Thread-4 DEBUG CoGroupedRDD: Adding one-to-one dependency with MapPartitionsRDD[36] at mapValues at ALS.scala:1688 20/01/23 18:54:04.322 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$count$1$adapted 20/01/23 18:54:04.324 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$count$1$adapted) is now cleaned +++ 20/01/23 18:54:04.325 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$runJob$5 20/01/23 18:54:04.327 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$runJob$5) is now cleaned +++ 20/01/23 18:54:04.327 Thread-4 INFO SparkContext: Starting job: count at ALS.scala:1053 20/01/23 18:54:04.330 dag-scheduler-event-loop INFO DAGScheduler: Got job 4 (count at ALS.scala:1053) with 10 output partitions 20/01/23 18:54:04.330 dag-scheduler-event-loop INFO DAGScheduler: Final stage: ResultStage 17 (count at ALS.scala:1053) 20/01/23 18:54:04.330 dag-scheduler-event-loop INFO DAGScheduler: Parents of final stage: List(ShuffleMapStage 16, ShuffleMapStage 14) 20/01/23 18:54:04.331 Executor task launch worker for task 21 INFO PythonRunner: Times: total = 87, boot = -1078, init = 1165, finish = 0 20/01/23 18:54:04.331 block-manager-slave-async-thread-pool-6 DEBUG BlockManagerSlaveEndpoint: removing RDD 19 20/01/23 18:54:04.332 block-manager-slave-async-thread-pool-6 INFO BlockManager: Removing RDD 19 20/01/23 18:54:04.332 dag-scheduler-event-loop INFO DAGScheduler: Missing parents: List() 20/01/23 18:54:04.332 dag-scheduler-event-loop DEBUG DAGScheduler: submitStage(ResultStage 17 (name=count at ALS.scala:1053;jobs=4)) 20/01/23 18:54:04.333 dag-scheduler-event-loop DEBUG DAGScheduler: missing: List() 20/01/23 18:54:04.333 dag-scheduler-event-loop INFO DAGScheduler: Submitting ResultStage 17 (itemFactors MapPartitionsRDD[55] at mapPartitions at ALS.scala:1040), which has no missing parents 20/01/23 18:54:04.333 dag-scheduler-event-loop DEBUG DAGScheduler: submitMissingTasks(ResultStage 17) 20/01/23 18:54:04.334 block-manager-slave-async-thread-pool-6 DEBUG BlockManager: Removing block rdd_19_7 20/01/23 18:54:04.334 block-manager-slave-async-thread-pool-6 DEBUG MemoryStore: Block rdd_19_7 of size 16 dropped from memory (free 383758030) 20/01/23 18:54:04.334 block-manager-slave-async-thread-pool-6 DEBUG BlockManager: Removing block rdd_19_6 20/01/23 18:54:04.334 block-manager-slave-async-thread-pool-6 DEBUG MemoryStore: Block rdd_19_6 of size 16 dropped from memory (free 383758046) 20/01/23 18:54:04.334 block-manager-slave-async-thread-pool-6 DEBUG BlockManager: Removing block rdd_19_0 20/01/23 18:54:04.334 block-manager-slave-async-thread-pool-6 DEBUG MemoryStore: Block rdd_19_0 of size 192 dropped from memory (free 383758238) 20/01/23 18:54:04.334 block-manager-slave-async-thread-pool-6 DEBUG BlockManager: Removing block rdd_19_9 20/01/23 18:54:04.335 block-manager-slave-async-thread-pool-6 DEBUG MemoryStore: Block rdd_19_9 of size 16 dropped from memory (free 383758254) 20/01/23 18:54:04.335 block-manager-slave-async-thread-pool-6 DEBUG BlockManager: Removing block rdd_19_4 20/01/23 18:54:04.335 block-manager-slave-async-thread-pool-6 DEBUG MemoryStore: Block rdd_19_4 of size 16 dropped from memory (free 383758270) 20/01/23 18:54:04.335 block-manager-slave-async-thread-pool-6 DEBUG BlockManager: Removing block rdd_19_5 20/01/23 18:54:04.335 block-manager-slave-async-thread-pool-6 DEBUG MemoryStore: Block rdd_19_5 of size 16 dropped from memory (free 383758286) 20/01/23 18:54:04.335 block-manager-slave-async-thread-pool-6 DEBUG BlockManager: Removing block rdd_19_1 20/01/23 18:54:04.335 block-manager-slave-async-thread-pool-6 DEBUG MemoryStore: Block rdd_19_1 of size 192 dropped from memory (free 383758478) 20/01/23 18:54:04.335 block-manager-slave-async-thread-pool-6 DEBUG BlockManager: Removing block rdd_19_2 20/01/23 18:54:04.335 block-manager-slave-async-thread-pool-6 DEBUG MemoryStore: Block rdd_19_2 of size 192 dropped from memory (free 383758670) 20/01/23 18:54:04.335 block-manager-slave-async-thread-pool-6 DEBUG BlockManager: Removing block rdd_19_8 20/01/23 18:54:04.335 block-manager-slave-async-thread-pool-6 DEBUG MemoryStore: Block rdd_19_8 of size 16 dropped from memory (free 383758686) 20/01/23 18:54:04.335 block-manager-slave-async-thread-pool-6 DEBUG BlockManager: Removing block rdd_19_3 20/01/23 18:54:04.335 block-manager-slave-async-thread-pool-6 DEBUG MemoryStore: Block rdd_19_3 of size 16 dropped from memory (free 383758702) 20/01/23 18:54:04.353 Executor task launch worker for task 23 INFO Executor: Finished task 3.0 in stage 6.0 (TID 23). 2120 bytes result sent to driver 20/01/23 18:54:04.353 Executor task launch worker for task 23 DEBUG ExecutorMetricsPoller: stageTCMP: (6, 0) -> 3 20/01/23 18:54:04.353 dispatcher-event-loop-2 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_6.0, runningTasks: 3 20/01/23 18:54:04.353 dispatcher-event-loop-2 DEBUG TaskSetManager: No tasks for locality level NO_PREF, so moving to locality level ANY 20/01/23 18:54:04.354 task-result-getter-1 INFO TaskSetManager: Finished task 3.0 in stage 6.0 (TID 23) in 135 ms on 192.168.1.225 (executor driver) (1/4) 20/01/23 18:54:04.355 Executor task launch worker for task 21 INFO Executor: Finished task 1.0 in stage 6.0 (TID 21). 2111 bytes result sent to driver 20/01/23 18:54:04.355 Executor task launch worker for task 21 DEBUG ExecutorMetricsPoller: stageTCMP: (6, 0) -> 2 20/01/23 18:54:04.355 dispatcher-event-loop-3 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_6.0, runningTasks: 2 20/01/23 18:54:04.356 task-result-getter-0 INFO TaskSetManager: Finished task 1.0 in stage 6.0 (TID 21) in 138 ms on 192.168.1.225 (executor driver) (2/4) 20/01/23 18:54:04.377 block-manager-slave-async-thread-pool-7 DEBUG BlockManagerSlaveEndpoint: removing RDD 20 20/01/23 18:54:04.377 block-manager-slave-async-thread-pool-7 INFO BlockManager: Removing RDD 20 20/01/23 18:54:04.377 block-manager-slave-async-thread-pool-7 DEBUG BlockManager: Removing block rdd_20_6 20/01/23 18:54:04.377 block-manager-slave-async-thread-pool-7 DEBUG MemoryStore: Block rdd_20_6 of size 16 dropped from memory (free 383758718) 20/01/23 18:54:04.378 block-manager-slave-async-thread-pool-7 DEBUG BlockManager: Removing block rdd_20_9 20/01/23 18:54:04.378 block-manager-slave-async-thread-pool-7 DEBUG MemoryStore: Block rdd_20_9 of size 16 dropped from memory (free 383758734) 20/01/23 18:54:04.378 block-manager-slave-async-thread-pool-7 DEBUG BlockManager: Removing block rdd_20_5 20/01/23 18:54:04.378 block-manager-slave-async-thread-pool-7 DEBUG MemoryStore: Block rdd_20_5 of size 16 dropped from memory (free 383758750) 20/01/23 18:54:04.378 block-manager-slave-async-thread-pool-7 DEBUG BlockManager: Removing block rdd_20_3 20/01/23 18:54:04.378 block-manager-slave-async-thread-pool-7 DEBUG MemoryStore: Block rdd_20_3 of size 16 dropped from memory (free 383758766) 20/01/23 18:54:04.378 block-manager-slave-async-thread-pool-7 DEBUG BlockManager: Removing block rdd_20_2 20/01/23 18:54:04.378 block-manager-slave-async-thread-pool-7 DEBUG MemoryStore: Block rdd_20_2 of size 296 dropped from memory (free 383759062) 20/01/23 18:54:04.378 block-manager-slave-async-thread-pool-7 DEBUG BlockManager: Removing block rdd_20_7 20/01/23 18:54:04.378 block-manager-slave-async-thread-pool-7 DEBUG MemoryStore: Block rdd_20_7 of size 16 dropped from memory (free 383759078) 20/01/23 18:54:04.378 block-manager-slave-async-thread-pool-7 DEBUG BlockManager: Removing block rdd_20_1 20/01/23 18:54:04.378 block-manager-slave-async-thread-pool-7 DEBUG MemoryStore: Block rdd_20_1 of size 296 dropped from memory (free 383759374) 20/01/23 18:54:04.378 block-manager-slave-async-thread-pool-7 DEBUG BlockManager: Removing block rdd_20_0 20/01/23 18:54:04.378 block-manager-slave-async-thread-pool-7 DEBUG MemoryStore: Block rdd_20_0 of size 296 dropped from memory (free 383759670) 20/01/23 18:54:04.379 block-manager-slave-async-thread-pool-7 DEBUG BlockManager: Removing block rdd_20_4 20/01/23 18:54:04.379 block-manager-slave-async-thread-pool-7 DEBUG MemoryStore: Block rdd_20_4 of size 16 dropped from memory (free 383759686) 20/01/23 18:54:04.379 block-manager-slave-async-thread-pool-7 DEBUG BlockManager: Removing block rdd_20_8 20/01/23 18:54:04.379 block-manager-slave-async-thread-pool-7 DEBUG MemoryStore: Block rdd_20_8 of size 16 dropped from memory (free 383759702) 20/01/23 18:54:04.380 Executor task launch worker for task 22 INFO PythonRunner: Times: total = 121, boot = -1140, init = 1261, finish = 0 20/01/23 18:54:04.382 Executor task launch worker for task 22 INFO Executor: Finished task 2.0 in stage 6.0 (TID 22). 2124 bytes result sent to driver 20/01/23 18:54:04.382 Executor task launch worker for task 22 DEBUG ExecutorMetricsPoller: stageTCMP: (6, 0) -> 1 20/01/23 18:54:04.381 Thread-4 DEBUG WholeStageCodegenExec: /* 001 */ public Object generate(Object[] references) { /* 002 */ return new GeneratedIteratorForCodegenStage1(references); /* 003 */ } /* 004 */ /* 005 */ // codegenStageId=1 /* 006 */ final class GeneratedIteratorForCodegenStage1 extends org.apache.spark.sql.execution.BufferedRowIterator { /* 007 */ private Object[] references; /* 008 */ private scala.collection.Iterator[] inputs; /* 009 */ private scala.collection.Iterator rdd_input_0; /* 010 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[] rdd_mutableStateArray_0 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[2]; /* 011 */ /* 012 */ public GeneratedIteratorForCodegenStage1(Object[] references) { /* 013 */ this.references = references; /* 014 */ } /* 015 */ /* 016 */ public void init(int index, scala.collection.Iterator[] inputs) { /* 017 */ partitionIndex = index; /* 018 */ this.inputs = inputs; /* 019 */ rdd_input_0 = inputs[0]; /* 020 */ rdd_mutableStateArray_0[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(2, 0); /* 021 */ rdd_mutableStateArray_0[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(3, 0); /* 022 */ /* 023 */ } /* 024 */ /* 025 */ protected void processNext() throws java.io.IOException { /* 026 */ while ( rdd_input_0.hasNext()) { /* 027 */ InternalRow rdd_row_0 = (InternalRow) rdd_input_0.next(); /* 028 */ ((org.apache.spark.sql.execution.metric.SQLMetric) references[0] /* numOutputRows */).add(1); /* 029 */ boolean rdd_isNull_1 = rdd_row_0.isNullAt(1); /* 030 */ double rdd_value_1 = rdd_isNull_1 ? /* 031 */ -1.0 : (rdd_row_0.getDouble(1)); /* 032 */ boolean rdd_isNull_0 = rdd_row_0.isNullAt(0); /* 033 */ double rdd_value_0 = rdd_isNull_0 ? /* 034 */ -1.0 : (rdd_row_0.getDouble(0)); /* 035 */ rdd_mutableStateArray_0[1].reset(); /* 036 */ /* 037 */ rdd_mutableStateArray_0[1].zeroOutNullBytes(); /* 038 */ /* 039 */ if (rdd_isNull_1) { /* 040 */ rdd_mutableStateArray_0[1].setNullAt(0); /* 041 */ } else { /* 042 */ rdd_mutableStateArray_0[1].write(0, rdd_value_1); /* 043 */ } /* 044 */ /* 045 */ if (rdd_isNull_0) { /* 046 */ rdd_mutableStateArray_0[1].setNullAt(1); /* 047 */ } else { /* 048 */ rdd_mutableStateArray_0[1].write(1, rdd_value_0); /* 049 */ } /* 050 */ /* 051 */ rdd_mutableStateArray_0[1].write(2, 1.0D); /* 052 */ append((rdd_mutableStateArray_0[1].getRow())); /* 053 */ if (shouldStop()) return; /* 054 */ } /* 055 */ } /* 056 */ /* 057 */ } 20/01/23 18:54:04.382 Executor task launch worker for task 20 INFO PythonRunner: Times: total = 144, boot = -1028, init = 1172, finish = 0 20/01/23 18:54:04.383 dispatcher-event-loop-1 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_6.0, runningTasks: 1 20/01/23 18:54:04.384 task-result-getter-2 INFO TaskSetManager: Finished task 2.0 in stage 6.0 (TID 22) in 166 ms on 192.168.1.225 (executor driver) (3/4) 20/01/23 18:54:04.385 Executor task launch worker for task 20 INFO Executor: Finished task 0.0 in stage 6.0 (TID 20). 2170 bytes result sent to driver 20/01/23 18:54:04.385 Executor task launch worker for task 20 DEBUG ExecutorMetricsPoller: removing (6, 0) from stageTCMP 20/01/23 18:54:04.386 dispatcher-event-loop-0 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_6.0, runningTasks: 0 20/01/23 18:54:04.387 task-result-getter-3 INFO TaskSetManager: Finished task 0.0 in stage 6.0 (TID 20) in 169 ms on 192.168.1.225 (executor driver) (4/4) 20/01/23 18:54:04.387 task-result-getter-3 INFO TaskSchedulerImpl: Removed TaskSet 6.0, whose tasks have all completed, from pool 20/01/23 18:54:04.391 dag-scheduler-event-loop INFO DAGScheduler: ResultStage 6 ( at /usr/lib/python3.6/runpy.py:85) finished in 0.183 s 20/01/23 18:54:04.391 dag-scheduler-event-loop DEBUG DAGScheduler: After removal of stage 6, remaining stages = 0 20/01/23 18:54:04.391 dag-scheduler-event-loop INFO DAGScheduler: Job 4 is finished. Cancelling potential speculative or zombie tasks for this job 20/01/23 18:54:04.391 dag-scheduler-event-loop INFO TaskSchedulerImpl: Killing all running tasks in stage 6: Stage finished 20/01/23 18:54:04.392 Thread-4 INFO DAGScheduler: Job 4 finished: at /usr/lib/python3.6/runpy.py:85, took 0.187191 s 20/01/23 18:54:04.395 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_9 stored as values in memory (estimated size 34.6 KiB, free 365.9 MiB) 20/01/23 18:54:04.395 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_9 locally took 0 ms 20/01/23 18:54:04.395 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_9 without replication took 0 ms 20/01/23 18:54:04.396 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_9_piece0 stored as bytes in memory (estimated size 15.9 KiB, free 365.9 MiB) 20/01/23 18:54:04.397 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added broadcast_9_piece0 in memory on 192.168.1.225:35863 (size: 15.9 KiB, free: 366.2 MiB) 20/01/23 18:54:04.397 dag-scheduler-event-loop DEBUG BlockManagerMaster: Updated info of block broadcast_9_piece0 20/01/23 18:54:04.397 dag-scheduler-event-loop DEBUG BlockManager: Told master about block broadcast_9_piece0 20/01/23 18:54:04.397 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_9_piece0 locally took 1 ms 20/01/23 18:54:04.397 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_9_piece0 without replication took 1 ms 20/01/23 18:54:04.397 dag-scheduler-event-loop INFO SparkContext: Created broadcast 9 from broadcast at DAGScheduler.scala:1200 20/01/23 18:54:04.398 dag-scheduler-event-loop INFO DAGScheduler: Submitting 10 missing tasks from ResultStage 17 (itemFactors MapPartitionsRDD[55] at mapPartitions at ALS.scala:1040) (first 15 tasks are for partitions Vector(0, 1, 2, 3, 4, 5, 6, 7, 8, 9)) 20/01/23 18:54:04.398 dag-scheduler-event-loop INFO TaskSchedulerImpl: Adding task set 17.0 with 10 tasks 20/01/23 18:54:04.398 dag-scheduler-event-loop DEBUG TaskSetManager: Epoch for TaskSet 17.0: 5 20/01/23 18:54:04.399 dag-scheduler-event-loop DEBUG TaskSetManager: Adding pending tasks took 0 ms 20/01/23 18:54:04.399 dag-scheduler-event-loop DEBUG TaskSetManager: Valid locality levels for TaskSet 17.0: PROCESS_LOCAL, NODE_LOCAL, ANY 20/01/23 18:54:04.399 dispatcher-event-loop-0 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_17.0, runningTasks: 0 20/01/23 18:54:04.400 dispatcher-event-loop-0 INFO TaskSetManager: Starting task 0.0 in stage 17.0 (TID 63, 192.168.1.225, executor driver, partition 0, PROCESS_LOCAL, 7433 bytes) 20/01/23 18:54:04.400 dispatcher-event-loop-0 INFO TaskSetManager: Starting task 1.0 in stage 17.0 (TID 64, 192.168.1.225, executor driver, partition 1, PROCESS_LOCAL, 7433 bytes) 20/01/23 18:54:04.400 dispatcher-event-loop-0 INFO TaskSetManager: Starting task 2.0 in stage 17.0 (TID 65, 192.168.1.225, executor driver, partition 2, PROCESS_LOCAL, 7433 bytes) 20/01/23 18:54:04.400 dispatcher-event-loop-0 INFO TaskSetManager: Starting task 3.0 in stage 17.0 (TID 66, 192.168.1.225, executor driver, partition 3, PROCESS_LOCAL, 7433 bytes) 20/01/23 18:54:04.401 Executor task launch worker for task 63 INFO Executor: Running task 0.0 in stage 17.0 (TID 63) 20/01/23 18:54:04.401 Executor task launch worker for task 66 INFO Executor: Running task 3.0 in stage 17.0 (TID 66) 20/01/23 18:54:04.401 Executor task launch worker for task 64 INFO Executor: Running task 1.0 in stage 17.0 (TID 64) 20/01/23 18:54:04.384 Thread-4 DEBUG CodeGenerator: /* 001 */ public Object generate(Object[] references) { /* 002 */ return new GeneratedIteratorForCodegenStage1(references); /* 003 */ } /* 004 */ /* 005 */ // codegenStageId=1 /* 006 */ final class GeneratedIteratorForCodegenStage1 extends org.apache.spark.sql.execution.BufferedRowIterator { /* 007 */ private Object[] references; /* 008 */ private scala.collection.Iterator[] inputs; /* 009 */ private scala.collection.Iterator rdd_input_0; /* 010 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[] rdd_mutableStateArray_0 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[2]; /* 011 */ /* 012 */ public GeneratedIteratorForCodegenStage1(Object[] references) { /* 013 */ this.references = references; /* 014 */ } /* 015 */ /* 016 */ public void init(int index, scala.collection.Iterator[] inputs) { /* 017 */ partitionIndex = index; /* 018 */ this.inputs = inputs; /* 019 */ rdd_input_0 = inputs[0]; /* 020 */ rdd_mutableStateArray_0[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(2, 0); /* 021 */ rdd_mutableStateArray_0[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(3, 0); /* 022 */ /* 023 */ } /* 024 */ /* 025 */ protected void processNext() throws java.io.IOException { /* 026 */ while ( rdd_input_0.hasNext()) { /* 027 */ InternalRow rdd_row_0 = (InternalRow) rdd_input_0.next(); /* 028 */ ((org.apache.spark.sql.execution.metric.SQLMetric) references[0] /* numOutputRows */).add(1); /* 029 */ boolean rdd_isNull_1 = rdd_row_0.isNullAt(1); /* 030 */ double rdd_value_1 = rdd_isNull_1 ? /* 031 */ -1.0 : (rdd_row_0.getDouble(1)); /* 032 */ boolean rdd_isNull_0 = rdd_row_0.isNullAt(0); /* 033 */ double rdd_value_0 = rdd_isNull_0 ? /* 034 */ -1.0 : (rdd_row_0.getDouble(0)); /* 035 */ rdd_mutableStateArray_0[1].reset(); /* 036 */ /* 037 */ rdd_mutableStateArray_0[1].zeroOutNullBytes(); /* 038 */ /* 039 */ if (rdd_isNull_1) { /* 040 */ rdd_mutableStateArray_0[1].setNullAt(0); /* 041 */ } else { /* 042 */ rdd_mutableStateArray_0[1].write(0, rdd_value_1); /* 043 */ } /* 044 */ /* 045 */ if (rdd_isNull_0) { /* 046 */ rdd_mutableStateArray_0[1].setNullAt(1); /* 047 */ } else { /* 048 */ rdd_mutableStateArray_0[1].write(1, rdd_value_0); /* 049 */ } /* 050 */ /* 051 */ rdd_mutableStateArray_0[1].write(2, 1.0D); /* 052 */ append((rdd_mutableStateArray_0[1].getRow())); /* 053 */ if (shouldStop()) return; /* 054 */ } /* 055 */ } /* 056 */ /* 057 */ } 20/01/23 18:54:04.401 Executor task launch worker for task 63 DEBUG ExecutorMetricsPoller: stageTCMP: (17, 0) -> 1 20/01/23 18:54:04.402 Executor task launch worker for task 65 INFO Executor: Running task 2.0 in stage 17.0 (TID 65) 20/01/23 18:54:04.402 Executor task launch worker for task 63 DEBUG BlockManager: Getting local block broadcast_9 20/01/23 18:54:04.402 Executor task launch worker for task 63 DEBUG BlockManager: Level for block broadcast_9 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:04.402 Executor task launch worker for task 65 DEBUG ExecutorMetricsPoller: stageTCMP: (17, 0) -> 2 20/01/23 18:54:04.407 Executor task launch worker for task 63 DEBUG BlockManager: Getting local block rdd_55_0 20/01/23 18:54:04.407 Executor task launch worker for task 63 DEBUG BlockManager: Block rdd_55_0 was not found 20/01/23 18:54:04.407 Executor task launch worker for task 63 DEBUG BlockManager: Getting remote block rdd_55_0 20/01/23 18:54:04.408 Executor task launch worker for task 64 DEBUG ExecutorMetricsPoller: stageTCMP: (17, 0) -> 3 20/01/23 18:54:04.408 Executor task launch worker for task 66 DEBUG ExecutorMetricsPoller: stageTCMP: (17, 0) -> 4 20/01/23 18:54:04.408 Executor task launch worker for task 65 DEBUG BlockManager: Getting local block rdd_55_2 20/01/23 18:54:04.408 Executor task launch worker for task 65 DEBUG BlockManager: Block rdd_55_2 was not found 20/01/23 18:54:04.409 Executor task launch worker for task 65 DEBUG BlockManager: Getting remote block rdd_55_2 20/01/23 18:54:04.412 Executor task launch worker for task 66 DEBUG BlockManager: Getting local block rdd_55_3 20/01/23 18:54:04.412 Executor task launch worker for task 66 DEBUG BlockManager: Block rdd_55_3 was not found 20/01/23 18:54:04.412 Executor task launch worker for task 66 DEBUG BlockManager: Getting remote block rdd_55_3 20/01/23 18:54:04.412 Executor task launch worker for task 64 DEBUG BlockManager: Getting local block rdd_55_1 20/01/23 18:54:04.412 Executor task launch worker for task 64 DEBUG BlockManager: Block rdd_55_1 was not found 20/01/23 18:54:04.412 Executor task launch worker for task 64 DEBUG BlockManager: Getting remote block rdd_55_1 20/01/23 18:54:04.413 Executor task launch worker for task 63 DEBUG BlockManager: Block rdd_55_0 is unknown by block manager master 20/01/23 18:54:04.413 Executor task launch worker for task 63 DEBUG BlockManager: Getting local block rdd_24_0 20/01/23 18:54:04.413 Executor task launch worker for task 63 DEBUG BlockManager: Level for block rdd_24_0 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:04.413 Executor task launch worker for task 63 INFO BlockManager: Found block rdd_24_0 locally 20/01/23 18:54:04.413 Executor task launch worker for task 63 DEBUG BlockManager: Getting local block rdd_24_0 20/01/23 18:54:04.413 Executor task launch worker for task 63 DEBUG BlockManager: Level for block rdd_24_0 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:04.413 Executor task launch worker for task 63 INFO BlockManager: Found block rdd_24_0 locally 20/01/23 18:54:04.413 Executor task launch worker for task 63 DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 4, partitions 0-1 20/01/23 18:54:04.413 Executor task launch worker for task 63 DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 20/01/23 18:54:04.414 Executor task launch worker for task 63 INFO ShuffleBlockFetcherIterator: Getting 3 (684.0 B) non-empty blocks including 3 (684.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 20/01/23 18:54:04.414 Executor task launch worker for task 63 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 20/01/23 18:54:04.414 Executor task launch worker for task 63 DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_4_33_0,0), (shuffle_4_34_0,1), (shuffle_4_35_0,2) 20/01/23 18:54:04.414 Executor task launch worker for task 63 DEBUG ShuffleBlockFetcherIterator: Got local blocks in 1 ms 20/01/23 18:54:04.416 Executor task launch worker for task 63 DEBUG TaskMemoryManager: Task 63 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@402811f3 20/01/23 18:54:04.417 Executor task launch worker for task 64 DEBUG BlockManager: Block rdd_55_1 is unknown by block manager master 20/01/23 18:54:04.417 Executor task launch worker for task 66 DEBUG BlockManager: Block rdd_55_3 is unknown by block manager master 20/01/23 18:54:04.417 Executor task launch worker for task 64 DEBUG BlockManager: Getting local block rdd_24_1 20/01/23 18:54:04.417 Executor task launch worker for task 66 DEBUG BlockManager: Getting local block rdd_24_3 20/01/23 18:54:04.417 Executor task launch worker for task 64 DEBUG BlockManager: Level for block rdd_24_1 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:04.417 Executor task launch worker for task 64 INFO BlockManager: Found block rdd_24_1 locally 20/01/23 18:54:04.417 Executor task launch worker for task 66 DEBUG BlockManager: Level for block rdd_24_3 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:04.417 Executor task launch worker for task 66 INFO BlockManager: Found block rdd_24_3 locally 20/01/23 18:54:04.417 Executor task launch worker for task 64 DEBUG BlockManager: Getting local block rdd_24_1 20/01/23 18:54:04.417 Executor task launch worker for task 66 DEBUG BlockManager: Getting local block rdd_24_3 20/01/23 18:54:04.417 Executor task launch worker for task 64 DEBUG BlockManager: Level for block rdd_24_1 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:04.417 Executor task launch worker for task 64 INFO BlockManager: Found block rdd_24_1 locally 20/01/23 18:54:04.417 Executor task launch worker for task 66 DEBUG BlockManager: Level for block rdd_24_3 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:04.418 Executor task launch worker for task 64 DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 4, partitions 1-2 20/01/23 18:54:04.418 Executor task launch worker for task 65 DEBUG BlockManager: Block rdd_55_2 is unknown by block manager master 20/01/23 18:54:04.418 Executor task launch worker for task 65 DEBUG BlockManager: Getting local block rdd_24_2 20/01/23 18:54:04.418 Executor task launch worker for task 65 DEBUG BlockManager: Level for block rdd_24_2 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:04.418 Executor task launch worker for task 64 DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 20/01/23 18:54:04.418 Executor task launch worker for task 66 INFO BlockManager: Found block rdd_24_3 locally 20/01/23 18:54:04.418 Executor task launch worker for task 66 DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 4, partitions 3-4 20/01/23 18:54:04.418 Executor task launch worker for task 64 INFO ShuffleBlockFetcherIterator: Getting 3 (730.0 B) non-empty blocks including 3 (730.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 20/01/23 18:54:04.418 Executor task launch worker for task 64 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 20/01/23 18:54:04.418 Executor task launch worker for task 64 DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_4_33_1,0), (shuffle_4_34_1,1), (shuffle_4_35_1,2) 20/01/23 18:54:04.418 Executor task launch worker for task 66 DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 20/01/23 18:54:04.419 Executor task launch worker for task 64 DEBUG ShuffleBlockFetcherIterator: Got local blocks in 0 ms 20/01/23 18:54:04.419 Executor task launch worker for task 66 INFO ShuffleBlockFetcherIterator: Getting 3 (684.0 B) non-empty blocks including 3 (684.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 20/01/23 18:54:04.419 Executor task launch worker for task 66 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 20/01/23 18:54:04.419 Executor task launch worker for task 66 DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_4_33_3,0), (shuffle_4_34_3,1), (shuffle_4_35_3,2) 20/01/23 18:54:04.420 Executor task launch worker for task 66 DEBUG ShuffleBlockFetcherIterator: Got local blocks in 1 ms 20/01/23 18:54:04.420 Executor task launch worker for task 64 DEBUG TaskMemoryManager: Task 64 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@133d70c5 20/01/23 18:54:04.421 Executor task launch worker for task 64 DEBUG TaskMemoryManager: Task 64 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@7ec5fb01 20/01/23 18:54:04.421 Executor task launch worker for task 66 DEBUG TaskMemoryManager: Task 66 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@31ad4c4 20/01/23 18:54:04.422 Executor task launch worker for task 66 DEBUG TaskMemoryManager: Task 66 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@43d2975f 20/01/23 18:54:04.422 Executor task launch worker for task 63 DEBUG TaskMemoryManager: Task 63 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@2a65126e 20/01/23 18:54:04.422 Executor task launch worker for task 66 DEBUG TaskMemoryManager: Task 66 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@5d70e376 20/01/23 18:54:04.423 Executor task launch worker for task 63 DEBUG TaskMemoryManager: Task 63 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@30be4f92 20/01/23 18:54:04.423 Executor task launch worker for task 66 INFO MemoryStore: Block rdd_55_3 stored as values in memory (estimated size 16.0 B, free 365.9 MiB) 20/01/23 18:54:04.418 Executor task launch worker for task 65 INFO BlockManager: Found block rdd_24_2 locally 20/01/23 18:54:04.423 Executor task launch worker for task 63 INFO MemoryStore: Block rdd_55_0 stored as values in memory (estimated size 88.0 B, free 365.9 MiB) 20/01/23 18:54:04.423 Executor task launch worker for task 65 DEBUG BlockManager: Getting local block rdd_24_2 20/01/23 18:54:04.423 Executor task launch worker for task 65 DEBUG BlockManager: Level for block rdd_24_2 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:04.423 Executor task launch worker for task 65 INFO BlockManager: Found block rdd_24_2 locally 20/01/23 18:54:04.423 Executor task launch worker for task 65 DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 4, partitions 2-3 20/01/23 18:54:04.423 Executor task launch worker for task 64 DEBUG TaskMemoryManager: Task 64 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@445b0358 20/01/23 18:54:04.424 Executor task launch worker for task 64 INFO MemoryStore: Block rdd_55_1 stored as values in memory (estimated size 88.0 B, free 365.9 MiB) 20/01/23 18:54:04.424 Executor task launch worker for task 65 DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 20/01/23 18:54:04.424 Executor task launch worker for task 65 INFO ShuffleBlockFetcherIterator: Getting 3 (707.0 B) non-empty blocks including 3 (707.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 20/01/23 18:54:04.424 Executor task launch worker for task 65 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 20/01/23 18:54:04.424 Executor task launch worker for task 65 DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_4_33_2,0), (shuffle_4_34_2,1), (shuffle_4_35_2,2) 20/01/23 18:54:04.425 Executor task launch worker for task 65 DEBUG ShuffleBlockFetcherIterator: Got local blocks in 1 ms 20/01/23 18:54:04.427 Executor task launch worker for task 65 DEBUG TaskMemoryManager: Task 65 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@51dd2c0 20/01/23 18:54:04.427 Executor task launch worker for task 65 DEBUG TaskMemoryManager: Task 65 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@688b9dec 20/01/23 18:54:04.428 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_55_3 in memory on 192.168.1.225:35863 (size: 16.0 B, free: 366.2 MiB) 20/01/23 18:54:04.428 Executor task launch worker for task 66 DEBUG BlockManagerMaster: Updated info of block rdd_55_3 20/01/23 18:54:04.428 Executor task launch worker for task 66 DEBUG BlockManager: Told master about block rdd_55_3 20/01/23 18:54:04.428 Executor task launch worker for task 66 DEBUG BlockManager: Put block rdd_55_3 locally took 11 ms 20/01/23 18:54:04.429 Executor task launch worker for task 66 DEBUG BlockManager: Putting block rdd_55_3 without replication took 11 ms 20/01/23 18:54:04.429 Executor task launch worker for task 66 DEBUG BlockManager: Getting local block rdd_55_3 20/01/23 18:54:04.429 Executor task launch worker for task 66 DEBUG BlockManager: Level for block rdd_55_3 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:04.429 block-manager-slave-async-thread-pool-11 DEBUG BlockManagerSlaveEndpoint: Done removing RDD 20, response is 0 20/01/23 18:54:04.430 Executor task launch worker for task 66 INFO Executor: Finished task 3.0 in stage 17.0 (TID 66). 2265 bytes result sent to driver 20/01/23 18:54:04.435 Thread-4 INFO CodeGenerator: Code generated in 52.95414 ms 20/01/23 18:54:04.436 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$doExecute$4$adapted 20/01/23 18:54:04.437 Executor task launch worker for task 65 DEBUG TaskMemoryManager: Task 65 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@249f6723 20/01/23 18:54:04.437 Executor task launch worker for task 65 INFO MemoryStore: Block rdd_55_2 stored as values in memory (estimated size 88.0 B, free 365.9 MiB) 20/01/23 18:54:04.438 dispatcher-event-loop-2 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_17.0, runningTasks: 3 20/01/23 18:54:04.438 dispatcher-event-loop-2 INFO TaskSetManager: Starting task 4.0 in stage 17.0 (TID 67, 192.168.1.225, executor driver, partition 4, PROCESS_LOCAL, 7433 bytes) 20/01/23 18:54:04.439 Executor task launch worker for task 67 INFO Executor: Running task 4.0 in stage 17.0 (TID 67) 20/01/23 18:54:04.439 task-result-getter-0 INFO TaskSetManager: Finished task 3.0 in stage 17.0 (TID 66) in 39 ms on 192.168.1.225 (executor driver) (1/10) 20/01/23 18:54:04.439 block-manager-slave-async-thread-pool-12 DEBUG BlockManagerSlaveEndpoint: removing RDD 25 20/01/23 18:54:04.439 block-manager-slave-async-thread-pool-12 INFO BlockManager: Removing RDD 25 20/01/23 18:54:04.439 block-manager-slave-async-thread-pool-12 DEBUG BlockManager: Removing block rdd_25_8 20/01/23 18:54:04.439 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_55_1 in memory on 192.168.1.225:35863 (size: 88.0 B, free: 366.2 MiB) 20/01/23 18:54:04.439 block-manager-slave-async-thread-pool-12 DEBUG MemoryStore: Block rdd_25_8 of size 16 dropped from memory (free 383707748) 20/01/23 18:54:04.440 Executor task launch worker for task 64 DEBUG BlockManagerMaster: Updated info of block rdd_55_1 20/01/23 18:54:04.440 Executor task launch worker for task 64 DEBUG BlockManager: Told master about block rdd_55_1 20/01/23 18:54:04.440 Executor task launch worker for task 64 DEBUG BlockManager: Put block rdd_55_1 locally took 22 ms 20/01/23 18:54:04.440 block-manager-slave-async-thread-pool-12 DEBUG BlockManager: Removing block rdd_25_1 20/01/23 18:54:04.440 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_55_0 in memory on 192.168.1.225:35863 (size: 88.0 B, free: 366.2 MiB) 20/01/23 18:54:04.440 block-manager-slave-async-thread-pool-12 DEBUG MemoryStore: Block rdd_25_1 of size 304 dropped from memory (free 383708052) 20/01/23 18:54:04.440 block-manager-slave-async-thread-pool-12 DEBUG BlockManager: Removing block rdd_25_9 20/01/23 18:54:04.441 Executor task launch worker for task 64 DEBUG BlockManager: Putting block rdd_55_1 without replication took 23 ms 20/01/23 18:54:04.441 Executor task launch worker for task 64 DEBUG BlockManager: Getting local block rdd_55_1 20/01/23 18:54:04.441 Executor task launch worker for task 64 DEBUG BlockManager: Level for block rdd_55_1 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:04.441 Executor task launch worker for task 63 DEBUG BlockManagerMaster: Updated info of block rdd_55_0 20/01/23 18:54:04.441 Executor task launch worker for task 63 DEBUG BlockManager: Told master about block rdd_55_0 20/01/23 18:54:04.441 Executor task launch worker for task 63 DEBUG BlockManager: Put block rdd_55_0 locally took 28 ms 20/01/23 18:54:04.441 block-manager-slave-async-thread-pool-12 DEBUG MemoryStore: Block rdd_25_9 of size 16 dropped from memory (free 383708068) 20/01/23 18:54:04.441 Executor task launch worker for task 63 DEBUG BlockManager: Putting block rdd_55_0 without replication took 28 ms 20/01/23 18:54:04.441 Executor task launch worker for task 63 DEBUG BlockManager: Getting local block rdd_55_0 20/01/23 18:54:04.441 block-manager-slave-async-thread-pool-12 DEBUG BlockManager: Removing block rdd_25_4 20/01/23 18:54:04.441 block-manager-slave-async-thread-pool-12 DEBUG MemoryStore: Block rdd_25_4 of size 16 dropped from memory (free 383708084) 20/01/23 18:54:04.441 block-manager-slave-async-thread-pool-12 DEBUG BlockManager: Removing block rdd_25_0 20/01/23 18:54:04.441 block-manager-slave-async-thread-pool-12 DEBUG MemoryStore: Block rdd_25_0 of size 288 dropped from memory (free 383708372) 20/01/23 18:54:04.441 block-manager-slave-async-thread-pool-12 DEBUG BlockManager: Removing block rdd_25_5 20/01/23 18:54:04.441 block-manager-slave-async-thread-pool-12 DEBUG MemoryStore: Block rdd_25_5 of size 16 dropped from memory (free 383708388) 20/01/23 18:54:04.442 block-manager-slave-async-thread-pool-12 DEBUG BlockManager: Removing block rdd_25_3 20/01/23 18:54:04.442 block-manager-slave-async-thread-pool-12 DEBUG MemoryStore: Block rdd_25_3 of size 16 dropped from memory (free 383708404) 20/01/23 18:54:04.442 block-manager-slave-async-thread-pool-12 DEBUG BlockManager: Removing block rdd_25_7 20/01/23 18:54:04.443 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$doExecute$4$adapted) is now cleaned +++ 20/01/23 18:54:04.439 Executor task launch worker for task 66 DEBUG ExecutorMetricsPoller: stageTCMP: (17, 0) -> 3 20/01/23 18:54:04.444 Executor task launch worker for task 67 DEBUG ExecutorMetricsPoller: stageTCMP: (17, 0) -> 4 20/01/23 18:54:04.444 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_55_2 in memory on 192.168.1.225:35863 (size: 88.0 B, free: 366.2 MiB) 20/01/23 18:54:04.441 Executor task launch worker for task 63 DEBUG BlockManager: Level for block rdd_55_0 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:04.449 Executor task launch worker for task 67 DEBUG BlockManager: Getting local block rdd_55_4 20/01/23 18:54:04.449 Executor task launch worker for task 67 DEBUG BlockManager: Block rdd_55_4 was not found 20/01/23 18:54:04.449 Executor task launch worker for task 67 DEBUG BlockManager: Getting remote block rdd_55_4 20/01/23 18:54:04.449 block-manager-slave-async-thread-pool-9 DEBUG BlockManagerSlaveEndpoint: Done removing RDD 19, response is 0 20/01/23 18:54:04.449 Executor task launch worker for task 67 DEBUG BlockManager: Block rdd_55_4 is unknown by block manager master 20/01/23 18:54:04.449 block-manager-slave-async-thread-pool-13 DEBUG BlockManagerSlaveEndpoint: removing RDD 16 20/01/23 18:54:04.449 block-manager-slave-async-thread-pool-13 INFO BlockManager: Removing RDD 16 20/01/23 18:54:04.449 Executor task launch worker for task 67 DEBUG BlockManager: Getting local block rdd_24_4 20/01/23 18:54:04.449 block-manager-slave-async-thread-pool-13 DEBUG BlockManager: Removing block rdd_16_0 20/01/23 18:54:04.449 Executor task launch worker for task 67 DEBUG BlockManager: Level for block rdd_24_4 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:04.449 Executor task launch worker for task 67 INFO BlockManager: Found block rdd_24_4 locally 20/01/23 18:54:04.449 Executor task launch worker for task 67 DEBUG BlockManager: Getting local block rdd_24_4 20/01/23 18:54:04.450 Executor task launch worker for task 67 DEBUG BlockManager: Level for block rdd_24_4 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:04.450 Executor task launch worker for task 67 INFO BlockManager: Found block rdd_24_4 locally 20/01/23 18:54:04.450 Executor task launch worker for task 67 DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 4, partitions 4-5 20/01/23 18:54:04.450 Executor task launch worker for task 67 DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 20/01/23 18:54:04.451 Executor task launch worker for task 67 INFO ShuffleBlockFetcherIterator: Getting 3 (684.0 B) non-empty blocks including 3 (684.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 20/01/23 18:54:04.451 Executor task launch worker for task 67 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 20/01/23 18:54:04.451 Executor task launch worker for task 67 DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_4_33_4,0), (shuffle_4_34_4,1), (shuffle_4_35_4,2) 20/01/23 18:54:04.451 Executor task launch worker for task 67 DEBUG ShuffleBlockFetcherIterator: Got local blocks in 1 ms 20/01/23 18:54:04.451 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$rdd$1 20/01/23 18:54:04.453 Executor task launch worker for task 65 DEBUG BlockManagerMaster: Updated info of block rdd_55_2 20/01/23 18:54:04.453 Executor task launch worker for task 65 DEBUG BlockManager: Told master about block rdd_55_2 20/01/23 18:54:04.453 Executor task launch worker for task 65 DEBUG BlockManager: Put block rdd_55_2 locally took 35 ms 20/01/23 18:54:04.453 Executor task launch worker for task 67 DEBUG TaskMemoryManager: Task 67 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@20d6ba10 20/01/23 18:54:04.453 Executor task launch worker for task 65 DEBUG BlockManager: Putting block rdd_55_2 without replication took 35 ms 20/01/23 18:54:04.453 Executor task launch worker for task 65 DEBUG BlockManager: Getting local block rdd_55_2 20/01/23 18:54:04.453 Executor task launch worker for task 65 DEBUG BlockManager: Level for block rdd_55_2 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:04.453 block-manager-slave-async-thread-pool-12 DEBUG MemoryStore: Block rdd_25_7 of size 16 dropped from memory (free 383708420) 20/01/23 18:54:04.453 block-manager-slave-async-thread-pool-12 DEBUG BlockManager: Removing block rdd_25_6 20/01/23 18:54:04.454 block-manager-slave-async-thread-pool-12 DEBUG MemoryStore: Block rdd_25_6 of size 16 dropped from memory (free 383708436) 20/01/23 18:54:04.454 block-manager-slave-async-thread-pool-12 DEBUG BlockManager: Removing block rdd_25_2 20/01/23 18:54:04.454 block-manager-slave-async-thread-pool-12 DEBUG MemoryStore: Block rdd_25_2 of size 296 dropped from memory (free 383708732) 20/01/23 18:54:04.454 Executor task launch worker for task 65 INFO Executor: Finished task 2.0 in stage 17.0 (TID 65). 2308 bytes result sent to driver 20/01/23 18:54:04.454 Executor task launch worker for task 65 DEBUG ExecutorMetricsPoller: stageTCMP: (17, 0) -> 3 20/01/23 18:54:04.455 dispatcher-event-loop-1 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_17.0, runningTasks: 3 20/01/23 18:54:04.455 Executor task launch worker for task 64 INFO Executor: Finished task 1.0 in stage 17.0 (TID 64). 2308 bytes result sent to driver 20/01/23 18:54:04.455 Executor task launch worker for task 64 DEBUG ExecutorMetricsPoller: stageTCMP: (17, 0) -> 2 20/01/23 18:54:04.455 block-manager-slave-async-thread-pool-13 DEBUG MemoryStore: Block rdd_16_0 of size 176 dropped from memory (free 383708908) 20/01/23 18:54:04.455 dispatcher-event-loop-1 INFO TaskSetManager: Starting task 5.0 in stage 17.0 (TID 68, 192.168.1.225, executor driver, partition 5, PROCESS_LOCAL, 7433 bytes) 20/01/23 18:54:04.455 block-manager-slave-async-thread-pool-13 DEBUG BlockManager: Removing block rdd_16_2 20/01/23 18:54:04.455 dispatcher-event-loop-1 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_17.0, runningTasks: 3 20/01/23 18:54:04.455 Executor task launch worker for task 67 DEBUG TaskMemoryManager: Task 67 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@65cb3111 20/01/23 18:54:04.456 Executor task launch worker for task 63 INFO Executor: Finished task 0.0 in stage 17.0 (TID 63). 2308 bytes result sent to driver 20/01/23 18:54:04.456 dispatcher-event-loop-1 INFO TaskSetManager: Starting task 6.0 in stage 17.0 (TID 69, 192.168.1.225, executor driver, partition 6, PROCESS_LOCAL, 7433 bytes) 20/01/23 18:54:04.456 Executor task launch worker for task 63 DEBUG ExecutorMetricsPoller: stageTCMP: (17, 0) -> 1 20/01/23 18:54:04.456 Executor task launch worker for task 67 DEBUG TaskMemoryManager: Task 67 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@4a9b7c8 20/01/23 18:54:04.456 Executor task launch worker for task 69 INFO Executor: Running task 6.0 in stage 17.0 (TID 69) 20/01/23 18:54:04.456 block-manager-slave-async-thread-pool-13 DEBUG MemoryStore: Block rdd_16_2 of size 328 dropped from memory (free 383709220) 20/01/23 18:54:04.456 dispatcher-event-loop-1 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_17.0, runningTasks: 3 20/01/23 18:54:04.456 Executor task launch worker for task 67 INFO MemoryStore: Block rdd_55_4 stored as values in memory (estimated size 16.0 B, free 365.9 MiB) 20/01/23 18:54:04.456 dispatcher-event-loop-1 INFO TaskSetManager: Starting task 7.0 in stage 17.0 (TID 70, 192.168.1.225, executor driver, partition 7, PROCESS_LOCAL, 7433 bytes) 20/01/23 18:54:04.456 block-manager-slave-async-thread-pool-13 DEBUG BlockManager: Removing block rdd_16_3 20/01/23 18:54:04.456 Executor task launch worker for task 69 DEBUG ExecutorMetricsPoller: stageTCMP: (17, 0) -> 2 20/01/23 18:54:04.457 block-manager-slave-async-thread-pool-13 DEBUG MemoryStore: Block rdd_16_3 of size 328 dropped from memory (free 383709548) 20/01/23 18:54:04.457 block-manager-slave-async-thread-pool-13 DEBUG BlockManager: Removing block rdd_16_1 20/01/23 18:54:04.457 block-manager-slave-async-thread-pool-13 DEBUG MemoryStore: Block rdd_16_1 of size 176 dropped from memory (free 383709724) 20/01/23 18:54:04.457 task-result-getter-3 INFO TaskSetManager: Finished task 2.0 in stage 17.0 (TID 65) in 57 ms on 192.168.1.225 (executor driver) (2/10) 20/01/23 18:54:04.457 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_55_4 in memory on 192.168.1.225:35863 (size: 16.0 B, free: 366.2 MiB) 20/01/23 18:54:04.457 task-result-getter-1 INFO TaskSetManager: Finished task 0.0 in stage 17.0 (TID 63) in 58 ms on 192.168.1.225 (executor driver) (3/10) 20/01/23 18:54:04.458 task-result-getter-2 INFO TaskSetManager: Finished task 1.0 in stage 17.0 (TID 64) in 58 ms on 192.168.1.225 (executor driver) (4/10) 20/01/23 18:54:04.458 Executor task launch worker for task 68 INFO Executor: Running task 5.0 in stage 17.0 (TID 68) 20/01/23 18:54:04.458 Executor task launch worker for task 67 DEBUG BlockManagerMaster: Updated info of block rdd_55_4 20/01/23 18:54:04.458 Executor task launch worker for task 67 DEBUG BlockManager: Told master about block rdd_55_4 20/01/23 18:54:04.458 Executor task launch worker for task 67 DEBUG BlockManager: Put block rdd_55_4 locally took 8 ms 20/01/23 18:54:04.458 Executor task launch worker for task 67 DEBUG BlockManager: Putting block rdd_55_4 without replication took 8 ms 20/01/23 18:54:04.458 Executor task launch worker for task 67 DEBUG BlockManager: Getting local block rdd_55_4 20/01/23 18:54:04.458 Executor task launch worker for task 67 DEBUG BlockManager: Level for block rdd_55_4 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:04.458 Executor task launch worker for task 68 DEBUG ExecutorMetricsPoller: stageTCMP: (17, 0) -> 3 20/01/23 18:54:04.459 Executor task launch worker for task 67 INFO Executor: Finished task 4.0 in stage 17.0 (TID 67). 2265 bytes result sent to driver 20/01/23 18:54:04.459 Executor task launch worker for task 67 DEBUG ExecutorMetricsPoller: stageTCMP: (17, 0) -> 2 20/01/23 18:54:04.460 dispatcher-event-loop-2 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_17.0, runningTasks: 3 20/01/23 18:54:04.460 dispatcher-event-loop-2 INFO TaskSetManager: Starting task 8.0 in stage 17.0 (TID 71, 192.168.1.225, executor driver, partition 8, PROCESS_LOCAL, 7433 bytes) 20/01/23 18:54:04.461 task-result-getter-0 INFO TaskSetManager: Finished task 4.0 in stage 17.0 (TID 67) in 23 ms on 192.168.1.225 (executor driver) (5/10) 20/01/23 18:54:04.461 Executor task launch worker for task 71 INFO Executor: Running task 8.0 in stage 17.0 (TID 71) 20/01/23 18:54:04.461 Executor task launch worker for task 69 DEBUG BlockManager: Getting local block rdd_55_6 20/01/23 18:54:04.462 Executor task launch worker for task 69 DEBUG BlockManager: Block rdd_55_6 was not found 20/01/23 18:54:04.462 Executor task launch worker for task 69 DEBUG BlockManager: Getting remote block rdd_55_6 20/01/23 18:54:04.461 Executor task launch worker for task 71 DEBUG ExecutorMetricsPoller: stageTCMP: (17, 0) -> 3 20/01/23 18:54:04.462 Executor task launch worker for task 70 INFO Executor: Running task 7.0 in stage 17.0 (TID 70) 20/01/23 18:54:04.462 Executor task launch worker for task 70 DEBUG ExecutorMetricsPoller: stageTCMP: (17, 0) -> 4 20/01/23 18:54:04.465 Executor task launch worker for task 68 DEBUG BlockManager: Getting local block rdd_55_5 20/01/23 18:54:04.465 Executor task launch worker for task 68 DEBUG BlockManager: Block rdd_55_5 was not found 20/01/23 18:54:04.465 Executor task launch worker for task 68 DEBUG BlockManager: Getting remote block rdd_55_5 20/01/23 18:54:04.465 Executor task launch worker for task 68 DEBUG BlockManager: Block rdd_55_5 is unknown by block manager master 20/01/23 18:54:04.466 Executor task launch worker for task 68 DEBUG BlockManager: Getting local block rdd_24_5 20/01/23 18:54:04.466 Executor task launch worker for task 68 DEBUG BlockManager: Level for block rdd_24_5 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:04.466 Executor task launch worker for task 68 INFO BlockManager: Found block rdd_24_5 locally 20/01/23 18:54:04.466 Executor task launch worker for task 68 DEBUG BlockManager: Getting local block rdd_24_5 20/01/23 18:54:04.466 Executor task launch worker for task 68 DEBUG BlockManager: Level for block rdd_24_5 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:04.466 Executor task launch worker for task 68 INFO BlockManager: Found block rdd_24_5 locally 20/01/23 18:54:04.466 Executor task launch worker for task 68 DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 4, partitions 5-6 20/01/23 18:54:04.466 Executor task launch worker for task 68 DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 20/01/23 18:54:04.467 Executor task launch worker for task 71 DEBUG BlockManager: Getting local block rdd_55_8 20/01/23 18:54:04.467 Executor task launch worker for task 71 DEBUG BlockManager: Block rdd_55_8 was not found 20/01/23 18:54:04.467 Executor task launch worker for task 71 DEBUG BlockManager: Getting remote block rdd_55_8 20/01/23 18:54:04.467 Executor task launch worker for task 71 DEBUG BlockManager: Block rdd_55_8 is unknown by block manager master 20/01/23 18:54:04.467 Executor task launch worker for task 70 DEBUG BlockManager: Getting local block rdd_55_7 20/01/23 18:54:04.467 Executor task launch worker for task 68 INFO ShuffleBlockFetcherIterator: Getting 3 (684.0 B) non-empty blocks including 3 (684.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 20/01/23 18:54:04.462 Executor task launch worker for task 69 DEBUG BlockManager: Block rdd_55_6 is unknown by block manager master 20/01/23 18:54:04.467 Executor task launch worker for task 68 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 20/01/23 18:54:04.467 Executor task launch worker for task 68 DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_4_33_5,0), (shuffle_4_34_5,1), (shuffle_4_35_5,2) 20/01/23 18:54:04.467 Executor task launch worker for task 69 DEBUG BlockManager: Getting local block rdd_24_6 20/01/23 18:54:04.467 Executor task launch worker for task 69 DEBUG BlockManager: Level for block rdd_24_6 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:04.467 Executor task launch worker for task 69 INFO BlockManager: Found block rdd_24_6 locally 20/01/23 18:54:04.467 Executor task launch worker for task 69 DEBUG BlockManager: Getting local block rdd_24_6 20/01/23 18:54:04.467 Executor task launch worker for task 69 DEBUG BlockManager: Level for block rdd_24_6 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:04.467 Executor task launch worker for task 69 INFO BlockManager: Found block rdd_24_6 locally 20/01/23 18:54:04.467 Executor task launch worker for task 68 DEBUG ShuffleBlockFetcherIterator: Got local blocks in 1 ms 20/01/23 18:54:04.470 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$pythonToJava$1 20/01/23 18:54:04.471 Executor task launch worker for task 68 DEBUG TaskMemoryManager: Task 68 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@6e835486 20/01/23 18:54:04.471 Executor task launch worker for task 68 DEBUG TaskMemoryManager: Task 68 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@197c6226 20/01/23 18:54:04.471 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$pythonToJava$1) is now cleaned +++ 20/01/23 18:54:04.472 Executor task launch worker for task 68 DEBUG TaskMemoryManager: Task 68 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@2c349004 20/01/23 18:54:04.472 Executor task launch worker for task 68 INFO MemoryStore: Block rdd_55_5 stored as values in memory (estimated size 16.0 B, free 365.9 MiB) 20/01/23 18:54:04.472 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_55_5 in memory on 192.168.1.225:35863 (size: 16.0 B, free: 366.2 MiB) 20/01/23 18:54:04.467 Executor task launch worker for task 70 DEBUG BlockManager: Block rdd_55_7 was not found 20/01/23 18:54:04.467 Executor task launch worker for task 71 DEBUG BlockManager: Getting local block rdd_24_8 20/01/23 18:54:04.473 Executor task launch worker for task 68 DEBUG BlockManagerMaster: Updated info of block rdd_55_5 20/01/23 18:54:04.473 Executor task launch worker for task 71 DEBUG BlockManager: Level for block rdd_24_8 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:04.473 Executor task launch worker for task 71 INFO BlockManager: Found block rdd_24_8 locally 20/01/23 18:54:04.473 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$toJavaArray$1 20/01/23 18:54:04.473 block-manager-slave-async-thread-pool-11 DEBUG BlockManagerSlaveEndpoint: Sent response: 0 to 192.168.1.225:45091 20/01/23 18:54:04.473 Executor task launch worker for task 71 DEBUG BlockManager: Getting local block rdd_24_8 20/01/23 18:54:04.473 Executor task launch worker for task 71 DEBUG BlockManager: Level for block rdd_24_8 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:04.473 Executor task launch worker for task 71 INFO BlockManager: Found block rdd_24_8 locally 20/01/23 18:54:04.473 Executor task launch worker for task 71 DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 4, partitions 8-9 20/01/23 18:54:04.474 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$toJavaArray$1) is now cleaned +++ 20/01/23 18:54:04.474 Executor task launch worker for task 71 DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 20/01/23 18:54:04.474 Executor task launch worker for task 71 INFO ShuffleBlockFetcherIterator: Getting 3 (684.0 B) non-empty blocks including 3 (684.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 20/01/23 18:54:04.474 Executor task launch worker for task 71 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 20/01/23 18:54:04.474 Executor task launch worker for task 71 DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_4_33_8,0), (shuffle_4_34_8,1), (shuffle_4_35_8,2) 20/01/23 18:54:04.475 Executor task launch worker for task 71 DEBUG ShuffleBlockFetcherIterator: Got local blocks in 1 ms 20/01/23 18:54:04.476 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$applySchemaToPythonRDD$1 20/01/23 18:54:04.477 Executor task launch worker for task 71 DEBUG TaskMemoryManager: Task 71 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@6c446f4a 20/01/23 18:54:04.477 Executor task launch worker for task 71 DEBUG TaskMemoryManager: Task 71 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@770bd263 20/01/23 18:54:04.477 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$applySchemaToPythonRDD$1) is now cleaned +++ 20/01/23 18:54:04.477 Executor task launch worker for task 71 DEBUG TaskMemoryManager: Task 71 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@5f0ee9b3 20/01/23 18:54:04.478 Executor task launch worker for task 71 INFO MemoryStore: Block rdd_55_8 stored as values in memory (estimated size 16.0 B, free 365.9 MiB) 20/01/23 18:54:04.478 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_55_8 in memory on 192.168.1.225:35863 (size: 16.0 B, free: 366.2 MiB) 20/01/23 18:54:04.478 Executor task launch worker for task 71 DEBUG BlockManagerMaster: Updated info of block rdd_55_8 20/01/23 18:54:04.478 Executor task launch worker for task 71 DEBUG BlockManager: Told master about block rdd_55_8 20/01/23 18:54:04.478 Executor task launch worker for task 71 DEBUG BlockManager: Put block rdd_55_8 locally took 11 ms 20/01/23 18:54:04.478 Executor task launch worker for task 71 DEBUG BlockManager: Putting block rdd_55_8 without replication took 11 ms 20/01/23 18:54:04.478 Executor task launch worker for task 71 DEBUG BlockManager: Getting local block rdd_55_8 20/01/23 18:54:04.478 Executor task launch worker for task 71 DEBUG BlockManager: Level for block rdd_55_8 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:04.479 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$rdd$1) is now cleaned +++ 20/01/23 18:54:04.479 Executor task launch worker for task 71 INFO Executor: Finished task 8.0 in stage 17.0 (TID 71). 2265 bytes result sent to driver 20/01/23 18:54:04.479 Executor task launch worker for task 71 DEBUG ExecutorMetricsPoller: stageTCMP: (17, 0) -> 3 20/01/23 18:54:04.480 dispatcher-event-loop-0 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_17.0, runningTasks: 3 20/01/23 18:54:04.480 dispatcher-event-loop-0 INFO TaskSetManager: Starting task 9.0 in stage 17.0 (TID 72, 192.168.1.225, executor driver, partition 9, PROCESS_LOCAL, 7433 bytes) 20/01/23 18:54:04.480 Executor task launch worker for task 72 INFO Executor: Running task 9.0 in stage 17.0 (TID 72) 20/01/23 18:54:04.481 task-result-getter-3 INFO TaskSetManager: Finished task 8.0 in stage 17.0 (TID 71) in 21 ms on 192.168.1.225 (executor driver) (6/10) 20/01/23 18:54:04.481 Executor task launch worker for task 72 DEBUG ExecutorMetricsPoller: stageTCMP: (17, 0) -> 4 20/01/23 18:54:04.473 Executor task launch worker for task 70 DEBUG BlockManager: Getting remote block rdd_55_7 20/01/23 18:54:04.468 Executor task launch worker for task 69 DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 4, partitions 6-7 20/01/23 18:54:04.473 Executor task launch worker for task 68 DEBUG BlockManager: Told master about block rdd_55_5 20/01/23 18:54:04.481 Executor task launch worker for task 68 DEBUG BlockManager: Put block rdd_55_5 locally took 15 ms 20/01/23 18:54:04.481 Executor task launch worker for task 68 DEBUG BlockManager: Putting block rdd_55_5 without replication took 15 ms 20/01/23 18:54:04.481 Executor task launch worker for task 68 DEBUG BlockManager: Getting local block rdd_55_5 20/01/23 18:54:04.481 Executor task launch worker for task 68 DEBUG BlockManager: Level for block rdd_55_5 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:04.482 Executor task launch worker for task 70 DEBUG BlockManager: Block rdd_55_7 is unknown by block manager master 20/01/23 18:54:04.482 Executor task launch worker for task 70 DEBUG BlockManager: Getting local block rdd_24_7 20/01/23 18:54:04.482 Executor task launch worker for task 70 DEBUG BlockManager: Level for block rdd_24_7 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:04.482 Executor task launch worker for task 70 INFO BlockManager: Found block rdd_24_7 locally 20/01/23 18:54:04.482 Executor task launch worker for task 70 DEBUG BlockManager: Getting local block rdd_24_7 20/01/23 18:54:04.482 Executor task launch worker for task 70 DEBUG BlockManager: Level for block rdd_24_7 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:04.482 Executor task launch worker for task 70 INFO BlockManager: Found block rdd_24_7 locally 20/01/23 18:54:04.482 Executor task launch worker for task 70 DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 4, partitions 7-8 20/01/23 18:54:04.483 Executor task launch worker for task 70 DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 20/01/23 18:54:04.483 Executor task launch worker for task 69 DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 20/01/23 18:54:04.483 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$evaluate$1 20/01/23 18:54:04.483 Executor task launch worker for task 70 INFO ShuffleBlockFetcherIterator: Getting 3 (684.0 B) non-empty blocks including 3 (684.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 20/01/23 18:54:04.483 Executor task launch worker for task 70 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 20/01/23 18:54:04.483 Executor task launch worker for task 70 DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_4_33_7,0), (shuffle_4_34_7,1), (shuffle_4_35_7,2) 20/01/23 18:54:04.483 Executor task launch worker for task 69 INFO ShuffleBlockFetcherIterator: Getting 3 (684.0 B) non-empty blocks including 3 (684.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 20/01/23 18:54:04.483 Executor task launch worker for task 69 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 1 ms 20/01/23 18:54:04.483 Executor task launch worker for task 69 DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_4_33_6,0), (shuffle_4_34_6,1), (shuffle_4_35_6,2) 20/01/23 18:54:04.483 Executor task launch worker for task 68 INFO Executor: Finished task 5.0 in stage 17.0 (TID 68). 2265 bytes result sent to driver 20/01/23 18:54:04.483 Executor task launch worker for task 68 DEBUG ExecutorMetricsPoller: stageTCMP: (17, 0) -> 3 20/01/23 18:54:04.484 Executor task launch worker for task 70 DEBUG ShuffleBlockFetcherIterator: Got local blocks in 1 ms 20/01/23 18:54:04.484 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$evaluate$1) is now cleaned +++ 20/01/23 18:54:04.484 Executor task launch worker for task 69 DEBUG ShuffleBlockFetcherIterator: Got local blocks in 2 ms 20/01/23 18:54:04.484 dispatcher-event-loop-3 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_17.0, runningTasks: 3 20/01/23 18:54:04.484 dispatcher-event-loop-3 DEBUG TaskSetManager: No tasks for locality level PROCESS_LOCAL, so moving to locality level NODE_LOCAL 20/01/23 18:54:04.484 dispatcher-event-loop-3 DEBUG TaskSetManager: No tasks for locality level NODE_LOCAL, so moving to locality level ANY 20/01/23 18:54:04.485 task-result-getter-1 INFO TaskSetManager: Finished task 5.0 in stage 17.0 (TID 68) in 30 ms on 192.168.1.225 (executor driver) (7/10) 20/01/23 18:54:04.485 Executor task launch worker for task 70 DEBUG TaskMemoryManager: Task 70 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@1a01dff5 20/01/23 18:54:04.485 Executor task launch worker for task 72 DEBUG BlockManager: Getting local block rdd_55_9 20/01/23 18:54:04.486 Executor task launch worker for task 72 DEBUG BlockManager: Block rdd_55_9 was not found 20/01/23 18:54:04.486 Executor task launch worker for task 72 DEBUG BlockManager: Getting remote block rdd_55_9 20/01/23 18:54:04.488 Executor task launch worker for task 69 DEBUG TaskMemoryManager: Task 69 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@4dcc11a8 20/01/23 18:54:04.489 Executor task launch worker for task 69 DEBUG TaskMemoryManager: Task 69 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@31e75241 20/01/23 18:54:04.489 Executor task launch worker for task 69 DEBUG TaskMemoryManager: Task 69 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@68bcb4fc 20/01/23 18:54:04.490 Executor task launch worker for task 69 INFO MemoryStore: Block rdd_55_6 stored as values in memory (estimated size 16.0 B, free 365.9 MiB) 20/01/23 18:54:04.490 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_55_6 in memory on 192.168.1.225:35863 (size: 16.0 B, free: 366.2 MiB) 20/01/23 18:54:04.490 Executor task launch worker for task 70 DEBUG TaskMemoryManager: Task 70 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@1736d1ca 20/01/23 18:54:04.491 Executor task launch worker for task 72 DEBUG BlockManager: Block rdd_55_9 is unknown by block manager master 20/01/23 18:54:04.491 Executor task launch worker for task 72 DEBUG BlockManager: Getting local block rdd_24_9 20/01/23 18:54:04.491 Executor task launch worker for task 72 DEBUG BlockManager: Level for block rdd_24_9 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:04.491 Executor task launch worker for task 70 DEBUG TaskMemoryManager: Task 70 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@5d9613cc 20/01/23 18:54:04.491 Executor task launch worker for task 72 INFO BlockManager: Found block rdd_24_9 locally 20/01/23 18:54:04.491 Executor task launch worker for task 72 DEBUG BlockManager: Getting local block rdd_24_9 20/01/23 18:54:04.491 Executor task launch worker for task 72 DEBUG BlockManager: Level for block rdd_24_9 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:04.491 Executor task launch worker for task 72 INFO BlockManager: Found block rdd_24_9 locally 20/01/23 18:54:04.491 Executor task launch worker for task 72 DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 4, partitions 9-10 20/01/23 18:54:04.491 Executor task launch worker for task 70 INFO MemoryStore: Block rdd_55_7 stored as values in memory (estimated size 16.0 B, free 365.9 MiB) 20/01/23 18:54:04.492 Executor task launch worker for task 72 DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 20/01/23 18:54:04.492 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_55_7 in memory on 192.168.1.225:35863 (size: 16.0 B, free: 366.2 MiB) 20/01/23 18:54:04.492 Executor task launch worker for task 72 INFO ShuffleBlockFetcherIterator: Getting 3 (684.0 B) non-empty blocks including 3 (684.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 20/01/23 18:54:04.492 Executor task launch worker for task 72 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 20/01/23 18:54:04.492 Executor task launch worker for task 72 DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_4_33_9,0), (shuffle_4_34_9,1), (shuffle_4_35_9,2) 20/01/23 18:54:04.493 Executor task launch worker for task 72 DEBUG ShuffleBlockFetcherIterator: Got local blocks in 1 ms 20/01/23 18:54:04.494 Executor task launch worker for task 72 DEBUG TaskMemoryManager: Task 72 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@659cbca6 20/01/23 18:54:04.495 Executor task launch worker for task 72 DEBUG TaskMemoryManager: Task 72 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@3e362f40 20/01/23 18:54:04.495 Executor task launch worker for task 72 DEBUG TaskMemoryManager: Task 72 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@40d4086a 20/01/23 18:54:04.495 Executor task launch worker for task 72 INFO MemoryStore: Block rdd_55_9 stored as values in memory (estimated size 16.0 B, free 365.9 MiB) 20/01/23 18:54:04.496 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_55_9 in memory on 192.168.1.225:35863 (size: 16.0 B, free: 366.2 MiB) 20/01/23 18:54:04.492 Executor task launch worker for task 70 DEBUG BlockManagerMaster: Updated info of block rdd_55_7 20/01/23 18:54:04.496 Executor task launch worker for task 70 DEBUG BlockManager: Told master about block rdd_55_7 20/01/23 18:54:04.496 Executor task launch worker for task 70 DEBUG BlockManager: Put block rdd_55_7 locally took 14 ms 20/01/23 18:54:04.496 Executor task launch worker for task 70 DEBUG BlockManager: Putting block rdd_55_7 without replication took 14 ms 20/01/23 18:54:04.496 Executor task launch worker for task 70 DEBUG BlockManager: Getting local block rdd_55_7 20/01/23 18:54:04.496 Executor task launch worker for task 70 DEBUG BlockManager: Level for block rdd_55_7 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:04.497 Executor task launch worker for task 70 INFO Executor: Finished task 7.0 in stage 17.0 (TID 70). 2265 bytes result sent to driver 20/01/23 18:54:04.497 Executor task launch worker for task 70 DEBUG ExecutorMetricsPoller: stageTCMP: (17, 0) -> 2 20/01/23 18:54:04.492 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$summary$1 20/01/23 18:54:04.498 dispatcher-event-loop-1 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_17.0, runningTasks: 2 20/01/23 18:54:04.499 task-result-getter-2 INFO TaskSetManager: Finished task 7.0 in stage 17.0 (TID 70) in 43 ms on 192.168.1.225 (executor driver) (8/10) 20/01/23 18:54:04.498 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$summary$1) is now cleaned +++ 20/01/23 18:54:04.501 Executor task launch worker for task 69 DEBUG BlockManagerMaster: Updated info of block rdd_55_6 20/01/23 18:54:04.501 Executor task launch worker for task 69 DEBUG BlockManager: Told master about block rdd_55_6 20/01/23 18:54:04.501 Executor task launch worker for task 69 DEBUG BlockManager: Put block rdd_55_6 locally took 33 ms 20/01/23 18:54:04.501 Executor task launch worker for task 69 DEBUG BlockManager: Putting block rdd_55_6 without replication took 34 ms 20/01/23 18:54:04.501 Executor task launch worker for task 69 DEBUG BlockManager: Getting local block rdd_55_6 20/01/23 18:54:04.501 Executor task launch worker for task 69 DEBUG BlockManager: Level for block rdd_55_6 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:04.502 Executor task launch worker for task 69 INFO Executor: Finished task 6.0 in stage 17.0 (TID 69). 2265 bytes result sent to driver 20/01/23 18:54:04.502 Executor task launch worker for task 69 DEBUG ExecutorMetricsPoller: stageTCMP: (17, 0) -> 1 20/01/23 18:54:04.503 dispatcher-event-loop-0 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_17.0, runningTasks: 1 20/01/23 18:54:04.503 task-result-getter-0 INFO TaskSetManager: Finished task 6.0 in stage 17.0 (TID 69) in 48 ms on 192.168.1.225 (executor driver) (9/10) 20/01/23 18:54:04.505 Executor task launch worker for task 72 DEBUG BlockManagerMaster: Updated info of block rdd_55_9 20/01/23 18:54:04.505 Executor task launch worker for task 72 DEBUG BlockManager: Told master about block rdd_55_9 20/01/23 18:54:04.505 Executor task launch worker for task 72 DEBUG BlockManager: Put block rdd_55_9 locally took 14 ms 20/01/23 18:54:04.505 Executor task launch worker for task 72 DEBUG BlockManager: Putting block rdd_55_9 without replication took 14 ms 20/01/23 18:54:04.505 Executor task launch worker for task 72 DEBUG BlockManager: Getting local block rdd_55_9 20/01/23 18:54:04.505 Executor task launch worker for task 72 DEBUG BlockManager: Level for block rdd_55_9 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:04.506 Executor task launch worker for task 72 INFO Executor: Finished task 9.0 in stage 17.0 (TID 72). 2265 bytes result sent to driver 20/01/23 18:54:04.506 Executor task launch worker for task 72 DEBUG ExecutorMetricsPoller: removing (17, 0) from stageTCMP 20/01/23 18:54:04.506 block-manager-slave-async-thread-pool-16 DEBUG BlockManagerSlaveEndpoint: Done removing RDD 25, response is 0 20/01/23 18:54:04.506 block-manager-slave-async-thread-pool-17 DEBUG BlockManagerSlaveEndpoint: Done removing RDD 16, response is 0 20/01/23 18:54:04.506 dispatcher-event-loop-2 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_17.0, runningTasks: 0 20/01/23 18:54:04.507 task-result-getter-3 INFO TaskSetManager: Finished task 9.0 in stage 17.0 (TID 72) in 27 ms on 192.168.1.225 (executor driver) (10/10) 20/01/23 18:54:04.507 task-result-getter-3 INFO TaskSchedulerImpl: Removed TaskSet 17.0, whose tasks have all completed, from pool 20/01/23 18:54:04.508 dag-scheduler-event-loop INFO DAGScheduler: ResultStage 17 (count at ALS.scala:1053) finished in 0.173 s 20/01/23 18:54:04.508 dag-scheduler-event-loop DEBUG DAGScheduler: After removal of stage 17, remaining stages = 4 20/01/23 18:54:04.508 dag-scheduler-event-loop DEBUG DAGScheduler: After removal of stage 14, remaining stages = 3 20/01/23 18:54:04.509 dag-scheduler-event-loop DEBUG DAGScheduler: After removal of stage 13, remaining stages = 2 20/01/23 18:54:04.509 dag-scheduler-event-loop DEBUG DAGScheduler: After removal of stage 16, remaining stages = 1 20/01/23 18:54:04.509 dag-scheduler-event-loop DEBUG DAGScheduler: After removal of stage 15, remaining stages = 0 20/01/23 18:54:04.509 dag-scheduler-event-loop INFO DAGScheduler: Job 4 is finished. Cancelling potential speculative or zombie tasks for this job 20/01/23 18:54:04.518 block-manager-slave-async-thread-pool-16 DEBUG BlockManagerSlaveEndpoint: Sent response: 0 to 192.168.1.225:45091 20/01/23 18:54:04.533 Thread-4 DEBUG CountVectorizer: Input schema: {"type":"struct","fields":[{"name":"id","type":"long","nullable":true,"metadata":{}},{"name":"words","type":{"type":"array","elementType":"string","containsNull":true},"nullable":true,"metadata":{}},{"name":"expected","type":{"type":"udt","class":"org.apache.spark.ml.linalg.VectorUDT","pyClass":"pyspark.ml.linalg.VectorUDT","sqlType":{"type":"struct","fields":[{"name":"type","type":"byte","nullable":false,"metadata":{}},{"name":"size","type":"integer","nullable":true,"metadata":{}},{"name":"indices","type":{"type":"array","elementType":"integer","containsNull":false},"nullable":true,"metadata":{}},{"name":"values","type":{"type":"array","elementType":"double","containsNull":false},"nullable":true,"metadata":{}}]}},"nullable":true,"metadata":{}}]} 20/01/23 18:54:04.534 Thread-4 DEBUG CountVectorizer: Expected output schema: {"type":"struct","fields":[{"name":"id","type":"long","nullable":true,"metadata":{}},{"name":"words","type":{"type":"array","elementType":"string","containsNull":true},"nullable":true,"metadata":{}},{"name":"expected","type":{"type":"udt","class":"org.apache.spark.ml.linalg.VectorUDT","pyClass":"pyspark.ml.linalg.VectorUDT","sqlType":{"type":"struct","fields":[{"name":"type","type":"byte","nullable":false,"metadata":{}},{"name":"size","type":"integer","nullable":true,"metadata":{}},{"name":"indices","type":{"type":"array","elementType":"integer","containsNull":false},"nullable":true,"metadata":{}},{"name":"values","type":{"type":"array","elementType":"double","containsNull":false},"nullable":true,"metadata":{}}]}},"nullable":true,"metadata":{}},{"name":"features","type":{"type":"udt","class":"org.apache.spark.ml.linalg.VectorUDT","pyClass":"pyspark.ml.linalg.VectorUDT","sqlType":{"type":"struct","fields":[{"name":"type","type":"byte","nullable":false,"metadata":{}},{"name":"size","type":"integer","nullable":true,"metadata":{}},{"name":"indices","type":{"type":"array","elementType":"integer","containsNull":false},"nullable":true,"metadata":{}},{"name":"values","type":{"type":"array","elementType":"double","containsNull":false},"nullable":true,"metadata":{}}]}},"nullable":false,"metadata":{}}]} 20/01/23 18:54:04.535 Thread-4 DEBUG Analyzer$ResolveReferences: Resolving 'words to words#38 20/01/23 18:54:04.545 dag-scheduler-event-loop INFO TaskSchedulerImpl: Killing all running tasks in stage 17: Stage finished 20/01/23 18:54:04.546 Thread-4 INFO DAGScheduler: Job 4 finished: count at ALS.scala:1053, took 0.218775 s 20/01/23 18:54:04.546 Thread-4 INFO MapPartitionsRDD: Removing RDD 36 from persistence list 20/01/23 18:54:04.546 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$colStats$1 20/01/23 18:54:04.547 Thread-4 INFO MapPartitionsRDD: Removing RDD 24 from persistence list 20/01/23 18:54:04.547 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$colStats$1) is now cleaned +++ 20/01/23 18:54:04.548 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$colStats$2 20/01/23 18:54:04.548 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$colStats$2) is now cleaned +++ 20/01/23 18:54:04.550 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$treeAggregate$5 20/01/23 18:54:04.557 block-manager-slave-async-thread-pool-18 DEBUG BlockManagerSlaveEndpoint: removing RDD 36 20/01/23 18:54:04.557 block-manager-slave-async-thread-pool-18 INFO BlockManager: Removing RDD 36 20/01/23 18:54:04.565 block-manager-slave-async-thread-pool-9 DEBUG BlockManagerSlaveEndpoint: Sent response: 0 to 192.168.1.225:45091 20/01/23 18:54:04.569 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$treeAggregate$5) is now cleaned +++ 20/01/23 18:54:04.573 block-manager-slave-async-thread-pool-17 DEBUG BlockManagerSlaveEndpoint: Sent response: 0 to 192.168.1.225:45091 20/01/23 18:54:04.589 block-manager-slave-async-thread-pool-20 DEBUG BlockManagerSlaveEndpoint: removing RDD 24 20/01/23 18:54:04.589 block-manager-slave-async-thread-pool-20 INFO BlockManager: Removing RDD 24 20/01/23 18:54:04.589 block-manager-slave-async-thread-pool-20 DEBUG BlockManager: Removing block rdd_24_0 20/01/23 18:54:04.589 block-manager-slave-async-thread-pool-20 DEBUG MemoryStore: Block rdd_24_0 of size 192 dropped from memory (free 383709836) 20/01/23 18:54:04.590 block-manager-slave-async-thread-pool-20 DEBUG BlockManager: Removing block rdd_24_4 20/01/23 18:54:04.590 block-manager-slave-async-thread-pool-20 DEBUG MemoryStore: Block rdd_24_4 of size 16 dropped from memory (free 383709852) 20/01/23 18:54:04.590 block-manager-slave-async-thread-pool-20 DEBUG BlockManager: Removing block rdd_24_5 20/01/23 18:54:04.590 block-manager-slave-async-thread-pool-20 DEBUG MemoryStore: Block rdd_24_5 of size 16 dropped from memory (free 383709868) 20/01/23 18:54:04.590 block-manager-slave-async-thread-pool-20 DEBUG BlockManager: Removing block rdd_24_6 20/01/23 18:54:04.590 block-manager-slave-async-thread-pool-20 DEBUG MemoryStore: Block rdd_24_6 of size 16 dropped from memory (free 383709884) 20/01/23 18:54:04.590 block-manager-slave-async-thread-pool-20 DEBUG BlockManager: Removing block rdd_24_7 20/01/23 18:54:04.590 block-manager-slave-async-thread-pool-20 DEBUG MemoryStore: Block rdd_24_7 of size 16 dropped from memory (free 383709900) 20/01/23 18:54:04.590 block-manager-slave-async-thread-pool-20 DEBUG BlockManager: Removing block rdd_24_8 20/01/23 18:54:04.590 block-manager-slave-async-thread-pool-20 DEBUG MemoryStore: Block rdd_24_8 of size 16 dropped from memory (free 383709916) 20/01/23 18:54:04.590 block-manager-slave-async-thread-pool-20 DEBUG BlockManager: Removing block rdd_24_3 20/01/23 18:54:04.591 block-manager-slave-async-thread-pool-20 DEBUG MemoryStore: Block rdd_24_3 of size 16 dropped from memory (free 383709932) 20/01/23 18:54:04.591 block-manager-slave-async-thread-pool-20 DEBUG BlockManager: Removing block rdd_24_2 20/01/23 18:54:04.591 block-manager-slave-async-thread-pool-20 DEBUG MemoryStore: Block rdd_24_2 of size 192 dropped from memory (free 383710124) 20/01/23 18:54:04.591 block-manager-slave-async-thread-pool-20 DEBUG BlockManager: Removing block rdd_24_9 20/01/23 18:54:04.591 block-manager-slave-async-thread-pool-20 DEBUG MemoryStore: Block rdd_24_9 of size 16 dropped from memory (free 383710140) 20/01/23 18:54:04.591 block-manager-slave-async-thread-pool-20 DEBUG BlockManager: Removing block rdd_24_1 20/01/23 18:54:04.591 block-manager-slave-async-thread-pool-20 DEBUG MemoryStore: Block rdd_24_1 of size 208 dropped from memory (free 383710348) 20/01/23 18:54:04.613 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$colStats$2 20/01/23 18:54:04.614 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$colStats$2) is now cleaned +++ 20/01/23 18:54:04.616 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$runJob$6 20/01/23 18:54:04.625 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$runJob$6) is now cleaned +++ 20/01/23 18:54:04.628 Thread-4 INFO SparkContext: Starting job: treeAggregate at Statistics.scala:58 20/01/23 18:54:04.630 dag-scheduler-event-loop INFO DAGScheduler: Got job 3 (treeAggregate at Statistics.scala:58) with 4 output partitions 20/01/23 18:54:04.630 dag-scheduler-event-loop INFO DAGScheduler: Final stage: ResultStage 5 (treeAggregate at Statistics.scala:58) 20/01/23 18:54:04.630 dag-scheduler-event-loop INFO DAGScheduler: Parents of final stage: List() 20/01/23 18:54:04.631 dag-scheduler-event-loop INFO DAGScheduler: Missing parents: List() 20/01/23 18:54:04.631 dag-scheduler-event-loop DEBUG DAGScheduler: submitStage(ResultStage 5 (name=treeAggregate at Statistics.scala:58;jobs=3)) 20/01/23 18:54:04.631 dag-scheduler-event-loop DEBUG DAGScheduler: missing: List() 20/01/23 18:54:04.632 dag-scheduler-event-loop INFO DAGScheduler: Submitting ResultStage 5 (MapPartitionsRDD[29] at treeAggregate at Statistics.scala:58), which has no missing parents 20/01/23 18:54:04.632 dag-scheduler-event-loop DEBUG DAGScheduler: submitMissingTasks(ResultStage 5) 20/01/23 18:54:04.635 Thread-4 DEBUG WholeStageCodegenExec: /* 001 */ public Object generate(Object[] references) { /* 002 */ return new GeneratedIteratorForCodegenStage1(references); /* 003 */ } /* 004 */ /* 005 */ // codegenStageId=1 /* 006 */ final class GeneratedIteratorForCodegenStage1 extends org.apache.spark.sql.execution.BufferedRowIterator { /* 007 */ private Object[] references; /* 008 */ private scala.collection.Iterator[] inputs; /* 009 */ private scala.collection.Iterator rdd_input_0; /* 010 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[] rdd_mutableStateArray_0 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[3]; /* 011 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[] rdd_mutableStateArray_1 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[4]; /* 012 */ /* 013 */ public GeneratedIteratorForCodegenStage1(Object[] references) { /* 014 */ this.references = references; /* 015 */ } /* 016 */ /* 017 */ public void init(int index, scala.collection.Iterator[] inputs) { /* 018 */ partitionIndex = index; /* 019 */ this.inputs = inputs; /* 020 */ rdd_input_0 = inputs[0]; /* 021 */ rdd_mutableStateArray_0[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(3, 64); /* 022 */ rdd_mutableStateArray_1[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[0], 8); /* 023 */ rdd_mutableStateArray_0[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(rdd_mutableStateArray_0[0], 4); /* 024 */ rdd_mutableStateArray_1[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[1], 4); /* 025 */ rdd_mutableStateArray_1[2] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[1], 8); /* 026 */ rdd_mutableStateArray_0[2] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(1, 32); /* 027 */ rdd_mutableStateArray_1[3] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[2], 8); /* 028 */ /* 029 */ } /* 030 */ /* 031 */ protected void processNext() throws java.io.IOException { /* 032 */ while ( rdd_input_0.hasNext()) { /* 033 */ InternalRow rdd_row_0 = (InternalRow) rdd_input_0.next(); /* 034 */ ((org.apache.spark.sql.execution.metric.SQLMetric) references[0] /* numOutputRows */).add(1); /* 035 */ boolean rdd_isNull_1 = rdd_row_0.isNullAt(1); /* 036 */ ArrayData rdd_value_1 = rdd_isNull_1 ? /* 037 */ null : (rdd_row_0.getArray(1)); /* 038 */ rdd_mutableStateArray_0[2].reset(); /* 039 */ /* 040 */ rdd_mutableStateArray_0[2].zeroOutNullBytes(); /* 041 */ /* 042 */ if (rdd_isNull_1) { /* 043 */ rdd_mutableStateArray_0[2].setNullAt(0); /* 044 */ } else { /* 045 */ // Remember the current cursor so that we can calculate how many bytes are /* 046 */ // written later. /* 047 */ final int project_previousCursor_0 = rdd_mutableStateArray_0[2].cursor(); /* 048 */ /* 049 */ final ArrayData project_tmpInput_0 = rdd_value_1; /* 050 */ if (project_tmpInput_0 instanceof UnsafeArrayData) { /* 051 */ rdd_mutableStateArray_0[2].write((UnsafeArrayData) project_tmpInput_0); /* 052 */ } else { /* 053 */ final int project_numElements_0 = project_tmpInput_0.numElements(); /* 054 */ rdd_mutableStateArray_1[3].initialize(project_numElements_0); /* 055 */ /* 056 */ for (int project_index_0 = 0; project_index_0 < project_numElements_0; project_index_0++) { /* 057 */ if (project_tmpInput_0.isNullAt(project_index_0)) { /* 058 */ rdd_mutableStateArray_1[3].setNull8Bytes(project_index_0); /* 059 */ } else { /* 060 */ rdd_mutableStateArray_1[3].write(project_index_0, project_tmpInput_0.getUTF8String(project_index_0)); /* 061 */ } /* 062 */ /* 063 */ } /* 064 */ } /* 065 */ /* 066 */ rdd_mutableStateArray_0[2].setOffsetAndSizeFromPreviousCursor(0, project_previousCursor_0); /* 067 */ } /* 068 */ append((rdd_mutableStateArray_0[2].getRow())); /* 069 */ if (shouldStop()) return; /* 070 */ } /* 071 */ } /* 072 */ /* 073 */ } 20/01/23 18:54:04.635 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$doExecute$4$adapted 20/01/23 18:54:04.636 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$doExecute$4$adapted) is now cleaned +++ 20/01/23 18:54:04.638 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$rdd$1 20/01/23 18:54:04.640 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$rdd$1) is now cleaned +++ 20/01/23 18:54:04.641 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$fit$2 20/01/23 18:54:04.642 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$fit$2) is now cleaned +++ 20/01/23 18:54:04.643 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$fit$4 20/01/23 18:54:04.643 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$fit$4) is now cleaned +++ 20/01/23 18:54:04.645 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$reduceByKey$2 20/01/23 18:54:04.646 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$reduceByKey$2) is now cleaned +++ 20/01/23 18:54:04.647 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$fit$9 20/01/23 18:54:04.647 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$fit$9) is now cleaned +++ 20/01/23 18:54:04.647 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$fit$9 20/01/23 18:54:04.652 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_6 stored as values in memory (estimated size 18.3 KiB, free 366.2 MiB) 20/01/23 18:54:04.653 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_6 locally took 1 ms 20/01/23 18:54:04.653 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_6 without replication took 1 ms 20/01/23 18:54:04.654 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_6_piece0 stored as bytes in memory (estimated size 9.0 KiB, free 366.2 MiB) 20/01/23 18:54:04.648 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$fit$9) is now cleaned +++ 20/01/23 18:54:04.657 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$fit$10$adapted 20/01/23 18:54:04.658 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$fit$10$adapted) is now cleaned +++ 20/01/23 18:54:04.661 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added broadcast_6_piece0 in memory on 192.168.1.225:45461 (size: 9.0 KiB, free: 366.3 MiB) 20/01/23 18:54:04.661 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$fit$11 20/01/23 18:54:04.662 dag-scheduler-event-loop DEBUG BlockManagerMaster: Updated info of block broadcast_6_piece0 20/01/23 18:54:04.662 dag-scheduler-event-loop DEBUG BlockManager: Told master about block broadcast_6_piece0 20/01/23 18:54:04.662 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_6_piece0 locally took 8 ms 20/01/23 18:54:04.662 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_6_piece0 without replication took 8 ms 20/01/23 18:54:04.662 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$fit$11) is now cleaned +++ 20/01/23 18:54:04.662 dag-scheduler-event-loop INFO SparkContext: Created broadcast 6 from broadcast at DAGScheduler.scala:1200 20/01/23 18:54:04.663 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$count$1$adapted 20/01/23 18:54:04.664 dag-scheduler-event-loop INFO DAGScheduler: Submitting 4 missing tasks from ResultStage 5 (MapPartitionsRDD[29] at treeAggregate at Statistics.scala:58) (first 15 tasks are for partitions Vector(0, 1, 2, 3)) 20/01/23 18:54:04.664 dag-scheduler-event-loop INFO TaskSchedulerImpl: Adding task set 5.0 with 4 tasks 20/01/23 18:54:04.664 dag-scheduler-event-loop DEBUG TaskSetManager: Epoch for TaskSet 5.0: 2 20/01/23 18:54:04.664 dag-scheduler-event-loop DEBUG TaskSetManager: Adding pending tasks took 0 ms 20/01/23 18:54:04.664 dag-scheduler-event-loop DEBUG TaskSetManager: Valid locality levels for TaskSet 5.0: NO_PREF, ANY 20/01/23 18:54:04.665 dispatcher-event-loop-2 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_5.0, runningTasks: 0 20/01/23 18:54:04.665 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$count$1$adapted) is now cleaned +++ 20/01/23 18:54:04.666 dispatcher-event-loop-2 INFO TaskSetManager: Starting task 0.0 in stage 5.0 (TID 14, 192.168.1.225, executor driver, partition 0, PROCESS_LOCAL, 7333 bytes) 20/01/23 18:54:04.666 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$runJob$5 20/01/23 18:54:04.666 dispatcher-event-loop-2 INFO TaskSetManager: Starting task 1.0 in stage 5.0 (TID 15, 192.168.1.225, executor driver, partition 1, PROCESS_LOCAL, 7333 bytes) 20/01/23 18:54:04.666 dispatcher-event-loop-2 INFO TaskSetManager: Starting task 2.0 in stage 5.0 (TID 16, 192.168.1.225, executor driver, partition 2, PROCESS_LOCAL, 7333 bytes) 20/01/23 18:54:04.667 dispatcher-event-loop-2 INFO TaskSetManager: Starting task 3.0 in stage 5.0 (TID 17, 192.168.1.225, executor driver, partition 3, PROCESS_LOCAL, 7378 bytes) 20/01/23 18:54:04.667 Executor task launch worker for task 14 INFO Executor: Running task 0.0 in stage 5.0 (TID 14) 20/01/23 18:54:04.667 Executor task launch worker for task 17 INFO Executor: Running task 3.0 in stage 5.0 (TID 17) 20/01/23 18:54:04.667 Executor task launch worker for task 16 INFO Executor: Running task 2.0 in stage 5.0 (TID 16) 20/01/23 18:54:04.668 Executor task launch worker for task 14 DEBUG ExecutorMetricsPoller: stageTCMP: (5, 0) -> 1 20/01/23 18:54:04.668 Executor task launch worker for task 17 DEBUG ExecutorMetricsPoller: stageTCMP: (5, 0) -> 2 20/01/23 18:54:04.669 Executor task launch worker for task 14 DEBUG BlockManager: Getting local block broadcast_6 20/01/23 18:54:04.669 Executor task launch worker for task 14 DEBUG BlockManager: Level for block broadcast_6 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:04.673 Executor task launch worker for task 16 DEBUG ExecutorMetricsPoller: stageTCMP: (5, 0) -> 3 20/01/23 18:54:04.673 Executor task launch worker for task 15 INFO Executor: Running task 1.0 in stage 5.0 (TID 15) 20/01/23 18:54:04.674 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$runJob$5) is now cleaned +++ 20/01/23 18:54:04.675 Thread-4 INFO SparkContext: Starting job: count at CountVectorizer.scala:233 20/01/23 18:54:04.642 block-manager-slave-async-thread-pool-23 DEBUG BlockManagerSlaveEndpoint: Done removing RDD 24, response is 0 20/01/23 18:54:04.677 dag-scheduler-event-loop DEBUG SortShuffleManager: Can't use serialized shuffle for shuffle 1 because the serializer, org.apache.spark.serializer.JavaSerializer, does not support object relocation 20/01/23 18:54:04.679 dag-scheduler-event-loop INFO DAGScheduler: Registering RDD 35 (flatMap at CountVectorizer.scala:212) as input to shuffle 1 20/01/23 18:54:04.679 dag-scheduler-event-loop INFO DAGScheduler: Got job 5 (count at CountVectorizer.scala:233) with 4 output partitions 20/01/23 18:54:04.679 dag-scheduler-event-loop INFO DAGScheduler: Final stage: ResultStage 8 (count at CountVectorizer.scala:233) 20/01/23 18:54:04.679 dag-scheduler-event-loop INFO DAGScheduler: Parents of final stage: List(ShuffleMapStage 7) 20/01/23 18:54:04.680 dag-scheduler-event-loop INFO DAGScheduler: Missing parents: List(ShuffleMapStage 7) 20/01/23 18:54:04.680 dag-scheduler-event-loop DEBUG DAGScheduler: submitStage(ResultStage 8 (name=count at CountVectorizer.scala:233;jobs=5)) 20/01/23 18:54:04.681 dag-scheduler-event-loop DEBUG DAGScheduler: missing: List(ShuffleMapStage 7) 20/01/23 18:54:04.681 dag-scheduler-event-loop DEBUG DAGScheduler: submitStage(ShuffleMapStage 7 (name=flatMap at CountVectorizer.scala:212;jobs=5)) 20/01/23 18:54:04.681 dag-scheduler-event-loop DEBUG DAGScheduler: missing: List() 20/01/23 18:54:04.681 dag-scheduler-event-loop INFO DAGScheduler: Submitting ShuffleMapStage 7 (MapPartitionsRDD[35] at flatMap at CountVectorizer.scala:212), which has no missing parents 20/01/23 18:54:04.681 dag-scheduler-event-loop DEBUG DAGScheduler: submitMissingTasks(ShuffleMapStage 7) 20/01/23 18:54:04.685 block-manager-slave-async-thread-pool-23 DEBUG BlockManagerSlaveEndpoint: Sent response: 0 to 192.168.1.225:45091 20/01/23 18:54:04.654 block-manager-slave-async-thread-pool-22 DEBUG BlockManagerSlaveEndpoint: Done removing RDD 36, response is 0 20/01/23 18:54:04.701 block-manager-slave-async-thread-pool-22 DEBUG BlockManagerSlaveEndpoint: Sent response: 0 to 192.168.1.225:45091 20/01/23 18:54:04.709 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_9 stored as values in memory (estimated size 21.5 KiB, free 366.2 MiB) 20/01/23 18:54:04.709 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_9 locally took 1 ms 20/01/23 18:54:04.709 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_9 without replication took 1 ms 20/01/23 18:54:04.710 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_9_piece0 stored as bytes in memory (estimated size 10.5 KiB, free 366.2 MiB) 20/01/23 18:54:04.713 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added broadcast_9_piece0 in memory on 192.168.1.225:41233 (size: 10.5 KiB, free: 366.3 MiB) 20/01/23 18:54:04.714 dag-scheduler-event-loop DEBUG BlockManagerMaster: Updated info of block broadcast_9_piece0 20/01/23 18:54:04.714 dag-scheduler-event-loop DEBUG BlockManager: Told master about block broadcast_9_piece0 20/01/23 18:54:04.714 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_9_piece0 locally took 3 ms 20/01/23 18:54:04.714 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_9_piece0 without replication took 3 ms 20/01/23 18:54:04.714 dag-scheduler-event-loop INFO SparkContext: Created broadcast 9 from broadcast at DAGScheduler.scala:1200 20/01/23 18:54:04.715 dag-scheduler-event-loop INFO DAGScheduler: Submitting 4 missing tasks from ShuffleMapStage 7 (MapPartitionsRDD[35] at flatMap at CountVectorizer.scala:212) (first 15 tasks are for partitions Vector(0, 1, 2, 3)) 20/01/23 18:54:04.715 dag-scheduler-event-loop INFO TaskSchedulerImpl: Adding task set 7.0 with 4 tasks 20/01/23 18:54:04.715 dag-scheduler-event-loop DEBUG TaskSetManager: Epoch for TaskSet 7.0: 1 20/01/23 18:54:04.715 dag-scheduler-event-loop DEBUG TaskSetManager: Adding pending tasks took 0 ms 20/01/23 18:54:04.715 dag-scheduler-event-loop DEBUG TaskSetManager: Valid locality levels for TaskSet 7.0: NO_PREF, ANY 20/01/23 18:54:04.716 dispatcher-event-loop-2 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_7.0, runningTasks: 0 20/01/23 18:54:04.716 dispatcher-event-loop-2 INFO TaskSetManager: Starting task 0.0 in stage 7.0 (TID 24, 192.168.1.225, executor driver, partition 0, PROCESS_LOCAL, 7419 bytes) 20/01/23 18:54:04.716 dispatcher-event-loop-2 INFO TaskSetManager: Starting task 1.0 in stage 7.0 (TID 25, 192.168.1.225, executor driver, partition 1, PROCESS_LOCAL, 7404 bytes) 20/01/23 18:54:04.717 dispatcher-event-loop-2 INFO TaskSetManager: Starting task 2.0 in stage 7.0 (TID 26, 192.168.1.225, executor driver, partition 2, PROCESS_LOCAL, 7387 bytes) 20/01/23 18:54:04.717 dispatcher-event-loop-2 INFO TaskSetManager: Starting task 3.0 in stage 7.0 (TID 27, 192.168.1.225, executor driver, partition 3, PROCESS_LOCAL, 7369 bytes) 20/01/23 18:54:04.717 Executor task launch worker for task 24 INFO Executor: Running task 0.0 in stage 7.0 (TID 24) 20/01/23 18:54:04.717 Executor task launch worker for task 25 INFO Executor: Running task 1.0 in stage 7.0 (TID 25) 20/01/23 18:54:04.718 Executor task launch worker for task 26 INFO Executor: Running task 2.0 in stage 7.0 (TID 26) 20/01/23 18:54:04.719 Executor task launch worker for task 26 DEBUG ExecutorMetricsPoller: stageTCMP: (7, 0) -> 1 20/01/23 18:54:04.719 Executor task launch worker for task 26 DEBUG BlockManager: Getting local block broadcast_9 20/01/23 18:54:04.719 Executor task launch worker for task 26 DEBUG BlockManager: Level for block broadcast_9 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:04.724 Executor task launch worker for task 27 INFO Executor: Running task 3.0 in stage 7.0 (TID 27) 20/01/23 18:54:04.724 Executor task launch worker for task 27 DEBUG ExecutorMetricsPoller: stageTCMP: (7, 0) -> 2 20/01/23 18:54:04.729 Executor task launch worker for task 25 DEBUG ExecutorMetricsPoller: stageTCMP: (7, 0) -> 3 20/01/23 18:54:04.734 Executor task launch worker for task 24 DEBUG ExecutorMetricsPoller: stageTCMP: (7, 0) -> 4 20/01/23 18:54:04.738 Executor task launch worker for task 15 DEBUG ExecutorMetricsPoller: stageTCMP: (5, 0) -> 4 20/01/23 18:54:04.741 Executor task launch worker for task 14 DEBUG GenerateSafeProjection: code for createexternalrow(input[0, double, true], input[1, double, true], input[2, double, false], StructField(prediction,DoubleType,true), StructField(label,DoubleType,true), StructField(1.0,DoubleType,false)): /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificSafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificSafeProjection extends org.apache.spark.sql.catalyst.expressions.codegen.BaseProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private InternalRow mutableRow; /* 009 */ /* 010 */ /* 011 */ public SpecificSafeProjection(Object[] references) { /* 012 */ this.references = references; /* 013 */ mutableRow = (InternalRow) references[references.length - 1]; /* 014 */ /* 015 */ } /* 016 */ /* 017 */ public void initialize(int partitionIndex) { /* 018 */ /* 019 */ } /* 020 */ /* 021 */ public java.lang.Object apply(java.lang.Object _i) { /* 022 */ InternalRow i = (InternalRow) _i; /* 023 */ Object[] values_0 = new Object[3]; /* 024 */ /* 025 */ boolean isNull_1 = i.isNullAt(0); /* 026 */ double value_1 = isNull_1 ? /* 027 */ -1.0 : (i.getDouble(0)); /* 028 */ if (isNull_1) { /* 029 */ values_0[0] = null; /* 030 */ } else { /* 031 */ values_0[0] = value_1; /* 032 */ } /* 033 */ /* 034 */ boolean isNull_2 = i.isNullAt(1); /* 035 */ double value_2 = isNull_2 ? /* 036 */ -1.0 : (i.getDouble(1)); /* 037 */ if (isNull_2) { /* 038 */ values_0[1] = null; /* 039 */ } else { /* 040 */ values_0[1] = value_2; /* 041 */ } /* 042 */ /* 043 */ double value_3 = i.getDouble(2); /* 044 */ if (false) { /* 045 */ values_0[2] = null; /* 046 */ } else { /* 047 */ values_0[2] = value_3; /* 048 */ } /* 049 */ /* 050 */ final org.apache.spark.sql.Row value_0 = new org.apache.spark.sql.catalyst.expressions.GenericRowWithSchema(values_0, ((org.apache.spark.sql.types.StructType) references[0] /* schema */)); /* 051 */ if (false) { /* 052 */ mutableRow.setNullAt(0); /* 053 */ } else { /* 054 */ /* 055 */ mutableRow.update(0, value_0); /* 056 */ } /* 057 */ /* 058 */ return mutableRow; /* 059 */ } /* 060 */ /* 061 */ /* 062 */ } 20/01/23 18:54:04.742 Executor task launch worker for task 14 DEBUG CodeGenerator: /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificSafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificSafeProjection extends org.apache.spark.sql.catalyst.expressions.codegen.BaseProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private InternalRow mutableRow; /* 009 */ /* 010 */ /* 011 */ public SpecificSafeProjection(Object[] references) { /* 012 */ this.references = references; /* 013 */ mutableRow = (InternalRow) references[references.length - 1]; /* 014 */ /* 015 */ } /* 016 */ /* 017 */ public void initialize(int partitionIndex) { /* 018 */ /* 019 */ } /* 020 */ /* 021 */ public java.lang.Object apply(java.lang.Object _i) { /* 022 */ InternalRow i = (InternalRow) _i; /* 023 */ Object[] values_0 = new Object[3]; /* 024 */ /* 025 */ boolean isNull_1 = i.isNullAt(0); /* 026 */ double value_1 = isNull_1 ? /* 027 */ -1.0 : (i.getDouble(0)); /* 028 */ if (isNull_1) { /* 029 */ values_0[0] = null; /* 030 */ } else { /* 031 */ values_0[0] = value_1; /* 032 */ } /* 033 */ /* 034 */ boolean isNull_2 = i.isNullAt(1); /* 035 */ double value_2 = isNull_2 ? /* 036 */ -1.0 : (i.getDouble(1)); /* 037 */ if (isNull_2) { /* 038 */ values_0[1] = null; /* 039 */ } else { /* 040 */ values_0[1] = value_2; /* 041 */ } /* 042 */ /* 043 */ double value_3 = i.getDouble(2); /* 044 */ if (false) { /* 045 */ values_0[2] = null; /* 046 */ } else { /* 047 */ values_0[2] = value_3; /* 048 */ } /* 049 */ /* 050 */ final org.apache.spark.sql.Row value_0 = new org.apache.spark.sql.catalyst.expressions.GenericRowWithSchema(values_0, ((org.apache.spark.sql.types.StructType) references[0] /* schema */)); /* 051 */ if (false) { /* 052 */ mutableRow.setNullAt(0); /* 053 */ } else { /* 054 */ /* 055 */ mutableRow.update(0, value_0); /* 056 */ } /* 057 */ /* 058 */ return mutableRow; /* 059 */ } /* 060 */ /* 061 */ /* 062 */ } 20/01/23 18:54:04.757 Executor task launch worker for task 17 DEBUG GenerateSafeProjection: code for createexternalrow(input[0, double, true], input[1, double, true], input[2, double, false], StructField(prediction,DoubleType,true), StructField(label,DoubleType,true), StructField(1.0,DoubleType,false)): /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificSafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificSafeProjection extends org.apache.spark.sql.catalyst.expressions.codegen.BaseProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private InternalRow mutableRow; /* 009 */ /* 010 */ /* 011 */ public SpecificSafeProjection(Object[] references) { /* 012 */ this.references = references; /* 013 */ mutableRow = (InternalRow) references[references.length - 1]; /* 014 */ /* 015 */ } /* 016 */ /* 017 */ public void initialize(int partitionIndex) { /* 018 */ /* 019 */ } /* 020 */ /* 021 */ public java.lang.Object apply(java.lang.Object _i) { /* 022 */ InternalRow i = (InternalRow) _i; /* 023 */ Object[] values_0 = new Object[3]; /* 024 */ /* 025 */ boolean isNull_1 = i.isNullAt(0); /* 026 */ double value_1 = isNull_1 ? /* 027 */ -1.0 : (i.getDouble(0)); /* 028 */ if (isNull_1) { /* 029 */ values_0[0] = null; /* 030 */ } else { /* 031 */ values_0[0] = value_1; /* 032 */ } /* 033 */ /* 034 */ boolean isNull_2 = i.isNullAt(1); /* 035 */ double value_2 = isNull_2 ? /* 036 */ -1.0 : (i.getDouble(1)); /* 037 */ if (isNull_2) { /* 038 */ values_0[1] = null; /* 039 */ } else { /* 040 */ values_0[1] = value_2; /* 041 */ } /* 042 */ /* 043 */ double value_3 = i.getDouble(2); /* 044 */ if (false) { /* 045 */ values_0[2] = null; /* 046 */ } else { /* 047 */ values_0[2] = value_3; /* 048 */ } /* 049 */ /* 050 */ final org.apache.spark.sql.Row value_0 = new org.apache.spark.sql.catalyst.expressions.GenericRowWithSchema(values_0, ((org.apache.spark.sql.types.StructType) references[0] /* schema */)); /* 051 */ if (false) { /* 052 */ mutableRow.setNullAt(0); /* 053 */ } else { /* 054 */ /* 055 */ mutableRow.update(0, value_0); /* 056 */ } /* 057 */ /* 058 */ return mutableRow; /* 059 */ } /* 060 */ /* 061 */ /* 062 */ } 20/01/23 18:54:04.767 Executor task launch worker for task 24 DEBUG GenerateSafeProjection: code for createexternalrow(staticinvoke(class scala.collection.mutable.WrappedArray$, ObjectType(interface scala.collection.Seq), make, mapobjects(lambdavariable(MapObject, StringType, true, -1), lambdavariable(MapObject, StringType, true, -1).toString, input[0, array, true], None).array, true, false), StructField(words,ArrayType(StringType,true),true)): /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificSafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificSafeProjection extends org.apache.spark.sql.catalyst.expressions.codegen.BaseProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private InternalRow mutableRow; /* 009 */ private boolean resultIsNull_0; /* 010 */ private UTF8String value_MapObject_lambda_variable_1; /* 011 */ private boolean isNull_MapObject_lambda_variable_1; /* 012 */ private boolean globalIsNull_0; /* 013 */ private java.lang.Object[] mutableStateArray_0 = new java.lang.Object[1]; /* 014 */ /* 015 */ public SpecificSafeProjection(Object[] references) { /* 016 */ this.references = references; /* 017 */ mutableRow = (InternalRow) references[references.length - 1]; /* 018 */ /* 019 */ /* 020 */ } /* 021 */ /* 022 */ public void initialize(int partitionIndex) { /* 023 */ /* 024 */ } /* 025 */ /* 026 */ public java.lang.Object apply(java.lang.Object _i) { /* 027 */ InternalRow i = (InternalRow) _i; /* 028 */ Object[] values_0 = new Object[1]; /* 029 */ /* 030 */ resultIsNull_0 = false; /* 031 */ if (!resultIsNull_0) { /* 032 */ ArrayData value_6 = MapObjects_0(i); /* 033 */ boolean isNull_2 = true; /* 034 */ java.lang.Object value_2 = null; /* 035 */ if (!globalIsNull_0) { /* 036 */ /* 037 */ isNull_2 = false; /* 038 */ if (!isNull_2) { /* 039 */ /* 040 */ Object funcResult_1 = null; /* 041 */ funcResult_1 = value_6.array(); /* 042 */ value_2 = (java.lang.Object) funcResult_1; /* 043 */ /* 044 */ } /* 045 */ } /* 046 */ resultIsNull_0 = isNull_2; /* 047 */ mutableStateArray_0[0] = value_2; /* 048 */ } /* 049 */ /* 050 */ boolean isNull_1 = resultIsNull_0; /* 051 */ scala.collection.Seq value_1 = null; /* 052 */ if (!resultIsNull_0) { /* 053 */ value_1 = scala.collection.mutable.WrappedArray.make(mutableStateArray_0[0]); /* 054 */ } /* 055 */ if (isNull_1) { /* 056 */ values_0[0] = null; /* 057 */ } else { /* 058 */ values_0[0] = value_1; /* 059 */ } /* 060 */ /* 061 */ final org.apache.spark.sql.Row value_0 = new org.apache.spark.sql.catalyst.expressions.GenericRowWithSchema(values_0, ((org.apache.spark.sql.types.StructType) references[0] /* schema */)); /* 062 */ if (false) { /* 063 */ mutableRow.setNullAt(0); /* 064 */ } else { /* 065 */ /* 066 */ mutableRow.update(0, value_0); /* 067 */ } /* 068 */ /* 069 */ return mutableRow; /* 070 */ } /* 071 */ /* 072 */ /* 073 */ private ArrayData MapObjects_0(InternalRow i) { /* 074 */ boolean isNull_4 = i.isNullAt(0); /* 075 */ ArrayData value_4 = isNull_4 ? /* 076 */ null : (i.getArray(0)); /* 077 */ ArrayData value_3 = null; /* 078 */ /* 079 */ if (!isNull_4) { /* 080 */ /* 081 */ int dataLength_0 = value_4.numElements(); /* 082 */ /* 083 */ java.lang.String[] convertedArray_0 = null; /* 084 */ convertedArray_0 = new java.lang.String[dataLength_0]; /* 085 */ /* 086 */ /* 087 */ int loopIndex_0 = 0; /* 088 */ /* 089 */ while (loopIndex_0 < dataLength_0) { /* 090 */ value_MapObject_lambda_variable_1 = (UTF8String) (value_4.getUTF8String(loopIndex_0)); /* 091 */ isNull_MapObject_lambda_variable_1 = value_4.isNullAt(loopIndex_0); /* 092 */ /* 093 */ boolean isNull_5 = true; /* 094 */ java.lang.String value_5 = null; /* 095 */ if (!isNull_MapObject_lambda_variable_1) { /* 096 */ /* 097 */ isNull_5 = false; /* 098 */ if (!isNull_5) { /* 099 */ /* 100 */ Object funcResult_0 = null; /* 101 */ funcResult_0 = value_MapObject_lambda_variable_1.toString(); /* 102 */ value_5 = (java.lang.String) funcResult_0; /* 103 */ /* 104 */ } /* 105 */ } /* 106 */ if (isNull_5) { /* 107 */ convertedArray_0[loopIndex_0] = null; /* 108 */ } else { /* 109 */ convertedArray_0[loopIndex_0] = value_5; /* 110 */ } /* 111 */ /* 112 */ loopIndex_0 += 1; /* 113 */ } /* 114 */ /* 115 */ value_3 = new org.apache.spark.sql.catalyst.util.GenericArrayData(convertedArray_0); /* 116 */ } /* 117 */ globalIsNull_0 = isNull_4; /* 118 */ return value_3; /* 119 */ } /* 120 */ /* 121 */ } 20/01/23 18:54:04.771 Executor task launch worker for task 16 DEBUG GenerateSafeProjection: code for createexternalrow(input[0, double, true], input[1, double, true], input[2, double, false], StructField(prediction,DoubleType,true), StructField(label,DoubleType,true), StructField(1.0,DoubleType,false)): /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificSafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificSafeProjection extends org.apache.spark.sql.catalyst.expressions.codegen.BaseProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private InternalRow mutableRow; /* 009 */ /* 010 */ /* 011 */ public SpecificSafeProjection(Object[] references) { /* 012 */ this.references = references; /* 013 */ mutableRow = (InternalRow) references[references.length - 1]; /* 014 */ /* 015 */ } /* 016 */ /* 017 */ public void initialize(int partitionIndex) { /* 018 */ /* 019 */ } /* 020 */ /* 021 */ public java.lang.Object apply(java.lang.Object _i) { /* 022 */ InternalRow i = (InternalRow) _i; /* 023 */ Object[] values_0 = new Object[3]; /* 024 */ /* 025 */ boolean isNull_1 = i.isNullAt(0); /* 026 */ double value_1 = isNull_1 ? /* 027 */ -1.0 : (i.getDouble(0)); /* 028 */ if (isNull_1) { /* 029 */ values_0[0] = null; /* 030 */ } else { /* 031 */ values_0[0] = value_1; /* 032 */ } /* 033 */ /* 034 */ boolean isNull_2 = i.isNullAt(1); /* 035 */ double value_2 = isNull_2 ? /* 036 */ -1.0 : (i.getDouble(1)); /* 037 */ if (isNull_2) { /* 038 */ values_0[1] = null; /* 039 */ } else { /* 040 */ values_0[1] = value_2; /* 041 */ } /* 042 */ /* 043 */ double value_3 = i.getDouble(2); /* 044 */ if (false) { /* 045 */ values_0[2] = null; /* 046 */ } else { /* 047 */ values_0[2] = value_3; /* 048 */ } /* 049 */ /* 050 */ final org.apache.spark.sql.Row value_0 = new org.apache.spark.sql.catalyst.expressions.GenericRowWithSchema(values_0, ((org.apache.spark.sql.types.StructType) references[0] /* schema */)); /* 051 */ if (false) { /* 052 */ mutableRow.setNullAt(0); /* 053 */ } else { /* 054 */ /* 055 */ mutableRow.update(0, value_0); /* 056 */ } /* 057 */ /* 058 */ return mutableRow; /* 059 */ } /* 060 */ /* 061 */ /* 062 */ } 20/01/23 18:54:04.773 Executor task launch worker for task 25 DEBUG GenerateSafeProjection: code for createexternalrow(staticinvoke(class scala.collection.mutable.WrappedArray$, ObjectType(interface scala.collection.Seq), make, mapobjects(lambdavariable(MapObject, StringType, true, -1), lambdavariable(MapObject, StringType, true, -1).toString, input[0, array, true], None).array, true, false), StructField(words,ArrayType(StringType,true),true)): /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificSafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificSafeProjection extends org.apache.spark.sql.catalyst.expressions.codegen.BaseProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private InternalRow mutableRow; /* 009 */ private boolean resultIsNull_0; /* 010 */ private UTF8String value_MapObject_lambda_variable_1; /* 011 */ private boolean isNull_MapObject_lambda_variable_1; /* 012 */ private boolean globalIsNull_0; /* 013 */ private java.lang.Object[] mutableStateArray_0 = new java.lang.Object[1]; /* 014 */ /* 015 */ public SpecificSafeProjection(Object[] references) { /* 016 */ this.references = references; /* 017 */ mutableRow = (InternalRow) references[references.length - 1]; /* 018 */ /* 019 */ /* 020 */ } /* 021 */ /* 022 */ public void initialize(int partitionIndex) { /* 023 */ /* 024 */ } /* 025 */ /* 026 */ public java.lang.Object apply(java.lang.Object _i) { /* 027 */ InternalRow i = (InternalRow) _i; /* 028 */ Object[] values_0 = new Object[1]; /* 029 */ /* 030 */ resultIsNull_0 = false; /* 031 */ if (!resultIsNull_0) { /* 032 */ ArrayData value_6 = MapObjects_0(i); /* 033 */ boolean isNull_2 = true; /* 034 */ java.lang.Object value_2 = null; /* 035 */ if (!globalIsNull_0) { /* 036 */ /* 037 */ isNull_2 = false; /* 038 */ if (!isNull_2) { /* 039 */ /* 040 */ Object funcResult_1 = null; /* 041 */ funcResult_1 = value_6.array(); /* 042 */ value_2 = (java.lang.Object) funcResult_1; /* 043 */ /* 044 */ } /* 045 */ } /* 046 */ resultIsNull_0 = isNull_2; /* 047 */ mutableStateArray_0[0] = value_2; /* 048 */ } /* 049 */ /* 050 */ boolean isNull_1 = resultIsNull_0; /* 051 */ scala.collection.Seq value_1 = null; /* 052 */ if (!resultIsNull_0) { /* 053 */ value_1 = scala.collection.mutable.WrappedArray.make(mutableStateArray_0[0]); /* 054 */ } /* 055 */ if (isNull_1) { /* 056 */ values_0[0] = null; /* 057 */ } else { /* 058 */ values_0[0] = value_1; /* 059 */ } /* 060 */ /* 061 */ final org.apache.spark.sql.Row value_0 = new org.apache.spark.sql.catalyst.expressions.GenericRowWithSchema(values_0, ((org.apache.spark.sql.types.StructType) references[0] /* schema */)); /* 062 */ if (false) { /* 063 */ mutableRow.setNullAt(0); /* 064 */ } else { /* 065 */ /* 066 */ mutableRow.update(0, value_0); /* 067 */ } /* 068 */ /* 069 */ return mutableRow; /* 070 */ } /* 071 */ /* 072 */ /* 073 */ private ArrayData MapObjects_0(InternalRow i) { /* 074 */ boolean isNull_4 = i.isNullAt(0); /* 075 */ ArrayData value_4 = isNull_4 ? /* 076 */ null : (i.getArray(0)); /* 077 */ ArrayData value_3 = null; /* 078 */ /* 079 */ if (!isNull_4) { /* 080 */ /* 081 */ int dataLength_0 = value_4.numElements(); /* 082 */ /* 083 */ java.lang.String[] convertedArray_0 = null; /* 084 */ convertedArray_0 = new java.lang.String[dataLength_0]; /* 085 */ /* 086 */ /* 087 */ int loopIndex_0 = 0; /* 088 */ /* 089 */ while (loopIndex_0 < dataLength_0) { /* 090 */ value_MapObject_lambda_variable_1 = (UTF8String) (value_4.getUTF8String(loopIndex_0)); /* 091 */ isNull_MapObject_lambda_variable_1 = value_4.isNullAt(loopIndex_0); /* 092 */ /* 093 */ boolean isNull_5 = true; /* 094 */ java.lang.String value_5 = null; /* 095 */ if (!isNull_MapObject_lambda_variable_1) { /* 096 */ /* 097 */ isNull_5 = false; /* 098 */ if (!isNull_5) { /* 099 */ /* 100 */ Object funcResult_0 = null; /* 101 */ funcResult_0 = value_MapObject_lambda_variable_1.toString(); /* 102 */ value_5 = (java.lang.String) funcResult_0; /* 103 */ /* 104 */ } /* 105 */ } /* 106 */ if (isNull_5) { /* 107 */ convertedArray_0[loopIndex_0] = null; /* 108 */ } else { /* 109 */ convertedArray_0[loopIndex_0] = value_5; /* 110 */ } /* 111 */ /* 112 */ loopIndex_0 += 1; /* 113 */ } /* 114 */ /* 115 */ value_3 = new org.apache.spark.sql.catalyst.util.GenericArrayData(convertedArray_0); /* 116 */ } /* 117 */ globalIsNull_0 = isNull_4; /* 118 */ return value_3; /* 119 */ } /* 120 */ /* 121 */ } 20/01/23 18:54:04.783 Executor task launch worker for task 14 INFO CodeGenerator: Code generated in 41.73641 ms 20/01/23 18:54:04.801 Executor task launch worker for task 14 INFO PythonRunner: Times: total = 46, boot = -867, init = 913, finish = 0 20/01/23 18:54:04.802 Executor task launch worker for task 16 INFO PythonRunner: Times: total = 95, boot = -895, init = 990, finish = 0 20/01/23 18:54:04.791 Executor task launch worker for task 15 DEBUG GenerateSafeProjection: code for createexternalrow(input[0, double, true], input[1, double, true], input[2, double, false], StructField(prediction,DoubleType,true), StructField(label,DoubleType,true), StructField(1.0,DoubleType,false)): /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificSafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificSafeProjection extends org.apache.spark.sql.catalyst.expressions.codegen.BaseProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private InternalRow mutableRow; /* 009 */ /* 010 */ /* 011 */ public SpecificSafeProjection(Object[] references) { /* 012 */ this.references = references; /* 013 */ mutableRow = (InternalRow) references[references.length - 1]; /* 014 */ /* 015 */ } /* 016 */ /* 017 */ public void initialize(int partitionIndex) { /* 018 */ /* 019 */ } /* 020 */ /* 021 */ public java.lang.Object apply(java.lang.Object _i) { /* 022 */ InternalRow i = (InternalRow) _i; /* 023 */ Object[] values_0 = new Object[3]; /* 024 */ /* 025 */ boolean isNull_1 = i.isNullAt(0); /* 026 */ double value_1 = isNull_1 ? /* 027 */ -1.0 : (i.getDouble(0)); /* 028 */ if (isNull_1) { /* 029 */ values_0[0] = null; /* 030 */ } else { /* 031 */ values_0[0] = value_1; /* 032 */ } /* 033 */ /* 034 */ boolean isNull_2 = i.isNullAt(1); /* 035 */ double value_2 = isNull_2 ? /* 036 */ -1.0 : (i.getDouble(1)); /* 037 */ if (isNull_2) { /* 038 */ values_0[1] = null; /* 039 */ } else { /* 040 */ values_0[1] = value_2; /* 041 */ } /* 042 */ /* 043 */ double value_3 = i.getDouble(2); /* 044 */ if (false) { /* 045 */ values_0[2] = null; /* 046 */ } else { /* 047 */ values_0[2] = value_3; /* 048 */ } /* 049 */ /* 050 */ final org.apache.spark.sql.Row value_0 = new org.apache.spark.sql.catalyst.expressions.GenericRowWithSchema(values_0, ((org.apache.spark.sql.types.StructType) references[0] /* schema */)); /* 051 */ if (false) { /* 052 */ mutableRow.setNullAt(0); /* 053 */ } else { /* 054 */ /* 055 */ mutableRow.update(0, value_0); /* 056 */ } /* 057 */ /* 058 */ return mutableRow; /* 059 */ } /* 060 */ /* 061 */ /* 062 */ } 20/01/23 18:54:04.805 Executor task launch worker for task 14 INFO Executor: Finished task 0.0 in stage 5.0 (TID 14). 2920 bytes result sent to driver 20/01/23 18:54:04.806 dispatcher-event-loop-3 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_5.0, runningTasks: 3 20/01/23 18:54:04.823 dispatcher-event-loop-3 DEBUG TaskSetManager: No tasks for locality level NO_PREF, so moving to locality level ANY 20/01/23 18:54:04.806 Executor task launch worker for task 14 DEBUG ExecutorMetricsPoller: stageTCMP: (5, 0) -> 3 20/01/23 18:54:04.823 task-result-getter-0 INFO TaskSetManager: Finished task 0.0 in stage 5.0 (TID 14) in 158 ms on 192.168.1.225 (executor driver) (1/4) 20/01/23 18:54:04.824 Executor task launch worker for task 16 INFO Executor: Finished task 2.0 in stage 5.0 (TID 16). 2920 bytes result sent to driver 20/01/23 18:54:04.824 Executor task launch worker for task 16 DEBUG ExecutorMetricsPoller: stageTCMP: (5, 0) -> 2 20/01/23 18:54:04.825 dispatcher-event-loop-1 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_5.0, runningTasks: 2 20/01/23 18:54:04.826 task-result-getter-3 INFO TaskSetManager: Finished task 2.0 in stage 5.0 (TID 16) in 160 ms on 192.168.1.225 (executor driver) (2/4) 20/01/23 18:54:04.812 Executor task launch worker for task 24 INFO PythonRunner: Times: total = 59, boot = -427, init = 486, finish = 0 20/01/23 18:54:04.829 Executor task launch worker for task 24 DEBUG TaskMemoryManager: Task 24 release 0.0 B from org.apache.spark.util.collection.ExternalSorter@58e1359 20/01/23 18:54:04.830 Executor task launch worker for task 24 INFO Executor: Finished task 0.0 in stage 7.0 (TID 24). 2082 bytes result sent to driver 20/01/23 18:54:04.831 Executor task launch worker for task 24 DEBUG ExecutorMetricsPoller: stageTCMP: (7, 0) -> 3 20/01/23 18:54:04.831 Executor task launch worker for task 15 INFO PythonRunner: Times: total = 55, boot = -900, init = 955, finish = 0 20/01/23 18:54:04.831 dispatcher-event-loop-3 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_7.0, runningTasks: 3 20/01/23 18:54:04.841 dispatcher-event-loop-3 DEBUG TaskSetManager: No tasks for locality level NO_PREF, so moving to locality level ANY 20/01/23 18:54:04.813 Executor task launch worker for task 27 DEBUG GenerateSafeProjection: code for createexternalrow(staticinvoke(class scala.collection.mutable.WrappedArray$, ObjectType(interface scala.collection.Seq), make, mapobjects(lambdavariable(MapObject, StringType, true, -1), lambdavariable(MapObject, StringType, true, -1).toString, input[0, array, true], None).array, true, false), StructField(words,ArrayType(StringType,true),true)): /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificSafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificSafeProjection extends org.apache.spark.sql.catalyst.expressions.codegen.BaseProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private InternalRow mutableRow; /* 009 */ private boolean resultIsNull_0; /* 010 */ private UTF8String value_MapObject_lambda_variable_1; /* 011 */ private boolean isNull_MapObject_lambda_variable_1; /* 012 */ private boolean globalIsNull_0; /* 013 */ private java.lang.Object[] mutableStateArray_0 = new java.lang.Object[1]; /* 014 */ /* 015 */ public SpecificSafeProjection(Object[] references) { /* 016 */ this.references = references; /* 017 */ mutableRow = (InternalRow) references[references.length - 1]; /* 018 */ /* 019 */ /* 020 */ } /* 021 */ /* 022 */ public void initialize(int partitionIndex) { /* 023 */ /* 024 */ } /* 025 */ /* 026 */ public java.lang.Object apply(java.lang.Object _i) { /* 027 */ InternalRow i = (InternalRow) _i; /* 028 */ Object[] values_0 = new Object[1]; /* 029 */ /* 030 */ resultIsNull_0 = false; /* 031 */ if (!resultIsNull_0) { /* 032 */ ArrayData value_6 = MapObjects_0(i); /* 033 */ boolean isNull_2 = true; /* 034 */ java.lang.Object value_2 = null; /* 035 */ if (!globalIsNull_0) { /* 036 */ /* 037 */ isNull_2 = false; /* 038 */ if (!isNull_2) { /* 039 */ /* 040 */ Object funcResult_1 = null; /* 041 */ funcResult_1 = value_6.array(); /* 042 */ value_2 = (java.lang.Object) funcResult_1; /* 043 */ /* 044 */ } /* 045 */ } /* 046 */ resultIsNull_0 = isNull_2; /* 047 */ mutableStateArray_0[0] = value_2; /* 048 */ } /* 049 */ /* 050 */ boolean isNull_1 = resultIsNull_0; /* 051 */ scala.collection.Seq value_1 = null; /* 052 */ if (!resultIsNull_0) { /* 053 */ value_1 = scala.collection.mutable.WrappedArray.make(mutableStateArray_0[0]); /* 054 */ } /* 055 */ if (isNull_1) { /* 056 */ values_0[0] = null; /* 057 */ } else { /* 058 */ values_0[0] = value_1; /* 059 */ } /* 060 */ /* 061 */ final org.apache.spark.sql.Row value_0 = new org.apache.spark.sql.catalyst.expressions.GenericRowWithSchema(values_0, ((org.apache.spark.sql.types.StructType) references[0] /* schema */)); /* 062 */ if (false) { /* 063 */ mutableRow.setNullAt(0); /* 064 */ } else { /* 065 */ /* 066 */ mutableRow.update(0, value_0); /* 067 */ } /* 068 */ /* 069 */ return mutableRow; /* 070 */ } /* 071 */ /* 072 */ /* 073 */ private ArrayData MapObjects_0(InternalRow i) { /* 074 */ boolean isNull_4 = i.isNullAt(0); /* 075 */ ArrayData value_4 = isNull_4 ? /* 076 */ null : (i.getArray(0)); /* 077 */ ArrayData value_3 = null; /* 078 */ /* 079 */ if (!isNull_4) { /* 080 */ /* 081 */ int dataLength_0 = value_4.numElements(); /* 082 */ /* 083 */ java.lang.String[] convertedArray_0 = null; /* 084 */ convertedArray_0 = new java.lang.String[dataLength_0]; /* 085 */ /* 086 */ /* 087 */ int loopIndex_0 = 0; /* 088 */ /* 089 */ while (loopIndex_0 < dataLength_0) { /* 090 */ value_MapObject_lambda_variable_1 = (UTF8String) (value_4.getUTF8String(loopIndex_0)); /* 091 */ isNull_MapObject_lambda_variable_1 = value_4.isNullAt(loopIndex_0); /* 092 */ /* 093 */ boolean isNull_5 = true; /* 094 */ java.lang.String value_5 = null; /* 095 */ if (!isNull_MapObject_lambda_variable_1) { /* 096 */ /* 097 */ isNull_5 = false; /* 098 */ if (!isNull_5) { /* 099 */ /* 100 */ Object funcResult_0 = null; /* 101 */ funcResult_0 = value_MapObject_lambda_variable_1.toString(); /* 102 */ value_5 = (java.lang.String) funcResult_0; /* 103 */ /* 104 */ } /* 105 */ } /* 106 */ if (isNull_5) { /* 107 */ convertedArray_0[loopIndex_0] = null; /* 108 */ } else { /* 109 */ convertedArray_0[loopIndex_0] = value_5; /* 110 */ } /* 111 */ /* 112 */ loopIndex_0 += 1; /* 113 */ } /* 114 */ /* 115 */ value_3 = new org.apache.spark.sql.catalyst.util.GenericArrayData(convertedArray_0); /* 116 */ } /* 117 */ globalIsNull_0 = isNull_4; /* 118 */ return value_3; /* 119 */ } /* 120 */ /* 121 */ } 20/01/23 18:54:04.835 Executor task launch worker for task 25 INFO PythonRunner: Times: total = 95, boot = -398, init = 493, finish = 0 20/01/23 18:54:04.841 Executor task launch worker for task 17 INFO PythonRunner: Times: total = 67, boot = -973, init = 1040, finish = 0 20/01/23 18:54:04.841 task-result-getter-1 INFO TaskSetManager: Finished task 0.0 in stage 7.0 (TID 24) in 125 ms on 192.168.1.225 (executor driver) (1/4) 20/01/23 18:54:04.832 Executor task launch worker for task 26 DEBUG GenerateSafeProjection: code for createexternalrow(staticinvoke(class scala.collection.mutable.WrappedArray$, ObjectType(interface scala.collection.Seq), make, mapobjects(lambdavariable(MapObject, StringType, true, -1), lambdavariable(MapObject, StringType, true, -1).toString, input[0, array, true], None).array, true, false), StructField(words,ArrayType(StringType,true),true)): /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificSafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificSafeProjection extends org.apache.spark.sql.catalyst.expressions.codegen.BaseProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private InternalRow mutableRow; /* 009 */ private boolean resultIsNull_0; /* 010 */ private UTF8String value_MapObject_lambda_variable_1; /* 011 */ private boolean isNull_MapObject_lambda_variable_1; /* 012 */ private boolean globalIsNull_0; /* 013 */ private java.lang.Object[] mutableStateArray_0 = new java.lang.Object[1]; /* 014 */ /* 015 */ public SpecificSafeProjection(Object[] references) { /* 016 */ this.references = references; /* 017 */ mutableRow = (InternalRow) references[references.length - 1]; /* 018 */ /* 019 */ /* 020 */ } /* 021 */ /* 022 */ public void initialize(int partitionIndex) { /* 023 */ /* 024 */ } /* 025 */ /* 026 */ public java.lang.Object apply(java.lang.Object _i) { /* 027 */ InternalRow i = (InternalRow) _i; /* 028 */ Object[] values_0 = new Object[1]; /* 029 */ /* 030 */ resultIsNull_0 = false; /* 031 */ if (!resultIsNull_0) { /* 032 */ ArrayData value_6 = MapObjects_0(i); /* 033 */ boolean isNull_2 = true; /* 034 */ java.lang.Object value_2 = null; /* 035 */ if (!globalIsNull_0) { /* 036 */ /* 037 */ isNull_2 = false; /* 038 */ if (!isNull_2) { /* 039 */ /* 040 */ Object funcResult_1 = null; /* 041 */ funcResult_1 = value_6.array(); /* 042 */ value_2 = (java.lang.Object) funcResult_1; /* 043 */ /* 044 */ } /* 045 */ } /* 046 */ resultIsNull_0 = isNull_2; /* 047 */ mutableStateArray_0[0] = value_2; /* 048 */ } /* 049 */ /* 050 */ boolean isNull_1 = resultIsNull_0; /* 051 */ scala.collection.Seq value_1 = null; /* 052 */ if (!resultIsNull_0) { /* 053 */ value_1 = scala.collection.mutable.WrappedArray.make(mutableStateArray_0[0]); /* 054 */ } /* 055 */ if (isNull_1) { /* 056 */ values_0[0] = null; /* 057 */ } else { /* 058 */ values_0[0] = value_1; /* 059 */ } /* 060 */ /* 061 */ final org.apache.spark.sql.Row value_0 = new org.apache.spark.sql.catalyst.expressions.GenericRowWithSchema(values_0, ((org.apache.spark.sql.types.StructType) references[0] /* schema */)); /* 062 */ if (false) { /* 063 */ mutableRow.setNullAt(0); /* 064 */ } else { /* 065 */ /* 066 */ mutableRow.update(0, value_0); /* 067 */ } /* 068 */ /* 069 */ return mutableRow; /* 070 */ } /* 071 */ /* 072 */ /* 073 */ private ArrayData MapObjects_0(InternalRow i) { /* 074 */ boolean isNull_4 = i.isNullAt(0); /* 075 */ ArrayData value_4 = isNull_4 ? /* 076 */ null : (i.getArray(0)); /* 077 */ ArrayData value_3 = null; /* 078 */ /* 079 */ if (!isNull_4) { /* 080 */ /* 081 */ int dataLength_0 = value_4.numElements(); /* 082 */ /* 083 */ java.lang.String[] convertedArray_0 = null; /* 084 */ convertedArray_0 = new java.lang.String[dataLength_0]; /* 085 */ /* 086 */ /* 087 */ int loopIndex_0 = 0; /* 088 */ /* 089 */ while (loopIndex_0 < dataLength_0) { /* 090 */ value_MapObject_lambda_variable_1 = (UTF8String) (value_4.getUTF8String(loopIndex_0)); /* 091 */ isNull_MapObject_lambda_variable_1 = value_4.isNullAt(loopIndex_0); /* 092 */ /* 093 */ boolean isNull_5 = true; /* 094 */ java.lang.String value_5 = null; /* 095 */ if (!isNull_MapObject_lambda_variable_1) { /* 096 */ /* 097 */ isNull_5 = false; /* 098 */ if (!isNull_5) { /* 099 */ /* 100 */ Object funcResult_0 = null; /* 101 */ funcResult_0 = value_MapObject_lambda_variable_1.toString(); /* 102 */ value_5 = (java.lang.String) funcResult_0; /* 103 */ /* 104 */ } /* 105 */ } /* 106 */ if (isNull_5) { /* 107 */ convertedArray_0[loopIndex_0] = null; /* 108 */ } else { /* 109 */ convertedArray_0[loopIndex_0] = value_5; /* 110 */ } /* 111 */ /* 112 */ loopIndex_0 += 1; /* 113 */ } /* 114 */ /* 115 */ value_3 = new org.apache.spark.sql.catalyst.util.GenericArrayData(convertedArray_0); /* 116 */ } /* 117 */ globalIsNull_0 = isNull_4; /* 118 */ return value_3; /* 119 */ } /* 120 */ /* 121 */ } 20/01/23 18:54:04.843 Executor task launch worker for task 25 DEBUG TaskMemoryManager: Task 25 release 0.0 B from org.apache.spark.util.collection.ExternalSorter@42f480a3 20/01/23 18:54:04.843 Executor task launch worker for task 26 INFO PythonRunner: Times: total = 70, boot = -370, init = 440, finish = 0 20/01/23 18:54:04.847 Executor task launch worker for task 26 DEBUG TaskMemoryManager: Task 26 release 0.0 B from org.apache.spark.util.collection.ExternalSorter@76b8422a 20/01/23 18:54:04.853 Executor task launch worker for task 25 INFO Executor: Finished task 1.0 in stage 7.0 (TID 25). 2082 bytes result sent to driver 20/01/23 18:54:04.853 Executor task launch worker for task 17 INFO Executor: Finished task 3.0 in stage 5.0 (TID 17). 3101 bytes result sent to driver 20/01/23 18:54:04.853 Executor task launch worker for task 25 DEBUG ExecutorMetricsPoller: stageTCMP: (7, 0) -> 2 20/01/23 18:54:04.854 dispatcher-event-loop-1 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_7.0, runningTasks: 2 20/01/23 18:54:04.856 dispatcher-event-loop-0 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_5.0, runningTasks: 1 20/01/23 18:54:04.856 Executor task launch worker for task 17 DEBUG ExecutorMetricsPoller: stageTCMP: (5, 0) -> 1 20/01/23 18:54:04.857 task-result-getter-2 INFO TaskSetManager: Finished task 3.0 in stage 5.0 (TID 17) in 191 ms on 192.168.1.225 (executor driver) (3/4) 20/01/23 18:54:04.859 Executor task launch worker for task 26 INFO Executor: Finished task 2.0 in stage 7.0 (TID 26). 2082 bytes result sent to driver 20/01/23 18:54:04.859 Executor task launch worker for task 27 INFO PythonRunner: Times: total = 90, boot = -396, init = 486, finish = 0 20/01/23 18:54:04.859 Executor task launch worker for task 26 DEBUG ExecutorMetricsPoller: stageTCMP: (7, 0) -> 1 20/01/23 18:54:04.860 Executor task launch worker for task 27 DEBUG TaskMemoryManager: Task 27 release 0.0 B from org.apache.spark.util.collection.ExternalSorter@3fee2d73 20/01/23 18:54:04.861 Executor task launch worker for task 27 INFO Executor: Finished task 3.0 in stage 7.0 (TID 27). 2082 bytes result sent to driver 20/01/23 18:54:04.862 Executor task launch worker for task 27 DEBUG ExecutorMetricsPoller: removing (7, 0) from stageTCMP 20/01/23 18:54:04.862 Executor task launch worker for task 15 INFO Executor: Finished task 1.0 in stage 5.0 (TID 15). 2920 bytes result sent to driver 20/01/23 18:54:04.862 Executor task launch worker for task 15 DEBUG ExecutorMetricsPoller: removing (5, 0) from stageTCMP 20/01/23 18:54:04.865 dag-scheduler-event-loop DEBUG DAGScheduler: ShuffleMapTask finished on driver 20/01/23 18:54:04.866 dispatcher-event-loop-1 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_7.0, runningTasks: 1 20/01/23 18:54:04.867 dispatcher-event-loop-1 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_7.0, runningTasks: 0 20/01/23 18:54:04.867 task-result-getter-0 INFO TaskSetManager: Finished task 1.0 in stage 7.0 (TID 25) in 151 ms on 192.168.1.225 (executor driver) (2/4) 20/01/23 18:54:04.867 task-result-getter-2 INFO TaskSetManager: Finished task 2.0 in stage 7.0 (TID 26) in 151 ms on 192.168.1.225 (executor driver) (3/4) 20/01/23 18:54:04.868 task-result-getter-3 INFO TaskSetManager: Finished task 3.0 in stage 7.0 (TID 27) in 151 ms on 192.168.1.225 (executor driver) (4/4) 20/01/23 18:54:04.868 task-result-getter-3 INFO TaskSchedulerImpl: Removed TaskSet 7.0, whose tasks have all completed, from pool 20/01/23 18:54:04.868 dag-scheduler-event-loop DEBUG DAGScheduler: ShuffleMapTask finished on driver 20/01/23 18:54:04.869 dag-scheduler-event-loop DEBUG DAGScheduler: ShuffleMapTask finished on driver 20/01/23 18:54:04.870 dag-scheduler-event-loop DEBUG DAGScheduler: ShuffleMapTask finished on driver 20/01/23 18:54:04.871 dag-scheduler-event-loop INFO DAGScheduler: ShuffleMapStage 7 (flatMap at CountVectorizer.scala:212) finished in 0.187 s 20/01/23 18:54:04.871 dag-scheduler-event-loop INFO DAGScheduler: looking for newly runnable stages 20/01/23 18:54:04.871 dag-scheduler-event-loop INFO DAGScheduler: running: Set() 20/01/23 18:54:04.871 dag-scheduler-event-loop INFO DAGScheduler: waiting: Set(ResultStage 8) 20/01/23 18:54:04.871 dag-scheduler-event-loop INFO DAGScheduler: failed: Set() 20/01/23 18:54:04.871 dag-scheduler-event-loop DEBUG MapOutputTrackerMaster: Increasing epoch to 2 20/01/23 18:54:04.871 dag-scheduler-event-loop DEBUG DAGScheduler: submitStage(ResultStage 8 (name=count at CountVectorizer.scala:233;jobs=5)) 20/01/23 18:54:04.873 dag-scheduler-event-loop DEBUG DAGScheduler: missing: List() 20/01/23 18:54:04.873 dag-scheduler-event-loop INFO DAGScheduler: Submitting ResultStage 8 (MapPartitionsRDD[38] at map at CountVectorizer.scala:230), which has no missing parents 20/01/23 18:54:04.873 dag-scheduler-event-loop DEBUG DAGScheduler: submitMissingTasks(ResultStage 8) 20/01/23 18:54:04.879 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_10 stored as values in memory (estimated size 5.3 KiB, free 366.2 MiB) 20/01/23 18:54:04.879 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_10 locally took 0 ms 20/01/23 18:54:04.879 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_10 without replication took 0 ms 20/01/23 18:54:04.880 dispatcher-event-loop-2 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_5.0, runningTasks: 0 20/01/23 18:54:04.880 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_10_piece0 stored as bytes in memory (estimated size 2.9 KiB, free 366.2 MiB) 20/01/23 18:54:04.881 task-result-getter-1 INFO TaskSetManager: Finished task 1.0 in stage 5.0 (TID 15) in 215 ms on 192.168.1.225 (executor driver) (4/4) 20/01/23 18:54:04.881 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added broadcast_10_piece0 in memory on 192.168.1.225:41233 (size: 2.9 KiB, free: 366.3 MiB) 20/01/23 18:54:04.881 task-result-getter-1 INFO TaskSchedulerImpl: Removed TaskSet 5.0, whose tasks have all completed, from pool 20/01/23 18:54:04.882 dag-scheduler-event-loop INFO DAGScheduler: ResultStage 5 (treeAggregate at Statistics.scala:58) finished in 0.249 s 20/01/23 18:54:04.882 dag-scheduler-event-loop DEBUG DAGScheduler: After removal of stage 5, remaining stages = 0 20/01/23 18:54:04.882 dag-scheduler-event-loop INFO DAGScheduler: Job 3 is finished. Cancelling potential speculative or zombie tasks for this job 20/01/23 18:54:04.882 dag-scheduler-event-loop INFO TaskSchedulerImpl: Killing all running tasks in stage 5: Stage finished 20/01/23 18:54:04.883 Thread-4 INFO DAGScheduler: Job 3 finished: treeAggregate at Statistics.scala:58, took 0.254285 s 20/01/23 18:54:04.889 dag-scheduler-event-loop DEBUG BlockManagerMaster: Updated info of block broadcast_10_piece0 20/01/23 18:54:04.889 dag-scheduler-event-loop DEBUG BlockManager: Told master about block broadcast_10_piece0 20/01/23 18:54:04.889 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_10_piece0 locally took 9 ms 20/01/23 18:54:04.889 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_10_piece0 without replication took 9 ms 20/01/23 18:54:04.890 dag-scheduler-event-loop INFO SparkContext: Created broadcast 10 from broadcast at DAGScheduler.scala:1200 20/01/23 18:54:04.890 dag-scheduler-event-loop INFO DAGScheduler: Submitting 4 missing tasks from ResultStage 8 (MapPartitionsRDD[38] at map at CountVectorizer.scala:230) (first 15 tasks are for partitions Vector(0, 1, 2, 3)) 20/01/23 18:54:04.890 dag-scheduler-event-loop INFO TaskSchedulerImpl: Adding task set 8.0 with 4 tasks 20/01/23 18:54:04.891 dag-scheduler-event-loop DEBUG TaskSetManager: Epoch for TaskSet 8.0: 2 20/01/23 18:54:04.891 dag-scheduler-event-loop DEBUG TaskSetManager: Adding pending tasks took 0 ms 20/01/23 18:54:04.891 dag-scheduler-event-loop DEBUG TaskSetManager: Valid locality levels for TaskSet 8.0: NODE_LOCAL, ANY 20/01/23 18:54:04.892 dispatcher-event-loop-3 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_8.0, runningTasks: 0 20/01/23 18:54:04.892 dispatcher-event-loop-3 INFO TaskSetManager: Starting task 0.0 in stage 8.0 (TID 28, 192.168.1.225, executor driver, partition 0, NODE_LOCAL, 7143 bytes) 20/01/23 18:54:04.892 dispatcher-event-loop-3 INFO TaskSetManager: Starting task 1.0 in stage 8.0 (TID 29, 192.168.1.225, executor driver, partition 1, NODE_LOCAL, 7143 bytes) 20/01/23 18:54:04.893 dispatcher-event-loop-3 INFO TaskSetManager: Starting task 2.0 in stage 8.0 (TID 30, 192.168.1.225, executor driver, partition 2, NODE_LOCAL, 7143 bytes) 20/01/23 18:54:04.893 dispatcher-event-loop-3 INFO TaskSetManager: Starting task 3.0 in stage 8.0 (TID 31, 192.168.1.225, executor driver, partition 3, NODE_LOCAL, 7143 bytes) 20/01/23 18:54:04.893 Executor task launch worker for task 30 INFO Executor: Running task 2.0 in stage 8.0 (TID 30) 20/01/23 18:54:04.893 Executor task launch worker for task 29 INFO Executor: Running task 1.0 in stage 8.0 (TID 29) 20/01/23 18:54:04.894 Executor task launch worker for task 30 DEBUG ExecutorMetricsPoller: stageTCMP: (8, 0) -> 1 20/01/23 18:54:04.894 Executor task launch worker for task 29 DEBUG ExecutorMetricsPoller: stageTCMP: (8, 0) -> 2 20/01/23 18:54:04.895 Executor task launch worker for task 30 DEBUG BlockManager: Getting local block broadcast_10 20/01/23 18:54:04.895 Executor task launch worker for task 30 DEBUG BlockManager: Level for block broadcast_10 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:04.898 Executor task launch worker for task 30 DEBUG BlockManager: Getting local block rdd_38_2 20/01/23 18:54:04.898 Executor task launch worker for task 29 DEBUG BlockManager: Getting local block rdd_38_1 20/01/23 18:54:04.898 Executor task launch worker for task 29 DEBUG BlockManager: Block rdd_38_1 was not found 20/01/23 18:54:04.898 Executor task launch worker for task 29 DEBUG BlockManager: Getting remote block rdd_38_1 20/01/23 18:54:04.898 Executor task launch worker for task 31 INFO Executor: Running task 3.0 in stage 8.0 (TID 31) 20/01/23 18:54:04.899 Executor task launch worker for task 31 DEBUG ExecutorMetricsPoller: stageTCMP: (8, 0) -> 3 20/01/23 18:54:04.899 Executor task launch worker for task 29 DEBUG BlockManager: Block rdd_38_1 is unknown by block manager master 20/01/23 18:54:04.899 Executor task launch worker for task 29 DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 1, partitions 1-2 20/01/23 18:54:04.900 Executor task launch worker for task 29 DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 20/01/23 18:54:04.900 Executor task launch worker for task 29 INFO ShuffleBlockFetcherIterator: Getting 4 (756.0 B) non-empty blocks including 4 (756.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 20/01/23 18:54:04.900 Executor task launch worker for task 29 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 20/01/23 18:54:04.900 Executor task launch worker for task 29 DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_1_24_1,0), (shuffle_1_25_1,1), (shuffle_1_26_1,2), (shuffle_1_27_1,3) 20/01/23 18:54:04.900 Executor task launch worker for task 31 DEBUG BlockManager: Getting local block rdd_38_3 20/01/23 18:54:04.900 Executor task launch worker for task 31 DEBUG BlockManager: Block rdd_38_3 was not found 20/01/23 18:54:04.900 Executor task launch worker for task 31 DEBUG BlockManager: Getting remote block rdd_38_3 20/01/23 18:54:04.901 Executor task launch worker for task 31 DEBUG BlockManager: Block rdd_38_3 is unknown by block manager master 20/01/23 18:54:04.901 Executor task launch worker for task 31 DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 1, partitions 3-4 20/01/23 18:54:04.901 Executor task launch worker for task 29 DEBUG ShuffleBlockFetcherIterator: Got local blocks in 1 ms 20/01/23 18:54:04.901 Executor task launch worker for task 31 DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 20/01/23 18:54:04.898 Executor task launch worker for task 30 DEBUG BlockManager: Block rdd_38_2 was not found 20/01/23 18:54:04.902 Executor task launch worker for task 30 DEBUG BlockManager: Getting remote block rdd_38_2 20/01/23 18:54:04.902 Executor task launch worker for task 31 INFO ShuffleBlockFetcherIterator: Getting 2 (378.0 B) non-empty blocks including 2 (378.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 20/01/23 18:54:04.902 Executor task launch worker for task 31 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 20/01/23 18:54:04.902 Executor task launch worker for task 31 DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_1_24_3,0), (shuffle_1_25_3,1) 20/01/23 18:54:04.902 Executor task launch worker for task 30 DEBUG BlockManager: Block rdd_38_2 is unknown by block manager master 20/01/23 18:54:04.902 Executor task launch worker for task 30 DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 1, partitions 2-3 20/01/23 18:54:04.903 Executor task launch worker for task 30 DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 20/01/23 18:54:04.903 Executor task launch worker for task 30 INFO ShuffleBlockFetcherIterator: Getting 3 (567.0 B) non-empty blocks including 3 (567.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 20/01/23 18:54:04.903 Executor task launch worker for task 30 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 20/01/23 18:54:04.903 Executor task launch worker for task 30 DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_1_24_2,0), (shuffle_1_25_2,1), (shuffle_1_26_2,2) 20/01/23 18:54:04.904 Executor task launch worker for task 30 DEBUG ShuffleBlockFetcherIterator: Got local blocks in 1 ms 20/01/23 18:54:04.902 Executor task launch worker for task 31 DEBUG ShuffleBlockFetcherIterator: Got local blocks in 1 ms 20/01/23 18:54:04.905 Executor task launch worker for task 30 DEBUG TaskMemoryManager: Task 30 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@1eb29339 20/01/23 18:54:04.906 Executor task launch worker for task 30 INFO MemoryStore: Block rdd_38_2 stored as values in memory (estimated size 120.0 B, free 366.2 MiB) 20/01/23 18:54:04.907 Executor task launch worker for task 31 DEBUG TaskMemoryManager: Task 31 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@24830765 20/01/23 18:54:04.907 Executor task launch worker for task 31 INFO MemoryStore: Block rdd_38_3 stored as values in memory (estimated size 120.0 B, free 366.2 MiB) 20/01/23 18:54:04.908 Executor task launch worker for task 28 INFO Executor: Running task 0.0 in stage 8.0 (TID 28) 20/01/23 18:54:04.908 Executor task launch worker for task 29 DEBUG TaskMemoryManager: Task 29 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@7a59bfdb 20/01/23 18:54:04.908 Executor task launch worker for task 29 INFO MemoryStore: Block rdd_38_1 stored as values in memory (estimated size 16.0 B, free 366.2 MiB) 20/01/23 18:54:04.909 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_38_2 in memory on 192.168.1.225:41233 (size: 120.0 B, free: 366.3 MiB) 20/01/23 18:54:04.909 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_38_1 in memory on 192.168.1.225:41233 (size: 16.0 B, free: 366.3 MiB) 20/01/23 18:54:04.910 Executor task launch worker for task 29 DEBUG BlockManagerMaster: Updated info of block rdd_38_1 20/01/23 18:54:04.910 Executor task launch worker for task 29 DEBUG BlockManager: Told master about block rdd_38_1 20/01/23 18:54:04.910 Executor task launch worker for task 29 DEBUG BlockManager: Put block rdd_38_1 locally took 10 ms 20/01/23 18:54:04.910 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_38_3 in memory on 192.168.1.225:41233 (size: 120.0 B, free: 366.3 MiB) 20/01/23 18:54:04.910 Executor task launch worker for task 28 DEBUG ExecutorMetricsPoller: stageTCMP: (8, 0) -> 4 20/01/23 18:54:04.911 Executor task launch worker for task 29 DEBUG BlockManager: Putting block rdd_38_1 without replication took 10 ms 20/01/23 18:54:04.911 Executor task launch worker for task 29 DEBUG BlockManager: Getting local block rdd_38_1 20/01/23 18:54:04.911 Executor task launch worker for task 29 DEBUG BlockManager: Level for block rdd_38_1 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:04.912 Executor task launch worker for task 29 INFO Executor: Finished task 1.0 in stage 8.0 (TID 29). 1305 bytes result sent to driver 20/01/23 18:54:04.912 Executor task launch worker for task 29 DEBUG ExecutorMetricsPoller: stageTCMP: (8, 0) -> 3 20/01/23 18:54:04.912 dispatcher-event-loop-0 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_8.0, runningTasks: 3 20/01/23 18:54:04.912 Executor task launch worker for task 30 DEBUG BlockManagerMaster: Updated info of block rdd_38_2 20/01/23 18:54:04.913 Executor task launch worker for task 30 DEBUG BlockManager: Told master about block rdd_38_2 20/01/23 18:54:04.913 Executor task launch worker for task 30 DEBUG BlockManager: Put block rdd_38_2 locally took 10 ms 20/01/23 18:54:04.913 Executor task launch worker for task 28 DEBUG BlockManager: Getting local block rdd_38_0 20/01/23 18:54:04.913 dispatcher-event-loop-0 DEBUG TaskSetManager: No tasks for locality level NODE_LOCAL, so moving to locality level ANY 20/01/23 18:54:04.913 Executor task launch worker for task 28 DEBUG BlockManager: Block rdd_38_0 was not found 20/01/23 18:54:04.913 Executor task launch worker for task 28 DEBUG BlockManager: Getting remote block rdd_38_0 20/01/23 18:54:04.913 Executor task launch worker for task 31 DEBUG BlockManagerMaster: Updated info of block rdd_38_3 20/01/23 18:54:04.913 Executor task launch worker for task 30 DEBUG BlockManager: Putting block rdd_38_2 without replication took 10 ms 20/01/23 18:54:04.913 Executor task launch worker for task 28 DEBUG BlockManager: Block rdd_38_0 is unknown by block manager master 20/01/23 18:54:04.913 Executor task launch worker for task 31 DEBUG BlockManager: Told master about block rdd_38_3 20/01/23 18:54:04.913 task-result-getter-1 INFO TaskSetManager: Finished task 1.0 in stage 8.0 (TID 29) in 21 ms on 192.168.1.225 (executor driver) (1/4) 20/01/23 18:54:04.913 Executor task launch worker for task 28 DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 1, partitions 0-1 20/01/23 18:54:04.914 Executor task launch worker for task 28 DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 20/01/23 18:54:04.914 Executor task launch worker for task 28 INFO ShuffleBlockFetcherIterator: Getting 1 (189.0 B) non-empty blocks including 1 (189.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 20/01/23 18:54:04.914 Executor task launch worker for task 28 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 20/01/23 18:54:04.914 Executor task launch worker for task 28 DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_1_24_0,0) 20/01/23 18:54:04.915 Executor task launch worker for task 28 DEBUG ShuffleBlockFetcherIterator: Got local blocks in 0 ms 20/01/23 18:54:04.916 Executor task launch worker for task 28 DEBUG TaskMemoryManager: Task 28 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@3bbc8343 20/01/23 18:54:04.916 Executor task launch worker for task 28 INFO MemoryStore: Block rdd_38_0 stored as values in memory (estimated size 120.0 B, free 366.2 MiB) 20/01/23 18:54:04.913 Executor task launch worker for task 30 DEBUG BlockManager: Getting local block rdd_38_2 20/01/23 18:54:04.916 Executor task launch worker for task 30 DEBUG BlockManager: Level for block rdd_38_2 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:04.917 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_38_0 in memory on 192.168.1.225:41233 (size: 120.0 B, free: 366.3 MiB) 20/01/23 18:54:04.917 Executor task launch worker for task 30 INFO Executor: Finished task 2.0 in stage 8.0 (TID 30). 1305 bytes result sent to driver 20/01/23 18:54:04.913 Executor task launch worker for task 31 DEBUG BlockManager: Put block rdd_38_3 locally took 12 ms 20/01/23 18:54:04.918 Executor task launch worker for task 31 DEBUG BlockManager: Putting block rdd_38_3 without replication took 16 ms 20/01/23 18:54:04.918 Executor task launch worker for task 31 DEBUG BlockManager: Getting local block rdd_38_3 20/01/23 18:54:04.918 Executor task launch worker for task 31 DEBUG BlockManager: Level for block rdd_38_3 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:04.918 Executor task launch worker for task 31 INFO Executor: Finished task 3.0 in stage 8.0 (TID 31). 1305 bytes result sent to driver 20/01/23 18:54:04.919 Executor task launch worker for task 30 DEBUG ExecutorMetricsPoller: stageTCMP: (8, 0) -> 2 20/01/23 18:54:04.919 Executor task launch worker for task 31 DEBUG ExecutorMetricsPoller: stageTCMP: (8, 0) -> 1 20/01/23 18:54:04.919 Executor task launch worker for task 28 DEBUG BlockManagerMaster: Updated info of block rdd_38_0 20/01/23 18:54:04.919 Executor task launch worker for task 28 DEBUG BlockManager: Told master about block rdd_38_0 20/01/23 18:54:04.919 Executor task launch worker for task 28 DEBUG BlockManager: Put block rdd_38_0 locally took 5 ms 20/01/23 18:54:04.919 dispatcher-event-loop-2 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_8.0, runningTasks: 2 20/01/23 18:54:04.919 Executor task launch worker for task 28 DEBUG BlockManager: Putting block rdd_38_0 without replication took 5 ms 20/01/23 18:54:04.919 Executor task launch worker for task 28 DEBUG BlockManager: Getting local block rdd_38_0 20/01/23 18:54:04.919 Executor task launch worker for task 28 DEBUG BlockManager: Level for block rdd_38_0 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:04.920 dispatcher-event-loop-2 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_8.0, runningTasks: 1 20/01/23 18:54:04.920 task-result-getter-2 INFO TaskSetManager: Finished task 3.0 in stage 8.0 (TID 31) in 27 ms on 192.168.1.225 (executor driver) (2/4) 20/01/23 18:54:04.920 Executor task launch worker for task 28 INFO Executor: Finished task 0.0 in stage 8.0 (TID 28). 1305 bytes result sent to driver 20/01/23 18:54:04.920 task-result-getter-0 INFO TaskSetManager: Finished task 2.0 in stage 8.0 (TID 30) in 27 ms on 192.168.1.225 (executor driver) (3/4) 20/01/23 18:54:04.920 dispatcher-event-loop-3 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_8.0, runningTasks: 0 20/01/23 18:54:04.921 Executor task launch worker for task 28 DEBUG ExecutorMetricsPoller: removing (8, 0) from stageTCMP 20/01/23 18:54:04.921 task-result-getter-3 INFO TaskSetManager: Finished task 0.0 in stage 8.0 (TID 28) in 29 ms on 192.168.1.225 (executor driver) (4/4) 20/01/23 18:54:04.921 task-result-getter-3 INFO TaskSchedulerImpl: Removed TaskSet 8.0, whose tasks have all completed, from pool 20/01/23 18:54:04.922 dag-scheduler-event-loop INFO DAGScheduler: ResultStage 8 (count at CountVectorizer.scala:233) finished in 0.048 s 20/01/23 18:54:04.922 dag-scheduler-event-loop DEBUG DAGScheduler: After removal of stage 8, remaining stages = 1 20/01/23 18:54:04.922 dag-scheduler-event-loop DEBUG DAGScheduler: After removal of stage 7, remaining stages = 0 20/01/23 18:54:04.922 dag-scheduler-event-loop INFO DAGScheduler: Job 5 is finished. Cancelling potential speculative or zombie tasks for this job 20/01/23 18:54:04.922 dag-scheduler-event-loop INFO TaskSchedulerImpl: Killing all running tasks in stage 8: Stage finished 20/01/23 18:54:04.922 Thread-4 INFO DAGScheduler: Job 5 finished: count at CountVectorizer.scala:233, took 0.247519 s 20/01/23 18:54:04.924 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$takeOrdered$2 20/01/23 18:54:04.938 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$takeOrdered$2) is now cleaned +++ 20/01/23 18:54:04.939 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$takeOrdered$3 20/01/23 18:54:04.941 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$takeOrdered$3) is now cleaned +++ 20/01/23 18:54:04.942 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$runJob$6 20/01/23 18:54:04.942 Thread-4 DEBUG Analyzer$ResolveReferences: Resolving 'prediction to prediction#30 20/01/23 18:54:04.942 Thread-4 DEBUG Analyzer$ResolveReferences: Resolving 'label to label#29 20/01/23 18:54:04.944 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$runJob$6) is now cleaned +++ 20/01/23 18:54:04.944 Thread-4 INFO SparkContext: Starting job: top at CountVectorizer.scala:236 20/01/23 18:54:04.946 dag-scheduler-event-loop INFO DAGScheduler: Got job 6 (top at CountVectorizer.scala:236) with 4 output partitions 20/01/23 18:54:04.946 dag-scheduler-event-loop INFO DAGScheduler: Final stage: ResultStage 10 (top at CountVectorizer.scala:236) 20/01/23 18:54:04.946 dag-scheduler-event-loop INFO DAGScheduler: Parents of final stage: List(ShuffleMapStage 9) 20/01/23 18:54:04.946 dag-scheduler-event-loop INFO DAGScheduler: Missing parents: List() 20/01/23 18:54:04.947 dag-scheduler-event-loop DEBUG DAGScheduler: submitStage(ResultStage 10 (name=top at CountVectorizer.scala:236;jobs=6)) 20/01/23 18:54:04.947 dag-scheduler-event-loop DEBUG DAGScheduler: missing: List() 20/01/23 18:54:04.947 dag-scheduler-event-loop INFO DAGScheduler: Submitting ResultStage 10 (MapPartitionsRDD[39] at top at CountVectorizer.scala:236), which has no missing parents 20/01/23 18:54:04.947 dag-scheduler-event-loop DEBUG DAGScheduler: submitMissingTasks(ResultStage 10) 20/01/23 18:54:04.950 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_11 stored as values in memory (estimated size 6.3 KiB, free 366.2 MiB) 20/01/23 18:54:04.950 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_11 locally took 0 ms 20/01/23 18:54:04.950 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_11 without replication took 1 ms 20/01/23 18:54:04.951 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_11_piece0 stored as bytes in memory (estimated size 3.2 KiB, free 366.2 MiB) 20/01/23 18:54:04.954 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added broadcast_11_piece0 in memory on 192.168.1.225:41233 (size: 3.2 KiB, free: 366.3 MiB) 20/01/23 18:54:04.954 dag-scheduler-event-loop DEBUG BlockManagerMaster: Updated info of block broadcast_11_piece0 20/01/23 18:54:04.954 dag-scheduler-event-loop DEBUG BlockManager: Told master about block broadcast_11_piece0 20/01/23 18:54:04.954 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_11_piece0 locally took 3 ms 20/01/23 18:54:04.954 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_11_piece0 without replication took 3 ms 20/01/23 18:54:04.955 dag-scheduler-event-loop INFO SparkContext: Created broadcast 11 from broadcast at DAGScheduler.scala:1200 20/01/23 18:54:04.955 dag-scheduler-event-loop INFO DAGScheduler: Submitting 4 missing tasks from ResultStage 10 (MapPartitionsRDD[39] at top at CountVectorizer.scala:236) (first 15 tasks are for partitions Vector(0, 1, 2, 3)) 20/01/23 18:54:04.955 dag-scheduler-event-loop INFO TaskSchedulerImpl: Adding task set 10.0 with 4 tasks 20/01/23 18:54:04.956 dag-scheduler-event-loop DEBUG TaskSetManager: Epoch for TaskSet 10.0: 2 20/01/23 18:54:04.956 dag-scheduler-event-loop DEBUG TaskSetManager: Adding pending tasks took 0 ms 20/01/23 18:54:04.956 dag-scheduler-event-loop DEBUG TaskSetManager: Valid locality levels for TaskSet 10.0: PROCESS_LOCAL, NODE_LOCAL, ANY 20/01/23 18:54:04.957 dispatcher-event-loop-0 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_10.0, runningTasks: 0 20/01/23 18:54:04.957 dispatcher-event-loop-0 INFO TaskSetManager: Starting task 0.0 in stage 10.0 (TID 32, 192.168.1.225, executor driver, partition 0, PROCESS_LOCAL, 7143 bytes) 20/01/23 18:54:04.957 dispatcher-event-loop-0 INFO TaskSetManager: Starting task 1.0 in stage 10.0 (TID 33, 192.168.1.225, executor driver, partition 1, PROCESS_LOCAL, 7143 bytes) 20/01/23 18:54:04.958 dispatcher-event-loop-0 INFO TaskSetManager: Starting task 2.0 in stage 10.0 (TID 34, 192.168.1.225, executor driver, partition 2, PROCESS_LOCAL, 7143 bytes) 20/01/23 18:54:04.958 dispatcher-event-loop-0 INFO TaskSetManager: Starting task 3.0 in stage 10.0 (TID 35, 192.168.1.225, executor driver, partition 3, PROCESS_LOCAL, 7143 bytes) 20/01/23 18:54:04.958 Executor task launch worker for task 32 INFO Executor: Running task 0.0 in stage 10.0 (TID 32) 20/01/23 18:54:04.958 Executor task launch worker for task 35 INFO Executor: Running task 3.0 in stage 10.0 (TID 35) 20/01/23 18:54:04.958 Executor task launch worker for task 34 INFO Executor: Running task 2.0 in stage 10.0 (TID 34) 20/01/23 18:54:04.958 Executor task launch worker for task 33 INFO Executor: Running task 1.0 in stage 10.0 (TID 33) 20/01/23 18:54:04.959 Executor task launch worker for task 32 DEBUG ExecutorMetricsPoller: stageTCMP: (10, 0) -> 1 20/01/23 18:54:04.959 Executor task launch worker for task 33 DEBUG ExecutorMetricsPoller: stageTCMP: (10, 0) -> 2 20/01/23 18:54:04.960 Executor task launch worker for task 33 DEBUG BlockManager: Getting local block broadcast_11 20/01/23 18:54:04.960 Executor task launch worker for task 33 DEBUG BlockManager: Level for block broadcast_11 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:04.962 Executor task launch worker for task 34 DEBUG ExecutorMetricsPoller: stageTCMP: (10, 0) -> 3 20/01/23 18:54:04.962 Executor task launch worker for task 35 DEBUG ExecutorMetricsPoller: stageTCMP: (10, 0) -> 4 20/01/23 18:54:04.962 Executor task launch worker for task 32 DEBUG BlockManager: Getting local block rdd_38_0 20/01/23 18:54:04.963 Executor task launch worker for task 32 DEBUG BlockManager: Level for block rdd_38_0 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:04.963 Executor task launch worker for task 32 INFO BlockManager: Found block rdd_38_0 locally 20/01/23 18:54:04.964 Executor task launch worker for task 32 INFO Executor: Finished task 0.0 in stage 10.0 (TID 32). 2600 bytes result sent to driver 20/01/23 18:54:04.964 Executor task launch worker for task 32 DEBUG ExecutorMetricsPoller: stageTCMP: (10, 0) -> 3 20/01/23 18:54:04.964 dispatcher-event-loop-2 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_10.0, runningTasks: 3 20/01/23 18:54:04.964 Executor task launch worker for task 33 DEBUG BlockManager: Getting local block rdd_38_1 20/01/23 18:54:04.964 Executor task launch worker for task 33 DEBUG BlockManager: Level for block rdd_38_1 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:04.964 Executor task launch worker for task 35 DEBUG BlockManager: Getting local block rdd_38_3 20/01/23 18:54:04.964 Executor task launch worker for task 35 DEBUG BlockManager: Level for block rdd_38_3 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:04.964 Executor task launch worker for task 35 INFO BlockManager: Found block rdd_38_3 locally 20/01/23 18:54:04.964 dispatcher-event-loop-2 DEBUG TaskSetManager: No tasks for locality level PROCESS_LOCAL, so moving to locality level NODE_LOCAL 20/01/23 18:54:04.965 dispatcher-event-loop-2 DEBUG TaskSetManager: No tasks for locality level NODE_LOCAL, so moving to locality level ANY 20/01/23 18:54:04.965 Executor task launch worker for task 34 DEBUG BlockManager: Getting local block rdd_38_2 20/01/23 18:54:04.965 Executor task launch worker for task 34 DEBUG BlockManager: Level for block rdd_38_2 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:04.965 Executor task launch worker for task 34 INFO BlockManager: Found block rdd_38_2 locally 20/01/23 18:54:04.965 task-result-getter-1 INFO TaskSetManager: Finished task 0.0 in stage 10.0 (TID 32) in 8 ms on 192.168.1.225 (executor driver) (1/4) 20/01/23 18:54:04.965 Executor task launch worker for task 35 INFO Executor: Finished task 3.0 in stage 10.0 (TID 35). 2600 bytes result sent to driver 20/01/23 18:54:04.966 Executor task launch worker for task 35 DEBUG ExecutorMetricsPoller: stageTCMP: (10, 0) -> 2 20/01/23 18:54:04.966 Executor task launch worker for task 34 INFO Executor: Finished task 2.0 in stage 10.0 (TID 34). 2557 bytes result sent to driver 20/01/23 18:54:04.966 Executor task launch worker for task 34 DEBUG ExecutorMetricsPoller: stageTCMP: (10, 0) -> 1 20/01/23 18:54:04.964 Executor task launch worker for task 33 INFO BlockManager: Found block rdd_38_1 locally 20/01/23 18:54:04.967 Executor task launch worker for task 33 INFO Executor: Finished task 1.0 in stage 10.0 (TID 33). 2426 bytes result sent to driver 20/01/23 18:54:04.967 Executor task launch worker for task 33 DEBUG ExecutorMetricsPoller: removing (10, 0) from stageTCMP 20/01/23 18:54:04.966 dispatcher-event-loop-3 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_10.0, runningTasks: 2 20/01/23 18:54:04.968 dispatcher-event-loop-3 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_10.0, runningTasks: 1 20/01/23 18:54:04.968 dispatcher-event-loop-3 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_10.0, runningTasks: 0 20/01/23 18:54:04.975 task-result-getter-0 INFO TaskSetManager: Finished task 2.0 in stage 10.0 (TID 34) in 17 ms on 192.168.1.225 (executor driver) (2/4) 20/01/23 18:54:04.976 task-result-getter-3 INFO TaskSetManager: Finished task 1.0 in stage 10.0 (TID 33) in 19 ms on 192.168.1.225 (executor driver) (3/4) 20/01/23 18:54:04.976 task-result-getter-2 INFO TaskSetManager: Finished task 3.0 in stage 10.0 (TID 35) in 18 ms on 192.168.1.225 (executor driver) (4/4) 20/01/23 18:54:04.976 task-result-getter-2 INFO TaskSchedulerImpl: Removed TaskSet 10.0, whose tasks have all completed, from pool 20/01/23 18:54:04.977 dag-scheduler-event-loop INFO DAGScheduler: ResultStage 10 (top at CountVectorizer.scala:236) finished in 0.029 s 20/01/23 18:54:04.977 dag-scheduler-event-loop DEBUG DAGScheduler: After removal of stage 10, remaining stages = 1 20/01/23 18:54:04.977 dag-scheduler-event-loop DEBUG DAGScheduler: After removal of stage 9, remaining stages = 0 20/01/23 18:54:04.977 dag-scheduler-event-loop INFO DAGScheduler: Job 6 is finished. Cancelling potential speculative or zombie tasks for this job 20/01/23 18:54:04.977 dag-scheduler-event-loop INFO TaskSchedulerImpl: Killing all running tasks in stage 10: Stage finished 20/01/23 18:54:04.978 Thread-4 INFO DAGScheduler: Job 6 finished: top at CountVectorizer.scala:236, took 0.033458 s 20/01/23 18:54:04.978 Thread-4 INFO MapPartitionsRDD: Removing RDD 38 from persistence list 20/01/23 18:54:04.982 block-manager-slave-async-thread-pool-9 DEBUG BlockManagerSlaveEndpoint: removing RDD 38 20/01/23 18:54:04.982 block-manager-slave-async-thread-pool-9 INFO BlockManager: Removing RDD 38 20/01/23 18:54:04.982 block-manager-slave-async-thread-pool-9 DEBUG BlockManager: Removing block rdd_38_1 20/01/23 18:54:04.983 block-manager-slave-async-thread-pool-9 DEBUG MemoryStore: Block rdd_38_1 of size 16 dropped from memory (free 383953271) 20/01/23 18:54:04.983 block-manager-slave-async-thread-pool-9 DEBUG BlockManager: Removing block rdd_38_0 20/01/23 18:54:04.983 block-manager-slave-async-thread-pool-9 DEBUG MemoryStore: Block rdd_38_0 of size 120 dropped from memory (free 383953391) 20/01/23 18:54:04.983 block-manager-slave-async-thread-pool-9 DEBUG BlockManager: Removing block rdd_38_2 20/01/23 18:54:04.983 block-manager-slave-async-thread-pool-9 DEBUG MemoryStore: Block rdd_38_2 of size 120 dropped from memory (free 383953511) 20/01/23 18:54:04.983 block-manager-slave-async-thread-pool-9 DEBUG BlockManager: Removing block rdd_38_3 20/01/23 18:54:04.983 block-manager-slave-async-thread-pool-9 DEBUG MemoryStore: Block rdd_38_3 of size 120 dropped from memory (free 383953631) 20/01/23 18:54:05.005 block-manager-slave-async-thread-pool-11 DEBUG BlockManagerSlaveEndpoint: Done removing RDD 38, response is 0 20/01/23 18:54:05.012 block-manager-slave-async-thread-pool-11 DEBUG BlockManagerSlaveEndpoint: Sent response: 0 to 192.168.1.225:42003 20/01/23 18:54:05.020 Thread-4 DEBUG CountVectorizerModel: Input schema: {"type":"struct","fields":[{"name":"id","type":"long","nullable":true,"metadata":{}},{"name":"words","type":{"type":"array","elementType":"string","containsNull":true},"nullable":true,"metadata":{}},{"name":"expected","type":{"type":"udt","class":"org.apache.spark.ml.linalg.VectorUDT","pyClass":"pyspark.ml.linalg.VectorUDT","sqlType":{"type":"struct","fields":[{"name":"type","type":"byte","nullable":false,"metadata":{}},{"name":"size","type":"integer","nullable":true,"metadata":{}},{"name":"indices","type":{"type":"array","elementType":"integer","containsNull":false},"nullable":true,"metadata":{}},{"name":"values","type":{"type":"array","elementType":"double","containsNull":false},"nullable":true,"metadata":{}}]}},"nullable":true,"metadata":{}}]} 20/01/23 18:54:05.022 Thread-4 DEBUG CountVectorizerModel: Expected output schema: {"type":"struct","fields":[{"name":"id","type":"long","nullable":true,"metadata":{}},{"name":"words","type":{"type":"array","elementType":"string","containsNull":true},"nullable":true,"metadata":{}},{"name":"expected","type":{"type":"udt","class":"org.apache.spark.ml.linalg.VectorUDT","pyClass":"pyspark.ml.linalg.VectorUDT","sqlType":{"type":"struct","fields":[{"name":"type","type":"byte","nullable":false,"metadata":{}},{"name":"size","type":"integer","nullable":true,"metadata":{}},{"name":"indices","type":{"type":"array","elementType":"integer","containsNull":false},"nullable":true,"metadata":{}},{"name":"values","type":{"type":"array","elementType":"double","containsNull":false},"nullable":true,"metadata":{}}]}},"nullable":true,"metadata":{}},{"name":"features","type":{"type":"udt","class":"org.apache.spark.ml.linalg.VectorUDT","pyClass":"pyspark.ml.linalg.VectorUDT","sqlType":{"type":"struct","fields":[{"name":"type","type":"byte","nullable":false,"metadata":{}},{"name":"size","type":"integer","nullable":true,"metadata":{}},{"name":"indices","type":{"type":"array","elementType":"integer","containsNull":false},"nullable":true,"metadata":{}},{"name":"values","type":{"type":"array","elementType":"double","containsNull":false},"nullable":true,"metadata":{}}]}},"nullable":false,"metadata":{"ml_attr":{"attrs":{},"num_attrs":3}}}]} 20/01/23 18:54:05.024 Thread-4 INFO MemoryStore: Block broadcast_12 stored as values in memory (estimated size 256.0 B, free 366.2 MiB) 20/01/23 18:54:05.024 Thread-4 DEBUG BlockManager: Put block broadcast_12 locally took 1 ms 20/01/23 18:54:05.025 Thread-4 DEBUG BlockManager: Putting block broadcast_12 without replication took 1 ms 20/01/23 18:54:05.047 Thread-4 DEBUG WholeStageCodegenExec: /* 001 */ public Object generate(Object[] references) { /* 002 */ return new GeneratedIteratorForCodegenStage1(references); /* 003 */ } /* 004 */ /* 005 */ // codegenStageId=1 /* 006 */ final class GeneratedIteratorForCodegenStage1 extends org.apache.spark.sql.execution.BufferedRowIterator { /* 007 */ private Object[] references; /* 008 */ private scala.collection.Iterator[] inputs; /* 009 */ private scala.collection.Iterator rdd_input_0; /* 010 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[] rdd_mutableStateArray_0 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[2]; /* 011 */ /* 012 */ public GeneratedIteratorForCodegenStage1(Object[] references) { /* 013 */ this.references = references; /* 014 */ } /* 015 */ /* 016 */ public void init(int index, scala.collection.Iterator[] inputs) { /* 017 */ partitionIndex = index; /* 018 */ this.inputs = inputs; /* 019 */ rdd_input_0 = inputs[0]; /* 020 */ rdd_mutableStateArray_0[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(2, 0); /* 021 */ rdd_mutableStateArray_0[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(3, 0); /* 022 */ /* 023 */ } /* 024 */ /* 025 */ protected void processNext() throws java.io.IOException { /* 026 */ while ( rdd_input_0.hasNext()) { /* 027 */ InternalRow rdd_row_0 = (InternalRow) rdd_input_0.next(); /* 028 */ ((org.apache.spark.sql.execution.metric.SQLMetric) references[0] /* numOutputRows */).add(1); /* 029 */ boolean rdd_isNull_1 = rdd_row_0.isNullAt(1); /* 030 */ double rdd_value_1 = rdd_isNull_1 ? /* 031 */ -1.0 : (rdd_row_0.getDouble(1)); /* 032 */ boolean rdd_isNull_0 = rdd_row_0.isNullAt(0); /* 033 */ double rdd_value_0 = rdd_isNull_0 ? /* 034 */ -1.0 : (rdd_row_0.getDouble(0)); /* 035 */ rdd_mutableStateArray_0[1].reset(); /* 036 */ /* 037 */ rdd_mutableStateArray_0[1].zeroOutNullBytes(); /* 038 */ /* 039 */ if (rdd_isNull_1) { /* 040 */ rdd_mutableStateArray_0[1].setNullAt(0); /* 041 */ } else { /* 042 */ rdd_mutableStateArray_0[1].write(0, rdd_value_1); /* 043 */ } /* 044 */ /* 045 */ if (rdd_isNull_0) { /* 046 */ rdd_mutableStateArray_0[1].setNullAt(1); /* 047 */ } else { /* 048 */ rdd_mutableStateArray_0[1].write(1, rdd_value_0); /* 049 */ } /* 050 */ /* 051 */ rdd_mutableStateArray_0[1].write(2, 1.0D); /* 052 */ append((rdd_mutableStateArray_0[1].getRow())); /* 053 */ if (shouldStop()) return; /* 054 */ } /* 055 */ } /* 056 */ /* 057 */ } 20/01/23 18:54:05.047 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$doExecute$4$adapted 20/01/23 18:54:05.049 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$doExecute$4$adapted) is now cleaned +++ 20/01/23 18:54:05.050 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$rdd$1 20/01/23 18:54:05.056 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$rdd$1) is now cleaned +++ 20/01/23 18:54:05.057 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$evaluate$1 20/01/23 18:54:05.065 Thread-4 INFO MemoryStore: Block broadcast_12_piece0 stored as bytes in memory (estimated size 264.0 B, free 366.2 MiB) 20/01/23 18:54:05.066 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added broadcast_12_piece0 in memory on 192.168.1.225:41233 (size: 264.0 B, free: 366.3 MiB) 20/01/23 18:54:05.066 Thread-4 DEBUG BlockManagerMaster: Updated info of block broadcast_12_piece0 20/01/23 18:54:05.067 Thread-4 DEBUG BlockManager: Told master about block broadcast_12_piece0 20/01/23 18:54:05.067 Thread-4 DEBUG BlockManager: Put block broadcast_12_piece0 locally took 41 ms 20/01/23 18:54:05.067 Thread-4 DEBUG BlockManager: Putting block broadcast_12_piece0 without replication took 41 ms 20/01/23 18:54:05.067 Thread-4 INFO SparkContext: Created broadcast 12 from broadcast at CountVectorizer.scala:306 20/01/23 18:54:05.067 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$evaluate$1) is now cleaned +++ 20/01/23 18:54:05.068 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$summary$1 20/01/23 18:54:05.069 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$summary$1) is now cleaned +++ 20/01/23 18:54:05.070 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$colStats$1 20/01/23 18:54:05.071 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$colStats$1) is now cleaned +++ 20/01/23 18:54:05.071 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$colStats$2 20/01/23 18:54:05.071 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$colStats$2) is now cleaned +++ 20/01/23 18:54:05.072 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$treeAggregate$5 20/01/23 18:54:05.073 Thread-4 DEBUG Analyzer$ResolveReferences: Resolving 'words to words#38 20/01/23 18:54:05.096 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$treeAggregate$5) is now cleaned +++ 20/01/23 18:54:05.100 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$colStats$2 20/01/23 18:54:05.101 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$colStats$2) is now cleaned +++ 20/01/23 18:54:05.101 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$runJob$6 20/01/23 18:54:05.107 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$runJob$6) is now cleaned +++ 20/01/23 18:54:05.108 Thread-4 INFO SparkContext: Starting job: treeAggregate at Statistics.scala:58 20/01/23 18:54:05.110 dag-scheduler-event-loop INFO DAGScheduler: Got job 4 (treeAggregate at Statistics.scala:58) with 4 output partitions 20/01/23 18:54:05.110 dag-scheduler-event-loop INFO DAGScheduler: Final stage: ResultStage 6 (treeAggregate at Statistics.scala:58) 20/01/23 18:54:05.110 dag-scheduler-event-loop INFO DAGScheduler: Parents of final stage: List() 20/01/23 18:54:05.111 dag-scheduler-event-loop INFO DAGScheduler: Missing parents: List() 20/01/23 18:54:05.111 dag-scheduler-event-loop DEBUG DAGScheduler: submitStage(ResultStage 6 (name=treeAggregate at Statistics.scala:58;jobs=4)) 20/01/23 18:54:05.111 dag-scheduler-event-loop DEBUG DAGScheduler: missing: List() 20/01/23 18:54:05.112 dag-scheduler-event-loop INFO DAGScheduler: Submitting ResultStage 6 (MapPartitionsRDD[36] at treeAggregate at Statistics.scala:58), which has no missing parents 20/01/23 18:54:05.112 dag-scheduler-event-loop DEBUG DAGScheduler: submitMissingTasks(ResultStage 6) 20/01/23 18:54:05.130 Thread-4 DEBUG Analyzer$ResolveReferences: Resolving 'features to features#46 20/01/23 18:54:05.130 Thread-4 DEBUG Analyzer$ResolveReferences: Resolving 'expected to expected#39 20/01/23 18:54:05.140 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_7 stored as values in memory (estimated size 18.3 KiB, free 366.2 MiB) 20/01/23 18:54:05.141 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_7 locally took 1 ms 20/01/23 18:54:05.141 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_7 without replication took 1 ms 20/01/23 18:54:05.142 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_7_piece0 stored as bytes in memory (estimated size 9.0 KiB, free 366.2 MiB) 20/01/23 18:54:05.143 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added broadcast_7_piece0 in memory on 192.168.1.225:45461 (size: 9.0 KiB, free: 366.3 MiB) 20/01/23 18:54:05.143 dag-scheduler-event-loop DEBUG BlockManagerMaster: Updated info of block broadcast_7_piece0 20/01/23 18:54:05.143 dag-scheduler-event-loop DEBUG BlockManager: Told master about block broadcast_7_piece0 20/01/23 18:54:05.143 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_7_piece0 locally took 1 ms 20/01/23 18:54:05.143 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_7_piece0 without replication took 1 ms 20/01/23 18:54:05.143 dag-scheduler-event-loop INFO SparkContext: Created broadcast 7 from broadcast at DAGScheduler.scala:1200 20/01/23 18:54:05.144 dag-scheduler-event-loop INFO DAGScheduler: Submitting 4 missing tasks from ResultStage 6 (MapPartitionsRDD[36] at treeAggregate at Statistics.scala:58) (first 15 tasks are for partitions Vector(0, 1, 2, 3)) 20/01/23 18:54:05.144 dag-scheduler-event-loop INFO TaskSchedulerImpl: Adding task set 6.0 with 4 tasks 20/01/23 18:54:05.145 dag-scheduler-event-loop DEBUG TaskSetManager: Epoch for TaskSet 6.0: 2 20/01/23 18:54:05.145 dag-scheduler-event-loop DEBUG TaskSetManager: Adding pending tasks took 0 ms 20/01/23 18:54:05.145 dag-scheduler-event-loop DEBUG TaskSetManager: Valid locality levels for TaskSet 6.0: NO_PREF, ANY 20/01/23 18:54:05.145 dispatcher-event-loop-3 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_6.0, runningTasks: 0 20/01/23 18:54:05.146 dispatcher-event-loop-3 INFO TaskSetManager: Starting task 0.0 in stage 6.0 (TID 18, 192.168.1.225, executor driver, partition 0, PROCESS_LOCAL, 7333 bytes) 20/01/23 18:54:05.146 dispatcher-event-loop-3 INFO TaskSetManager: Starting task 1.0 in stage 6.0 (TID 19, 192.168.1.225, executor driver, partition 1, PROCESS_LOCAL, 7333 bytes) 20/01/23 18:54:05.147 dispatcher-event-loop-3 INFO TaskSetManager: Starting task 2.0 in stage 6.0 (TID 20, 192.168.1.225, executor driver, partition 2, PROCESS_LOCAL, 7333 bytes) 20/01/23 18:54:05.147 dispatcher-event-loop-3 INFO TaskSetManager: Starting task 3.0 in stage 6.0 (TID 21, 192.168.1.225, executor driver, partition 3, PROCESS_LOCAL, 7378 bytes) 20/01/23 18:54:05.148 Executor task launch worker for task 18 INFO Executor: Running task 0.0 in stage 6.0 (TID 18) 20/01/23 18:54:05.148 Executor task launch worker for task 18 DEBUG ExecutorMetricsPoller: stageTCMP: (6, 0) -> 1 20/01/23 18:54:05.149 Executor task launch worker for task 18 DEBUG BlockManager: Getting local block broadcast_7 20/01/23 18:54:05.149 Executor task launch worker for task 18 DEBUG BlockManager: Level for block broadcast_7 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:05.152 Executor task launch worker for task 20 INFO Executor: Running task 2.0 in stage 6.0 (TID 20) 20/01/23 18:54:05.153 Executor task launch worker for task 20 DEBUG ExecutorMetricsPoller: stageTCMP: (6, 0) -> 2 20/01/23 18:54:05.157 Executor task launch worker for task 21 INFO Executor: Running task 3.0 in stage 6.0 (TID 21) 20/01/23 18:54:05.157 Executor task launch worker for task 21 DEBUG ExecutorMetricsPoller: stageTCMP: (6, 0) -> 3 20/01/23 18:54:05.148 Executor task launch worker for task 19 INFO Executor: Running task 1.0 in stage 6.0 (TID 19) 20/01/23 18:54:05.162 Executor task launch worker for task 19 DEBUG ExecutorMetricsPoller: stageTCMP: (6, 0) -> 4 20/01/23 18:54:05.168 Executor task launch worker for task 18 DEBUG GenerateSafeProjection: code for createexternalrow(input[0, double, true], input[1, double, true], input[2, double, false], StructField(prediction,DoubleType,true), StructField(label,DoubleType,true), StructField(1.0,DoubleType,false)): /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificSafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificSafeProjection extends org.apache.spark.sql.catalyst.expressions.codegen.BaseProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private InternalRow mutableRow; /* 009 */ /* 010 */ /* 011 */ public SpecificSafeProjection(Object[] references) { /* 012 */ this.references = references; /* 013 */ mutableRow = (InternalRow) references[references.length - 1]; /* 014 */ /* 015 */ } /* 016 */ /* 017 */ public void initialize(int partitionIndex) { /* 018 */ /* 019 */ } /* 020 */ /* 021 */ public java.lang.Object apply(java.lang.Object _i) { /* 022 */ InternalRow i = (InternalRow) _i; /* 023 */ Object[] values_0 = new Object[3]; /* 024 */ /* 025 */ boolean isNull_1 = i.isNullAt(0); /* 026 */ double value_1 = isNull_1 ? /* 027 */ -1.0 : (i.getDouble(0)); /* 028 */ if (isNull_1) { /* 029 */ values_0[0] = null; /* 030 */ } else { /* 031 */ values_0[0] = value_1; /* 032 */ } /* 033 */ /* 034 */ boolean isNull_2 = i.isNullAt(1); /* 035 */ double value_2 = isNull_2 ? /* 036 */ -1.0 : (i.getDouble(1)); /* 037 */ if (isNull_2) { /* 038 */ values_0[1] = null; /* 039 */ } else { /* 040 */ values_0[1] = value_2; /* 041 */ } /* 042 */ /* 043 */ double value_3 = i.getDouble(2); /* 044 */ if (false) { /* 045 */ values_0[2] = null; /* 046 */ } else { /* 047 */ values_0[2] = value_3; /* 048 */ } /* 049 */ /* 050 */ final org.apache.spark.sql.Row value_0 = new org.apache.spark.sql.catalyst.expressions.GenericRowWithSchema(values_0, ((org.apache.spark.sql.types.StructType) references[0] /* schema */)); /* 051 */ if (false) { /* 052 */ mutableRow.setNullAt(0); /* 053 */ } else { /* 054 */ /* 055 */ mutableRow.update(0, value_0); /* 056 */ } /* 057 */ /* 058 */ return mutableRow; /* 059 */ } /* 060 */ /* 061 */ /* 062 */ } 20/01/23 18:54:05.200 Executor task launch worker for task 20 DEBUG GenerateSafeProjection: code for createexternalrow(input[0, double, true], input[1, double, true], input[2, double, false], StructField(prediction,DoubleType,true), StructField(label,DoubleType,true), StructField(1.0,DoubleType,false)): /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificSafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificSafeProjection extends org.apache.spark.sql.catalyst.expressions.codegen.BaseProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private InternalRow mutableRow; /* 009 */ /* 010 */ /* 011 */ public SpecificSafeProjection(Object[] references) { /* 012 */ this.references = references; /* 013 */ mutableRow = (InternalRow) references[references.length - 1]; /* 014 */ /* 015 */ } /* 016 */ /* 017 */ public void initialize(int partitionIndex) { /* 018 */ /* 019 */ } /* 020 */ /* 021 */ public java.lang.Object apply(java.lang.Object _i) { /* 022 */ InternalRow i = (InternalRow) _i; /* 023 */ Object[] values_0 = new Object[3]; /* 024 */ /* 025 */ boolean isNull_1 = i.isNullAt(0); /* 026 */ double value_1 = isNull_1 ? /* 027 */ -1.0 : (i.getDouble(0)); /* 028 */ if (isNull_1) { /* 029 */ values_0[0] = null; /* 030 */ } else { /* 031 */ values_0[0] = value_1; /* 032 */ } /* 033 */ /* 034 */ boolean isNull_2 = i.isNullAt(1); /* 035 */ double value_2 = isNull_2 ? /* 036 */ -1.0 : (i.getDouble(1)); /* 037 */ if (isNull_2) { /* 038 */ values_0[1] = null; /* 039 */ } else { /* 040 */ values_0[1] = value_2; /* 041 */ } /* 042 */ /* 043 */ double value_3 = i.getDouble(2); /* 044 */ if (false) { /* 045 */ values_0[2] = null; /* 046 */ } else { /* 047 */ values_0[2] = value_3; /* 048 */ } /* 049 */ /* 050 */ final org.apache.spark.sql.Row value_0 = new org.apache.spark.sql.catalyst.expressions.GenericRowWithSchema(values_0, ((org.apache.spark.sql.types.StructType) references[0] /* schema */)); /* 051 */ if (false) { /* 052 */ mutableRow.setNullAt(0); /* 053 */ } else { /* 054 */ /* 055 */ mutableRow.update(0, value_0); /* 056 */ } /* 057 */ /* 058 */ return mutableRow; /* 059 */ } /* 060 */ /* 061 */ /* 062 */ } 20/01/23 18:54:05.225 Thread-4 DEBUG WholeStageCodegenExec: /* 001 */ public Object generate(Object[] references) { /* 002 */ return new GeneratedIteratorForCodegenStage1(references); /* 003 */ } /* 004 */ /* 005 */ // codegenStageId=1 /* 006 */ final class GeneratedIteratorForCodegenStage1 extends org.apache.spark.sql.execution.BufferedRowIterator { /* 007 */ private Object[] references; /* 008 */ private scala.collection.Iterator[] inputs; /* 009 */ private scala.collection.Iterator rdd_input_0; /* 010 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[] rdd_mutableStateArray_0 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[5]; /* 011 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[] rdd_mutableStateArray_1 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[7]; /* 012 */ /* 013 */ public GeneratedIteratorForCodegenStage1(Object[] references) { /* 014 */ this.references = references; /* 015 */ } /* 016 */ /* 017 */ public void init(int index, scala.collection.Iterator[] inputs) { /* 018 */ partitionIndex = index; /* 019 */ this.inputs = inputs; /* 020 */ wholestagecodegen_init_0_0(); /* 021 */ wholestagecodegen_init_0_1(); /* 022 */ /* 023 */ } /* 024 */ /* 025 */ private void project_writeFields_1_1(InternalRow project_tmpInput_3) { /* 026 */ if ((project_tmpInput_3.isNullAt(3))) { /* 027 */ rdd_mutableStateArray_0[4].setNullAt(3); /* 028 */ } else { /* 029 */ // Remember the current cursor so that we can calculate how many bytes are /* 030 */ // written later. /* 031 */ final int project_previousCursor_5 = rdd_mutableStateArray_0[4].cursor(); /* 032 */ /* 033 */ final ArrayData project_tmpInput_5 = (project_tmpInput_3.getArray(3)); /* 034 */ if (project_tmpInput_5 instanceof UnsafeArrayData) { /* 035 */ rdd_mutableStateArray_0[4].write((UnsafeArrayData) project_tmpInput_5); /* 036 */ } else { /* 037 */ final int project_numElements_3 = project_tmpInput_5.numElements(); /* 038 */ rdd_mutableStateArray_1[6].initialize(project_numElements_3); /* 039 */ /* 040 */ for (int project_index_3 = 0; project_index_3 < project_numElements_3; project_index_3++) { /* 041 */ rdd_mutableStateArray_1[6].write(project_index_3, project_tmpInput_5.getDouble(project_index_3)); /* 042 */ } /* 043 */ } /* 044 */ /* 045 */ rdd_mutableStateArray_0[4].setOffsetAndSizeFromPreviousCursor(3, project_previousCursor_5); /* 046 */ } /* 047 */ /* 048 */ } /* 049 */ /* 050 */ private void project_writeFields_1_0(InternalRow project_tmpInput_3) { /* 051 */ rdd_mutableStateArray_0[4].write(0, (project_tmpInput_3.getByte(0))); /* 052 */ /* 053 */ if ((project_tmpInput_3.isNullAt(1))) { /* 054 */ rdd_mutableStateArray_0[4].setNullAt(1); /* 055 */ } else { /* 056 */ rdd_mutableStateArray_0[4].write(1, (project_tmpInput_3.getInt(1))); /* 057 */ } /* 058 */ /* 059 */ if ((project_tmpInput_3.isNullAt(2))) { /* 060 */ rdd_mutableStateArray_0[4].setNullAt(2); /* 061 */ } else { /* 062 */ // Remember the current cursor so that we can calculate how many bytes are /* 063 */ // written later. /* 064 */ final int project_previousCursor_4 = rdd_mutableStateArray_0[4].cursor(); /* 065 */ /* 066 */ final ArrayData project_tmpInput_4 = (project_tmpInput_3.getArray(2)); /* 067 */ if (project_tmpInput_4 instanceof UnsafeArrayData) { /* 068 */ rdd_mutableStateArray_0[4].write((UnsafeArrayData) project_tmpInput_4); /* 069 */ } else { /* 070 */ final int project_numElements_2 = project_tmpInput_4.numElements(); /* 071 */ rdd_mutableStateArray_1[5].initialize(project_numElements_2); /* 072 */ /* 073 */ for (int project_index_2 = 0; project_index_2 < project_numElements_2; project_index_2++) { /* 074 */ rdd_mutableStateArray_1[5].write(project_index_2, project_tmpInput_4.getInt(project_index_2)); /* 075 */ } /* 076 */ } /* 077 */ /* 078 */ rdd_mutableStateArray_0[4].setOffsetAndSizeFromPreviousCursor(2, project_previousCursor_4); /* 079 */ } /* 080 */ /* 081 */ } /* 082 */ /* 083 */ private void project_writeFields_0_1(InternalRow project_tmpInput_0) { /* 084 */ if ((project_tmpInput_0.isNullAt(3))) { /* 085 */ rdd_mutableStateArray_0[3].setNullAt(3); /* 086 */ } else { /* 087 */ // Remember the current cursor so that we can calculate how many bytes are /* 088 */ // written later. /* 089 */ final int project_previousCursor_2 = rdd_mutableStateArray_0[3].cursor(); /* 090 */ /* 091 */ final ArrayData project_tmpInput_2 = (project_tmpInput_0.getArray(3)); /* 092 */ if (project_tmpInput_2 instanceof UnsafeArrayData) { /* 093 */ rdd_mutableStateArray_0[3].write((UnsafeArrayData) project_tmpInput_2); /* 094 */ } else { /* 095 */ final int project_numElements_1 = project_tmpInput_2.numElements(); /* 096 */ rdd_mutableStateArray_1[4].initialize(project_numElements_1); /* 097 */ /* 098 */ for (int project_index_1 = 0; project_index_1 < project_numElements_1; project_index_1++) { /* 099 */ rdd_mutableStateArray_1[4].write(project_index_1, project_tmpInput_2.getDouble(project_index_1)); /* 100 */ } /* 101 */ } /* 102 */ /* 103 */ rdd_mutableStateArray_0[3].setOffsetAndSizeFromPreviousCursor(3, project_previousCursor_2); /* 104 */ } /* 105 */ /* 106 */ } /* 107 */ /* 108 */ private void wholestagecodegen_init_0_1() { /* 109 */ rdd_mutableStateArray_1[4] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[3], 8); /* 110 */ rdd_mutableStateArray_0[4] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(rdd_mutableStateArray_0[2], 4); /* 111 */ rdd_mutableStateArray_1[5] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[4], 4); /* 112 */ rdd_mutableStateArray_1[6] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[4], 8); /* 113 */ /* 114 */ } /* 115 */ /* 116 */ private void project_writeFields_0_0(InternalRow project_tmpInput_0) { /* 117 */ rdd_mutableStateArray_0[3].write(0, (project_tmpInput_0.getByte(0))); /* 118 */ /* 119 */ if ((project_tmpInput_0.isNullAt(1))) { /* 120 */ rdd_mutableStateArray_0[3].setNullAt(1); /* 121 */ } else { /* 122 */ rdd_mutableStateArray_0[3].write(1, (project_tmpInput_0.getInt(1))); /* 123 */ } /* 124 */ /* 125 */ if ((project_tmpInput_0.isNullAt(2))) { /* 126 */ rdd_mutableStateArray_0[3].setNullAt(2); /* 127 */ } else { /* 128 */ // Remember the current cursor so that we can calculate how many bytes are /* 129 */ // written later. /* 130 */ final int project_previousCursor_1 = rdd_mutableStateArray_0[3].cursor(); /* 131 */ /* 132 */ final ArrayData project_tmpInput_1 = (project_tmpInput_0.getArray(2)); /* 133 */ if (project_tmpInput_1 instanceof UnsafeArrayData) { /* 134 */ rdd_mutableStateArray_0[3].write((UnsafeArrayData) project_tmpInput_1); /* 135 */ } else { /* 136 */ final int project_numElements_0 = project_tmpInput_1.numElements(); /* 137 */ rdd_mutableStateArray_1[3].initialize(project_numElements_0); /* 138 */ /* 139 */ for (int project_index_0 = 0; project_index_0 < project_numElements_0; project_index_0++) { /* 140 */ rdd_mutableStateArray_1[3].write(project_index_0, project_tmpInput_1.getInt(project_index_0)); /* 141 */ } /* 142 */ } /* 143 */ /* 144 */ rdd_mutableStateArray_0[3].setOffsetAndSizeFromPreviousCursor(2, project_previousCursor_1); /* 145 */ } /* 146 */ /* 147 */ } /* 148 */ /* 149 */ protected void processNext() throws java.io.IOException { /* 150 */ while ( rdd_input_0.hasNext()) { /* 151 */ InternalRow rdd_row_0 = (InternalRow) rdd_input_0.next(); /* 152 */ ((org.apache.spark.sql.execution.metric.SQLMetric) references[0] /* numOutputRows */).add(1); /* 153 */ boolean rdd_isNull_1 = rdd_row_0.isNullAt(1); /* 154 */ ArrayData rdd_value_1 = rdd_isNull_1 ? /* 155 */ null : (rdd_row_0.getArray(1)); /* 156 */ Object project_arg_0 = rdd_isNull_1 ? null : ((scala.Function1[]) references[1] /* converters */)[0].apply(rdd_value_1); /* 157 */ /* 158 */ InternalRow project_result_0 = null; /* 159 */ try { /* 160 */ project_result_0 = (InternalRow)((scala.Function1[]) references[1] /* converters */)[1].apply(((scala.Function1) references[3] /* udf */).apply(project_arg_0)); /* 161 */ } catch (Exception e) { /* 162 */ throw new org.apache.spark.SparkException(((java.lang.String) references[2] /* errMsg */), e); /* 163 */ } /* 164 */ /* 165 */ boolean project_isNull_0 = project_result_0 == null; /* 166 */ InternalRow project_value_0 = null; /* 167 */ if (!project_isNull_0) { /* 168 */ project_value_0 = project_result_0; /* 169 */ } /* 170 */ boolean rdd_isNull_2 = rdd_row_0.isNullAt(2); /* 171 */ InternalRow rdd_value_2 = rdd_isNull_2 ? /* 172 */ null : (rdd_row_0.getStruct(2, 4)); /* 173 */ rdd_mutableStateArray_0[2].reset(); /* 174 */ /* 175 */ rdd_mutableStateArray_0[2].zeroOutNullBytes(); /* 176 */ /* 177 */ if (project_isNull_0) { /* 178 */ rdd_mutableStateArray_0[2].setNullAt(0); /* 179 */ } else { /* 180 */ final InternalRow project_tmpInput_0 = project_value_0; /* 181 */ if (project_tmpInput_0 instanceof UnsafeRow) { /* 182 */ rdd_mutableStateArray_0[2].write(0, (UnsafeRow) project_tmpInput_0); /* 183 */ } else { /* 184 */ // Remember the current cursor so that we can calculate how many bytes are /* 185 */ // written later. /* 186 */ final int project_previousCursor_0 = rdd_mutableStateArray_0[2].cursor(); /* 187 */ /* 188 */ rdd_mutableStateArray_0[3].resetRowWriter(); /* 189 */ project_writeFields_0_0(project_tmpInput_0); /* 190 */ project_writeFields_0_1(project_tmpInput_0); /* 191 */ /* 192 */ rdd_mutableStateArray_0[2].setOffsetAndSizeFromPreviousCursor(0, project_previousCursor_0); /* 193 */ } /* 194 */ } /* 195 */ /* 196 */ if (rdd_isNull_2) { /* 197 */ rdd_mutableStateArray_0[2].setNullAt(1); /* 198 */ } else { /* 199 */ final InternalRow project_tmpInput_3 = rdd_value_2; /* 200 */ if (project_tmpInput_3 instanceof UnsafeRow) { /* 201 */ rdd_mutableStateArray_0[2].write(1, (UnsafeRow) project_tmpInput_3); /* 202 */ } else { /* 203 */ // Remember the current cursor so that we can calculate how many bytes are /* 204 */ // written later. /* 205 */ final int project_previousCursor_3 = rdd_mutableStateArray_0[2].cursor(); /* 206 */ /* 207 */ rdd_mutableStateArray_0[4].resetRowWriter(); /* 208 */ project_writeFields_1_0(project_tmpInput_3); /* 209 */ project_writeFields_1_1(project_tmpInput_3); /* 210 */ /* 211 */ rdd_mutableStateArray_0[2].setOffsetAndSizeFromPreviousCursor(1, project_previousCursor_3); /* 212 */ } /* 213 */ } /* 214 */ append((rdd_mutableStateArray_0[2].getRow())); /* 215 */ if (shouldStop()) return; /* 216 */ } /* 217 */ } /* 218 */ /* 219 */ private void wholestagecodegen_init_0_0() { /* 220 */ rdd_input_0 = inputs[0]; /* 221 */ rdd_mutableStateArray_0[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(3, 64); /* 222 */ rdd_mutableStateArray_1[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[0], 8); /* 223 */ rdd_mutableStateArray_0[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(rdd_mutableStateArray_0[0], 4); /* 224 */ rdd_mutableStateArray_1[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[1], 4); /* 225 */ rdd_mutableStateArray_1[2] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[1], 8); /* 226 */ rdd_mutableStateArray_0[2] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(2, 64); /* 227 */ rdd_mutableStateArray_0[3] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(rdd_mutableStateArray_0[2], 4); /* 228 */ rdd_mutableStateArray_1[3] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[3], 4); /* 229 */ /* 230 */ } /* 231 */ /* 232 */ } 20/01/23 18:54:05.226 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$doExecute$4$adapted 20/01/23 18:54:05.226 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$doExecute$4$adapted) is now cleaned +++ 20/01/23 18:54:05.228 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$collect$2 20/01/23 18:54:05.230 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$collect$2) is now cleaned +++ 20/01/23 18:54:05.215 Executor task launch worker for task 18 INFO PythonRunner: Times: total = 49, boot = -396, init = 445, finish = 0 20/01/23 18:54:05.252 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$runJob$5 20/01/23 18:54:05.254 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$runJob$5) is now cleaned +++ 20/01/23 18:54:05.256 Thread-4 INFO SparkContext: Starting job: at /usr/lib/python3.6/runpy.py:85 20/01/23 18:54:05.257 dag-scheduler-event-loop INFO DAGScheduler: Got job 7 ( at /usr/lib/python3.6/runpy.py:85) with 4 output partitions 20/01/23 18:54:05.257 dag-scheduler-event-loop INFO DAGScheduler: Final stage: ResultStage 11 ( at /usr/lib/python3.6/runpy.py:85) 20/01/23 18:54:05.257 dag-scheduler-event-loop INFO DAGScheduler: Parents of final stage: List() 20/01/23 18:54:05.257 dag-scheduler-event-loop INFO DAGScheduler: Missing parents: List() 20/01/23 18:54:05.258 dag-scheduler-event-loop DEBUG DAGScheduler: submitStage(ResultStage 11 (name= at /usr/lib/python3.6/runpy.py:85;jobs=7)) 20/01/23 18:54:05.258 dag-scheduler-event-loop DEBUG DAGScheduler: missing: List() 20/01/23 18:54:05.258 dag-scheduler-event-loop INFO DAGScheduler: Submitting ResultStage 11 (MapPartitionsRDD[41] at at /usr/lib/python3.6/runpy.py:85), which has no missing parents 20/01/23 18:54:05.258 dag-scheduler-event-loop DEBUG DAGScheduler: submitMissingTasks(ResultStage 11) 20/01/23 18:54:05.261 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_13 stored as values in memory (estimated size 25.6 KiB, free 366.1 MiB) 20/01/23 18:54:05.261 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_13 locally took 0 ms 20/01/23 18:54:05.262 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_13 without replication took 0 ms 20/01/23 18:54:05.233 Executor task launch worker for task 20 INFO PythonRunner: Times: total = 69, boot = -364, init = 433, finish = 0 20/01/23 18:54:05.222 Executor task launch worker for task 21 DEBUG GenerateSafeProjection: code for createexternalrow(input[0, double, true], input[1, double, true], input[2, double, false], StructField(prediction,DoubleType,true), StructField(label,DoubleType,true), StructField(1.0,DoubleType,false)): /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificSafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificSafeProjection extends org.apache.spark.sql.catalyst.expressions.codegen.BaseProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private InternalRow mutableRow; /* 009 */ /* 010 */ /* 011 */ public SpecificSafeProjection(Object[] references) { /* 012 */ this.references = references; /* 013 */ mutableRow = (InternalRow) references[references.length - 1]; /* 014 */ /* 015 */ } /* 016 */ /* 017 */ public void initialize(int partitionIndex) { /* 018 */ /* 019 */ } /* 020 */ /* 021 */ public java.lang.Object apply(java.lang.Object _i) { /* 022 */ InternalRow i = (InternalRow) _i; /* 023 */ Object[] values_0 = new Object[3]; /* 024 */ /* 025 */ boolean isNull_1 = i.isNullAt(0); /* 026 */ double value_1 = isNull_1 ? /* 027 */ -1.0 : (i.getDouble(0)); /* 028 */ if (isNull_1) { /* 029 */ values_0[0] = null; /* 030 */ } else { /* 031 */ values_0[0] = value_1; /* 032 */ } /* 033 */ /* 034 */ boolean isNull_2 = i.isNullAt(1); /* 035 */ double value_2 = isNull_2 ? /* 036 */ -1.0 : (i.getDouble(1)); /* 037 */ if (isNull_2) { /* 038 */ values_0[1] = null; /* 039 */ } else { /* 040 */ values_0[1] = value_2; /* 041 */ } /* 042 */ /* 043 */ double value_3 = i.getDouble(2); /* 044 */ if (false) { /* 045 */ values_0[2] = null; /* 046 */ } else { /* 047 */ values_0[2] = value_3; /* 048 */ } /* 049 */ /* 050 */ final org.apache.spark.sql.Row value_0 = new org.apache.spark.sql.catalyst.expressions.GenericRowWithSchema(values_0, ((org.apache.spark.sql.types.StructType) references[0] /* schema */)); /* 051 */ if (false) { /* 052 */ mutableRow.setNullAt(0); /* 053 */ } else { /* 054 */ /* 055 */ mutableRow.update(0, value_0); /* 056 */ } /* 057 */ /* 058 */ return mutableRow; /* 059 */ } /* 060 */ /* 061 */ /* 062 */ } 20/01/23 18:54:05.262 Executor task launch worker for task 18 INFO Executor: Finished task 0.0 in stage 6.0 (TID 18). 2920 bytes result sent to driver 20/01/23 18:54:05.263 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_13_piece0 stored as bytes in memory (estimated size 10.9 KiB, free 366.1 MiB) 20/01/23 18:54:05.261 Executor task launch worker for task 19 DEBUG GenerateSafeProjection: code for createexternalrow(input[0, double, true], input[1, double, true], input[2, double, false], StructField(prediction,DoubleType,true), StructField(label,DoubleType,true), StructField(1.0,DoubleType,false)): /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificSafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificSafeProjection extends org.apache.spark.sql.catalyst.expressions.codegen.BaseProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private InternalRow mutableRow; /* 009 */ /* 010 */ /* 011 */ public SpecificSafeProjection(Object[] references) { /* 012 */ this.references = references; /* 013 */ mutableRow = (InternalRow) references[references.length - 1]; /* 014 */ /* 015 */ } /* 016 */ /* 017 */ public void initialize(int partitionIndex) { /* 018 */ /* 019 */ } /* 020 */ /* 021 */ public java.lang.Object apply(java.lang.Object _i) { /* 022 */ InternalRow i = (InternalRow) _i; /* 023 */ Object[] values_0 = new Object[3]; /* 024 */ /* 025 */ boolean isNull_1 = i.isNullAt(0); /* 026 */ double value_1 = isNull_1 ? /* 027 */ -1.0 : (i.getDouble(0)); /* 028 */ if (isNull_1) { /* 029 */ values_0[0] = null; /* 030 */ } else { /* 031 */ values_0[0] = value_1; /* 032 */ } /* 033 */ /* 034 */ boolean isNull_2 = i.isNullAt(1); /* 035 */ double value_2 = isNull_2 ? /* 036 */ -1.0 : (i.getDouble(1)); /* 037 */ if (isNull_2) { /* 038 */ values_0[1] = null; /* 039 */ } else { /* 040 */ values_0[1] = value_2; /* 041 */ } /* 042 */ /* 043 */ double value_3 = i.getDouble(2); /* 044 */ if (false) { /* 045 */ values_0[2] = null; /* 046 */ } else { /* 047 */ values_0[2] = value_3; /* 048 */ } /* 049 */ /* 050 */ final org.apache.spark.sql.Row value_0 = new org.apache.spark.sql.catalyst.expressions.GenericRowWithSchema(values_0, ((org.apache.spark.sql.types.StructType) references[0] /* schema */)); /* 051 */ if (false) { /* 052 */ mutableRow.setNullAt(0); /* 053 */ } else { /* 054 */ /* 055 */ mutableRow.update(0, value_0); /* 056 */ } /* 057 */ /* 058 */ return mutableRow; /* 059 */ } /* 060 */ /* 061 */ /* 062 */ } 20/01/23 18:54:05.263 Executor task launch worker for task 18 DEBUG ExecutorMetricsPoller: stageTCMP: (6, 0) -> 3 20/01/23 18:54:05.263 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added broadcast_13_piece0 in memory on 192.168.1.225:41233 (size: 10.9 KiB, free: 366.2 MiB) 20/01/23 18:54:05.264 dag-scheduler-event-loop DEBUG BlockManagerMaster: Updated info of block broadcast_13_piece0 20/01/23 18:54:05.264 dag-scheduler-event-loop DEBUG BlockManager: Told master about block broadcast_13_piece0 20/01/23 18:54:05.264 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_13_piece0 locally took 1 ms 20/01/23 18:54:05.264 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_13_piece0 without replication took 1 ms 20/01/23 18:54:05.264 dag-scheduler-event-loop INFO SparkContext: Created broadcast 13 from broadcast at DAGScheduler.scala:1200 20/01/23 18:54:05.265 dag-scheduler-event-loop INFO DAGScheduler: Submitting 4 missing tasks from ResultStage 11 (MapPartitionsRDD[41] at at /usr/lib/python3.6/runpy.py:85) (first 15 tasks are for partitions Vector(0, 1, 2, 3)) 20/01/23 18:54:05.265 dag-scheduler-event-loop INFO TaskSchedulerImpl: Adding task set 11.0 with 4 tasks 20/01/23 18:54:05.264 Executor task launch worker for task 19 INFO PythonRunner: Times: total = 54, boot = -347, init = 401, finish = 0 20/01/23 18:54:05.265 Executor task launch worker for task 20 INFO Executor: Finished task 2.0 in stage 6.0 (TID 20). 2920 bytes result sent to driver 20/01/23 18:54:05.264 dispatcher-event-loop-1 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_6.0, runningTasks: 3 20/01/23 18:54:05.266 Executor task launch worker for task 19 INFO Executor: Finished task 1.0 in stage 6.0 (TID 19). 2920 bytes result sent to driver 20/01/23 18:54:05.266 Executor task launch worker for task 19 DEBUG ExecutorMetricsPoller: stageTCMP: (6, 0) -> 2 20/01/23 18:54:05.266 dispatcher-event-loop-1 DEBUG TaskSetManager: No tasks for locality level NO_PREF, so moving to locality level ANY 20/01/23 18:54:05.267 dispatcher-event-loop-1 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_6.0, runningTasks: 2 20/01/23 18:54:05.264 Executor task launch worker for task 21 INFO PythonRunner: Times: total = 95, boot = -322, init = 417, finish = 0 20/01/23 18:54:05.267 task-result-getter-0 INFO TaskSetManager: Finished task 0.0 in stage 6.0 (TID 18) in 121 ms on 192.168.1.225 (executor driver) (1/4) 20/01/23 18:54:05.268 task-result-getter-3 INFO TaskSetManager: Finished task 1.0 in stage 6.0 (TID 19) in 121 ms on 192.168.1.225 (executor driver) (2/4) 20/01/23 18:54:05.268 Executor task launch worker for task 21 INFO Executor: Finished task 3.0 in stage 6.0 (TID 21). 3101 bytes result sent to driver 20/01/23 18:54:05.268 Executor task launch worker for task 21 DEBUG ExecutorMetricsPoller: stageTCMP: (6, 0) -> 1 20/01/23 18:54:05.269 dispatcher-event-loop-2 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_6.0, runningTasks: 1 20/01/23 18:54:05.270 task-result-getter-2 INFO TaskSetManager: Finished task 3.0 in stage 6.0 (TID 21) in 122 ms on 192.168.1.225 (executor driver) (3/4) 20/01/23 18:54:05.270 dag-scheduler-event-loop DEBUG TaskSetManager: Epoch for TaskSet 11.0: 2 20/01/23 18:54:05.270 dag-scheduler-event-loop DEBUG TaskSetManager: Adding pending tasks took 0 ms 20/01/23 18:54:05.270 dag-scheduler-event-loop DEBUG TaskSetManager: Valid locality levels for TaskSet 11.0: NO_PREF, ANY 20/01/23 18:54:05.273 dispatcher-event-loop-2 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_11.0, runningTasks: 0 20/01/23 18:54:05.273 Thread-4 INFO Instrumentation: [3e99e817] training finished 20/01/23 18:54:05.273 dispatcher-event-loop-2 INFO TaskSetManager: Starting task 0.0 in stage 11.0 (TID 36, 192.168.1.225, executor driver, partition 0, PROCESS_LOCAL, 7430 bytes) 20/01/23 18:54:05.274 dispatcher-event-loop-2 INFO TaskSetManager: Starting task 1.0 in stage 11.0 (TID 37, 192.168.1.225, executor driver, partition 1, PROCESS_LOCAL, 7415 bytes) 20/01/23 18:54:05.274 dispatcher-event-loop-2 INFO TaskSetManager: Starting task 2.0 in stage 11.0 (TID 38, 192.168.1.225, executor driver, partition 2, PROCESS_LOCAL, 7398 bytes) 20/01/23 18:54:05.274 dispatcher-event-loop-2 INFO TaskSetManager: Starting task 3.0 in stage 11.0 (TID 39, 192.168.1.225, executor driver, partition 3, PROCESS_LOCAL, 7380 bytes) 20/01/23 18:54:05.274 Executor task launch worker for task 37 INFO Executor: Running task 1.0 in stage 11.0 (TID 37) 20/01/23 18:54:05.274 Executor task launch worker for task 38 INFO Executor: Running task 2.0 in stage 11.0 (TID 38) 20/01/23 18:54:05.275 Executor task launch worker for task 37 DEBUG ExecutorMetricsPoller: stageTCMP: (11, 0) -> 1 20/01/23 18:54:05.274 Executor task launch worker for task 36 INFO Executor: Running task 0.0 in stage 11.0 (TID 36) 20/01/23 18:54:05.275 Executor task launch worker for task 36 DEBUG ExecutorMetricsPoller: stageTCMP: (11, 0) -> 3 20/01/23 18:54:05.276 Executor task launch worker for task 37 DEBUG BlockManager: Getting local block broadcast_13 20/01/23 18:54:05.276 Executor task launch worker for task 37 DEBUG BlockManager: Level for block broadcast_13 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:05.279 Executor task launch worker for task 39 INFO Executor: Running task 3.0 in stage 11.0 (TID 39) 20/01/23 18:54:05.275 Executor task launch worker for task 38 DEBUG ExecutorMetricsPoller: stageTCMP: (11, 0) -> 2 20/01/23 18:54:05.279 Executor task launch worker for task 39 DEBUG ExecutorMetricsPoller: stageTCMP: (11, 0) -> 4 20/01/23 18:54:05.281 Executor task launch worker for task 20 DEBUG ExecutorMetricsPoller: removing (6, 0) from stageTCMP 20/01/23 18:54:05.283 dispatcher-event-loop-3 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_6.0, runningTasks: 0 20/01/23 18:54:05.283 task-result-getter-1 INFO TaskSetManager: Finished task 2.0 in stage 6.0 (TID 20) in 137 ms on 192.168.1.225 (executor driver) (4/4) 20/01/23 18:54:05.284 task-result-getter-1 INFO TaskSchedulerImpl: Removed TaskSet 6.0, whose tasks have all completed, from pool 20/01/23 18:54:05.285 dag-scheduler-event-loop INFO DAGScheduler: ResultStage 6 (treeAggregate at Statistics.scala:58) finished in 0.169 s 20/01/23 18:54:05.285 dag-scheduler-event-loop DEBUG DAGScheduler: After removal of stage 6, remaining stages = 0 20/01/23 18:54:05.285 dag-scheduler-event-loop INFO DAGScheduler: Job 4 is finished. Cancelling potential speculative or zombie tasks for this job 20/01/23 18:54:05.285 dag-scheduler-event-loop INFO TaskSchedulerImpl: Killing all running tasks in stage 6: Stage finished 20/01/23 18:54:05.285 Thread-4 INFO DAGScheduler: Job 4 finished: treeAggregate at Statistics.scala:58, took 0.177393 s 20/01/23 18:54:05.305 Thread-4 DEBUG Analyzer$ResolveReferences: Resolving 'user to user#0L 20/01/23 18:54:05.305 Thread-4 DEBUG Analyzer$ResolveReferences: Resolving 'item to item#1L 20/01/23 18:54:05.305 Thread-4 DEBUG Analyzer$ResolveReferences: Resolving 'rating to rating#2 20/01/23 18:54:05.307 Thread-4 DEBUG Analyzer$ResolveReferences: Resolving 'prediction to prediction#30 20/01/23 18:54:05.308 Thread-4 DEBUG Analyzer$ResolveReferences: Resolving 'label to label#29 20/01/23 18:54:05.341 Executor task launch worker for task 37 DEBUG BlockManager: Getting local block broadcast_12 20/01/23 18:54:05.342 Executor task launch worker for task 37 DEBUG BlockManager: Level for block broadcast_12 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:05.342 Executor task launch worker for task 37 INFO PythonRunner: Times: total = 54, boot = -471, init = 525, finish = 0 20/01/23 18:54:05.364 Thread-4 DEBUG WholeStageCodegenExec: /* 001 */ public Object generate(Object[] references) { /* 002 */ return new GeneratedIteratorForCodegenStage1(references); /* 003 */ } /* 004 */ /* 005 */ // codegenStageId=1 /* 006 */ final class GeneratedIteratorForCodegenStage1 extends org.apache.spark.sql.execution.BufferedRowIterator { /* 007 */ private Object[] references; /* 008 */ private scala.collection.Iterator[] inputs; /* 009 */ private scala.collection.Iterator rdd_input_0; /* 010 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[] rdd_mutableStateArray_0 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[2]; /* 011 */ /* 012 */ public GeneratedIteratorForCodegenStage1(Object[] references) { /* 013 */ this.references = references; /* 014 */ } /* 015 */ /* 016 */ public void init(int index, scala.collection.Iterator[] inputs) { /* 017 */ partitionIndex = index; /* 018 */ this.inputs = inputs; /* 019 */ rdd_input_0 = inputs[0]; /* 020 */ rdd_mutableStateArray_0[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(2, 0); /* 021 */ rdd_mutableStateArray_0[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(3, 0); /* 022 */ /* 023 */ } /* 024 */ /* 025 */ protected void processNext() throws java.io.IOException { /* 026 */ while ( rdd_input_0.hasNext()) { /* 027 */ InternalRow rdd_row_0 = (InternalRow) rdd_input_0.next(); /* 028 */ ((org.apache.spark.sql.execution.metric.SQLMetric) references[0] /* numOutputRows */).add(1); /* 029 */ boolean rdd_isNull_1 = rdd_row_0.isNullAt(1); /* 030 */ double rdd_value_1 = rdd_isNull_1 ? /* 031 */ -1.0 : (rdd_row_0.getDouble(1)); /* 032 */ boolean rdd_isNull_0 = rdd_row_0.isNullAt(0); /* 033 */ double rdd_value_0 = rdd_isNull_0 ? /* 034 */ -1.0 : (rdd_row_0.getDouble(0)); /* 035 */ rdd_mutableStateArray_0[1].reset(); /* 036 */ /* 037 */ rdd_mutableStateArray_0[1].zeroOutNullBytes(); /* 038 */ /* 039 */ if (rdd_isNull_1) { /* 040 */ rdd_mutableStateArray_0[1].setNullAt(0); /* 041 */ } else { /* 042 */ rdd_mutableStateArray_0[1].write(0, rdd_value_1); /* 043 */ } /* 044 */ /* 045 */ if (rdd_isNull_0) { /* 046 */ rdd_mutableStateArray_0[1].setNullAt(1); /* 047 */ } else { /* 048 */ rdd_mutableStateArray_0[1].write(1, rdd_value_0); /* 049 */ } /* 050 */ /* 051 */ rdd_mutableStateArray_0[1].write(2, 1.0D); /* 052 */ append((rdd_mutableStateArray_0[1].getRow())); /* 053 */ if (shouldStop()) return; /* 054 */ } /* 055 */ } /* 056 */ /* 057 */ } 20/01/23 18:54:05.364 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$doExecute$4$adapted 20/01/23 18:54:05.369 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$doExecute$4$adapted) is now cleaned +++ 20/01/23 18:54:05.371 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$rdd$1 20/01/23 18:54:05.378 Executor task launch worker for task 39 INFO PythonRunner: Times: total = 66, boot = -420, init = 486, finish = 0 20/01/23 18:54:05.380 Executor task launch worker for task 37 INFO Executor: Finished task 1.0 in stage 11.0 (TID 37). 2124 bytes result sent to driver 20/01/23 18:54:05.381 Executor task launch worker for task 37 DEBUG ExecutorMetricsPoller: stageTCMP: (11, 0) -> 3 20/01/23 18:54:05.381 Executor task launch worker for task 36 INFO PythonRunner: Times: total = 86, boot = -433, init = 519, finish = 0 20/01/23 18:54:05.382 Executor task launch worker for task 39 INFO Executor: Finished task 3.0 in stage 11.0 (TID 39). 2100 bytes result sent to driver 20/01/23 18:54:05.382 Executor task launch worker for task 39 DEBUG ExecutorMetricsPoller: stageTCMP: (11, 0) -> 2 20/01/23 18:54:05.383 Executor task launch worker for task 36 INFO Executor: Finished task 0.0 in stage 11.0 (TID 36). 2127 bytes result sent to driver 20/01/23 18:54:05.383 Executor task launch worker for task 36 DEBUG ExecutorMetricsPoller: stageTCMP: (11, 0) -> 1 20/01/23 18:54:05.376 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$rdd$1) is now cleaned +++ 20/01/23 18:54:05.384 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$evaluate$1 20/01/23 18:54:05.385 dispatcher-event-loop-0 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_11.0, runningTasks: 3 20/01/23 18:54:05.385 dispatcher-event-loop-0 DEBUG TaskSetManager: No tasks for locality level NO_PREF, so moving to locality level ANY 20/01/23 18:54:05.385 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$evaluate$1) is now cleaned +++ 20/01/23 18:54:05.386 dispatcher-event-loop-0 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_11.0, runningTasks: 2 20/01/23 18:54:05.386 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$summary$1 20/01/23 18:54:05.386 dispatcher-event-loop-0 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_11.0, runningTasks: 1 20/01/23 18:54:05.387 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$summary$1) is now cleaned +++ 20/01/23 18:54:05.387 task-result-getter-3 INFO TaskSetManager: Finished task 0.0 in stage 11.0 (TID 36) in 114 ms on 192.168.1.225 (executor driver) (1/4) 20/01/23 18:54:05.388 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$colStats$1 20/01/23 18:54:05.389 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$colStats$1) is now cleaned +++ 20/01/23 18:54:05.389 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$colStats$2 20/01/23 18:54:05.389 task-result-getter-1 INFO TaskSetManager: Finished task 1.0 in stage 11.0 (TID 37) in 116 ms on 192.168.1.225 (executor driver) (2/4) 20/01/23 18:54:05.389 task-result-getter-0 INFO TaskSetManager: Finished task 3.0 in stage 11.0 (TID 39) in 115 ms on 192.168.1.225 (executor driver) (3/4) 20/01/23 18:54:05.390 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$colStats$2) is now cleaned +++ 20/01/23 18:54:05.390 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$treeAggregate$5 20/01/23 18:54:05.397 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$treeAggregate$5) is now cleaned +++ 20/01/23 18:54:05.398 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$colStats$2 20/01/23 18:54:05.399 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$colStats$2) is now cleaned +++ 20/01/23 18:54:05.399 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$runJob$6 20/01/23 18:54:05.404 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$runJob$6) is now cleaned +++ 20/01/23 18:54:05.404 Executor task launch worker for task 38 INFO PythonRunner: Times: total = 111, boot = -432, init = 543, finish = 0 20/01/23 18:54:05.404 Thread-4 INFO SparkContext: Starting job: treeAggregate at Statistics.scala:58 20/01/23 18:54:05.405 Executor task launch worker for task 38 INFO Executor: Finished task 2.0 in stage 11.0 (TID 38). 2111 bytes result sent to driver 20/01/23 18:54:05.405 dag-scheduler-event-loop INFO DAGScheduler: Got job 5 (treeAggregate at Statistics.scala:58) with 4 output partitions 20/01/23 18:54:05.407 Executor task launch worker for task 38 DEBUG ExecutorMetricsPoller: removing (11, 0) from stageTCMP 20/01/23 18:54:05.407 dag-scheduler-event-loop INFO DAGScheduler: Final stage: ResultStage 7 (treeAggregate at Statistics.scala:58) 20/01/23 18:54:05.407 dag-scheduler-event-loop INFO DAGScheduler: Parents of final stage: List() 20/01/23 18:54:05.408 dag-scheduler-event-loop INFO DAGScheduler: Missing parents: List() 20/01/23 18:54:05.408 dag-scheduler-event-loop DEBUG DAGScheduler: submitStage(ResultStage 7 (name=treeAggregate at Statistics.scala:58;jobs=5)) 20/01/23 18:54:05.408 dag-scheduler-event-loop DEBUG DAGScheduler: missing: List() 20/01/23 18:54:05.408 dag-scheduler-event-loop INFO DAGScheduler: Submitting ResultStage 7 (MapPartitionsRDD[43] at treeAggregate at Statistics.scala:58), which has no missing parents 20/01/23 18:54:05.408 dag-scheduler-event-loop DEBUG DAGScheduler: submitMissingTasks(ResultStage 7) 20/01/23 18:54:05.410 dispatcher-event-loop-2 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_11.0, runningTasks: 0 20/01/23 18:54:05.411 task-result-getter-2 INFO TaskSetManager: Finished task 2.0 in stage 11.0 (TID 38) in 137 ms on 192.168.1.225 (executor driver) (4/4) 20/01/23 18:54:05.411 task-result-getter-2 INFO TaskSchedulerImpl: Removed TaskSet 11.0, whose tasks have all completed, from pool 20/01/23 18:54:05.412 dag-scheduler-event-loop INFO DAGScheduler: ResultStage 11 ( at /usr/lib/python3.6/runpy.py:85) finished in 0.152 s 20/01/23 18:54:05.412 dag-scheduler-event-loop DEBUG DAGScheduler: After removal of stage 11, remaining stages = 0 20/01/23 18:54:05.412 dag-scheduler-event-loop INFO DAGScheduler: Job 7 is finished. Cancelling potential speculative or zombie tasks for this job 20/01/23 18:54:05.412 dag-scheduler-event-loop INFO TaskSchedulerImpl: Killing all running tasks in stage 11: Stage finished 20/01/23 18:54:05.412 Thread-4 INFO DAGScheduler: Job 7 finished: at /usr/lib/python3.6/runpy.py:85, took 0.156255 s 20/01/23 18:54:05.417 Thread-4 DEBUG WholeStageCodegenExec: /* 001 */ public Object generate(Object[] references) { /* 002 */ return new GeneratedIteratorForCodegenStage1(references); /* 003 */ } /* 004 */ /* 005 */ // codegenStageId=1 /* 006 */ final class GeneratedIteratorForCodegenStage1 extends org.apache.spark.sql.execution.BufferedRowIterator { /* 007 */ private Object[] references; /* 008 */ private scala.collection.Iterator[] inputs; /* 009 */ private scala.collection.Iterator rdd_input_0; /* 010 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[] rdd_mutableStateArray_0 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[2]; /* 011 */ /* 012 */ public GeneratedIteratorForCodegenStage1(Object[] references) { /* 013 */ this.references = references; /* 014 */ } /* 015 */ /* 016 */ public void init(int index, scala.collection.Iterator[] inputs) { /* 017 */ partitionIndex = index; /* 018 */ this.inputs = inputs; /* 019 */ rdd_input_0 = inputs[0]; /* 020 */ rdd_mutableStateArray_0[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(3, 0); /* 021 */ rdd_mutableStateArray_0[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(3, 0); /* 022 */ /* 023 */ } /* 024 */ /* 025 */ protected void processNext() throws java.io.IOException { /* 026 */ while ( rdd_input_0.hasNext()) { /* 027 */ InternalRow rdd_row_0 = (InternalRow) rdd_input_0.next(); /* 028 */ ((org.apache.spark.sql.execution.metric.SQLMetric) references[0] /* numOutputRows */).add(1); /* 029 */ boolean rdd_isNull_0 = rdd_row_0.isNullAt(0); /* 030 */ long rdd_value_0 = rdd_isNull_0 ? /* 031 */ -1L : (rdd_row_0.getLong(0)); /* 032 */ /* 033 */ Long project_conv_0 = rdd_value_0; /* 034 */ Object project_arg_0 = rdd_isNull_0 ? null : project_conv_0; /* 035 */ /* 036 */ Integer project_result_0 = null; /* 037 */ try { /* 038 */ project_result_0 = (Integer)((scala.Function1) references[3] /* udf */).apply(project_arg_0); /* 039 */ } catch (Exception e) { /* 040 */ throw new org.apache.spark.SparkException(((java.lang.String) references[2] /* errMsg */), e); /* 041 */ } /* 042 */ /* 043 */ boolean project_isNull_0 = project_result_0 == null; /* 044 */ int project_value_0 = -1; /* 045 */ if (!project_isNull_0) { /* 046 */ project_value_0 = project_result_0; /* 047 */ } /* 048 */ boolean rdd_isNull_1 = rdd_row_0.isNullAt(1); /* 049 */ long rdd_value_1 = rdd_isNull_1 ? /* 050 */ -1L : (rdd_row_0.getLong(1)); /* 051 */ /* 052 */ Long project_conv_1 = rdd_value_1; /* 053 */ Object project_arg_1 = rdd_isNull_1 ? null : project_conv_1; /* 054 */ /* 055 */ Integer project_result_1 = null; /* 056 */ try { /* 057 */ project_result_1 = (Integer)((scala.Function1) references[6] /* udf */).apply(project_arg_1); /* 058 */ } catch (Exception e) { /* 059 */ throw new org.apache.spark.SparkException(((java.lang.String) references[5] /* errMsg */), e); /* 060 */ } /* 061 */ /* 062 */ boolean project_isNull_2 = project_result_1 == null; /* 063 */ int project_value_2 = -1; /* 064 */ if (!project_isNull_2) { /* 065 */ project_value_2 = project_result_1; /* 066 */ } /* 067 */ boolean rdd_isNull_2 = rdd_row_0.isNullAt(2); /* 068 */ double rdd_value_2 = rdd_isNull_2 ? /* 069 */ -1.0 : (rdd_row_0.getDouble(2)); /* 070 */ boolean project_isNull_4 = rdd_isNull_2; /* 071 */ float project_value_4 = -1.0f; /* 072 */ if (!rdd_isNull_2) { /* 073 */ project_value_4 = (float) rdd_value_2; /* 074 */ } /* 075 */ rdd_mutableStateArray_0[1].reset(); /* 076 */ /* 077 */ rdd_mutableStateArray_0[1].zeroOutNullBytes(); /* 078 */ /* 079 */ rdd_mutableStateArray_0[1].write(0, project_value_0); /* 080 */ /* 081 */ rdd_mutableStateArray_0[1].write(1, project_value_2); /* 082 */ /* 083 */ if (project_isNull_4) { /* 084 */ rdd_mutableStateArray_0[1].setNullAt(2); /* 085 */ } else { /* 086 */ rdd_mutableStateArray_0[1].write(2, project_value_4); /* 087 */ } /* 088 */ append((rdd_mutableStateArray_0[1].getRow())); /* 089 */ if (shouldStop()) return; /* 090 */ } /* 091 */ } /* 092 */ /* 093 */ } 20/01/23 18:54:05.418 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$doExecute$4$adapted 20/01/23 18:54:05.423 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_8 stored as values in memory (estimated size 18.3 KiB, free 366.1 MiB) 20/01/23 18:54:05.423 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_8 locally took 1 ms 20/01/23 18:54:05.423 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_8 without replication took 1 ms 20/01/23 18:54:05.424 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_8_piece0 stored as bytes in memory (estimated size 9.0 KiB, free 366.1 MiB) 20/01/23 18:54:05.426 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$doExecute$4$adapted) is now cleaned +++ 20/01/23 18:54:05.425 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added broadcast_8_piece0 in memory on 192.168.1.225:45461 (size: 9.0 KiB, free: 366.2 MiB) 20/01/23 18:54:05.427 dag-scheduler-event-loop DEBUG BlockManagerMaster: Updated info of block broadcast_8_piece0 20/01/23 18:54:05.427 dag-scheduler-event-loop DEBUG BlockManager: Told master about block broadcast_8_piece0 20/01/23 18:54:05.427 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_8_piece0 locally took 2 ms 20/01/23 18:54:05.427 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_8_piece0 without replication took 2 ms 20/01/23 18:54:05.427 dag-scheduler-event-loop INFO SparkContext: Created broadcast 8 from broadcast at DAGScheduler.scala:1200 20/01/23 18:54:05.428 dag-scheduler-event-loop INFO DAGScheduler: Submitting 4 missing tasks from ResultStage 7 (MapPartitionsRDD[43] at treeAggregate at Statistics.scala:58) (first 15 tasks are for partitions Vector(0, 1, 2, 3)) 20/01/23 18:54:05.428 dag-scheduler-event-loop INFO TaskSchedulerImpl: Adding task set 7.0 with 4 tasks 20/01/23 18:54:05.428 dag-scheduler-event-loop DEBUG TaskSetManager: Epoch for TaskSet 7.0: 2 20/01/23 18:54:05.428 dag-scheduler-event-loop DEBUG TaskSetManager: Adding pending tasks took 0 ms 20/01/23 18:54:05.428 dag-scheduler-event-loop DEBUG TaskSetManager: Valid locality levels for TaskSet 7.0: NO_PREF, ANY 20/01/23 18:54:05.429 dispatcher-event-loop-0 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_7.0, runningTasks: 0 20/01/23 18:54:05.429 dispatcher-event-loop-0 INFO TaskSetManager: Starting task 0.0 in stage 7.0 (TID 22, 192.168.1.225, executor driver, partition 0, PROCESS_LOCAL, 7333 bytes) 20/01/23 18:54:05.430 dispatcher-event-loop-0 INFO TaskSetManager: Starting task 1.0 in stage 7.0 (TID 23, 192.168.1.225, executor driver, partition 1, PROCESS_LOCAL, 7333 bytes) 20/01/23 18:54:05.430 dispatcher-event-loop-0 INFO TaskSetManager: Starting task 2.0 in stage 7.0 (TID 24, 192.168.1.225, executor driver, partition 2, PROCESS_LOCAL, 7333 bytes) 20/01/23 18:54:05.430 dispatcher-event-loop-0 INFO TaskSetManager: Starting task 3.0 in stage 7.0 (TID 25, 192.168.1.225, executor driver, partition 3, PROCESS_LOCAL, 7378 bytes) 20/01/23 18:54:05.431 Executor task launch worker for task 23 INFO Executor: Running task 1.0 in stage 7.0 (TID 23) 20/01/23 18:54:05.431 Executor task launch worker for task 25 INFO Executor: Running task 3.0 in stage 7.0 (TID 25) 20/01/23 18:54:05.431 Executor task launch worker for task 22 INFO Executor: Running task 0.0 in stage 7.0 (TID 22) 20/01/23 18:54:05.432 Executor task launch worker for task 22 DEBUG ExecutorMetricsPoller: stageTCMP: (7, 0) -> 2 20/01/23 18:54:05.432 Executor task launch worker for task 22 DEBUG BlockManager: Getting local block broadcast_8 20/01/23 18:54:05.432 Executor task launch worker for task 22 DEBUG BlockManager: Level for block broadcast_8 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:05.431 Executor task launch worker for task 24 INFO Executor: Running task 2.0 in stage 7.0 (TID 24) 20/01/23 18:54:05.436 Executor task launch worker for task 24 DEBUG ExecutorMetricsPoller: stageTCMP: (7, 0) -> 3 20/01/23 18:54:05.431 Executor task launch worker for task 23 DEBUG ExecutorMetricsPoller: stageTCMP: (7, 0) -> 1 20/01/23 18:54:05.440 Executor task launch worker for task 25 DEBUG ExecutorMetricsPoller: stageTCMP: (7, 0) -> 4 20/01/23 18:54:05.442 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$rdd$1 20/01/23 18:54:05.444 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$rdd$1) is now cleaned +++ 20/01/23 18:54:05.445 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$fit$2 20/01/23 18:54:05.446 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$fit$2) is now cleaned +++ 20/01/23 18:54:05.446 Thread-4 INFO Instrumentation: [f94ba937] Stage class: ALS 20/01/23 18:54:05.446 Thread-4 INFO Instrumentation: [f94ba937] Stage uid: ALS_56177bd83433 20/01/23 18:54:05.446 Thread-4 INFO Instrumentation: [f94ba937] training: numPartitions=4 storageLevel=StorageLevel(1 replicas) 20/01/23 18:54:05.447 Thread-4 INFO Instrumentation: [f94ba937] {"rank":1,"maxIter":1,"intermediateStorageLevel":"MEMORY_ONLY_2","finalStorageLevel":"DISK_ONLY"} 20/01/23 18:54:05.450 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$take$2 20/01/23 18:54:05.452 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$take$2) is now cleaned +++ 20/01/23 18:54:05.455 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$runJob$5 20/01/23 18:54:05.456 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$runJob$5) is now cleaned +++ 20/01/23 18:54:05.458 Thread-4 INFO SparkContext: Starting job: isEmpty at ALS.scala:931 20/01/23 18:54:05.459 dag-scheduler-event-loop INFO DAGScheduler: Got job 5 (isEmpty at ALS.scala:931) with 1 output partitions 20/01/23 18:54:05.459 dag-scheduler-event-loop INFO DAGScheduler: Final stage: ResultStage 18 (isEmpty at ALS.scala:931) 20/01/23 18:54:05.459 dag-scheduler-event-loop INFO DAGScheduler: Parents of final stage: List() 20/01/23 18:54:05.460 dag-scheduler-event-loop INFO DAGScheduler: Missing parents: List() 20/01/23 18:54:05.460 dag-scheduler-event-loop DEBUG DAGScheduler: submitStage(ResultStage 18 (name=isEmpty at ALS.scala:931;jobs=5)) 20/01/23 18:54:05.463 Executor task launch worker for task 22 DEBUG GenerateSafeProjection: code for createexternalrow(input[0, double, true], input[1, double, true], input[2, double, false], StructField(prediction,DoubleType,true), StructField(label,DoubleType,true), StructField(1.0,DoubleType,false)): /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificSafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificSafeProjection extends org.apache.spark.sql.catalyst.expressions.codegen.BaseProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private InternalRow mutableRow; /* 009 */ /* 010 */ /* 011 */ public SpecificSafeProjection(Object[] references) { /* 012 */ this.references = references; /* 013 */ mutableRow = (InternalRow) references[references.length - 1]; /* 014 */ /* 015 */ } /* 016 */ /* 017 */ public void initialize(int partitionIndex) { /* 018 */ /* 019 */ } /* 020 */ /* 021 */ public java.lang.Object apply(java.lang.Object _i) { /* 022 */ InternalRow i = (InternalRow) _i; /* 023 */ Object[] values_0 = new Object[3]; /* 024 */ /* 025 */ boolean isNull_1 = i.isNullAt(0); /* 026 */ double value_1 = isNull_1 ? /* 027 */ -1.0 : (i.getDouble(0)); /* 028 */ if (isNull_1) { /* 029 */ values_0[0] = null; /* 030 */ } else { /* 031 */ values_0[0] = value_1; /* 032 */ } /* 033 */ /* 034 */ boolean isNull_2 = i.isNullAt(1); /* 035 */ double value_2 = isNull_2 ? /* 036 */ -1.0 : (i.getDouble(1)); /* 037 */ if (isNull_2) { /* 038 */ values_0[1] = null; /* 039 */ } else { /* 040 */ values_0[1] = value_2; /* 041 */ } /* 042 */ /* 043 */ double value_3 = i.getDouble(2); /* 044 */ if (false) { /* 045 */ values_0[2] = null; /* 046 */ } else { /* 047 */ values_0[2] = value_3; /* 048 */ } /* 049 */ /* 050 */ final org.apache.spark.sql.Row value_0 = new org.apache.spark.sql.catalyst.expressions.GenericRowWithSchema(values_0, ((org.apache.spark.sql.types.StructType) references[0] /* schema */)); /* 051 */ if (false) { /* 052 */ mutableRow.setNullAt(0); /* 053 */ } else { /* 054 */ /* 055 */ mutableRow.update(0, value_0); /* 056 */ } /* 057 */ /* 058 */ return mutableRow; /* 059 */ } /* 060 */ /* 061 */ /* 062 */ } 20/01/23 18:54:05.463 dag-scheduler-event-loop DEBUG DAGScheduler: missing: List() 20/01/23 18:54:05.464 dag-scheduler-event-loop INFO DAGScheduler: Submitting ResultStage 18 (MapPartitionsRDD[60] at map at ALS.scala:678), which has no missing parents 20/01/23 18:54:05.464 dag-scheduler-event-loop DEBUG DAGScheduler: submitMissingTasks(ResultStage 18) 20/01/23 18:54:05.466 Thread-4 DEBUG CountVectorizer: Input schema: {"type":"struct","fields":[{"name":"id","type":"long","nullable":true,"metadata":{}},{"name":"words","type":{"type":"array","elementType":"string","containsNull":true},"nullable":true,"metadata":{}},{"name":"expected","type":{"type":"udt","class":"org.apache.spark.ml.linalg.VectorUDT","pyClass":"pyspark.ml.linalg.VectorUDT","sqlType":{"type":"struct","fields":[{"name":"type","type":"byte","nullable":false,"metadata":{}},{"name":"size","type":"integer","nullable":true,"metadata":{}},{"name":"indices","type":{"type":"array","elementType":"integer","containsNull":false},"nullable":true,"metadata":{}},{"name":"values","type":{"type":"array","elementType":"double","containsNull":false},"nullable":true,"metadata":{}}]}},"nullable":true,"metadata":{}}]} 20/01/23 18:54:05.467 Thread-4 DEBUG CountVectorizer: Expected output schema: {"type":"struct","fields":[{"name":"id","type":"long","nullable":true,"metadata":{}},{"name":"words","type":{"type":"array","elementType":"string","containsNull":true},"nullable":true,"metadata":{}},{"name":"expected","type":{"type":"udt","class":"org.apache.spark.ml.linalg.VectorUDT","pyClass":"pyspark.ml.linalg.VectorUDT","sqlType":{"type":"struct","fields":[{"name":"type","type":"byte","nullable":false,"metadata":{}},{"name":"size","type":"integer","nullable":true,"metadata":{}},{"name":"indices","type":{"type":"array","elementType":"integer","containsNull":false},"nullable":true,"metadata":{}},{"name":"values","type":{"type":"array","elementType":"double","containsNull":false},"nullable":true,"metadata":{}}]}},"nullable":true,"metadata":{}},{"name":"features","type":{"type":"udt","class":"org.apache.spark.ml.linalg.VectorUDT","pyClass":"pyspark.ml.linalg.VectorUDT","sqlType":{"type":"struct","fields":[{"name":"type","type":"byte","nullable":false,"metadata":{}},{"name":"size","type":"integer","nullable":true,"metadata":{}},{"name":"indices","type":{"type":"array","elementType":"integer","containsNull":false},"nullable":true,"metadata":{}},{"name":"values","type":{"type":"array","elementType":"double","containsNull":false},"nullable":true,"metadata":{}}]}},"nullable":false,"metadata":{}}]} 20/01/23 18:54:05.467 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_10 stored as values in memory (estimated size 25.1 KiB, free 365.9 MiB) 20/01/23 18:54:05.467 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_10 locally took 0 ms 20/01/23 18:54:05.467 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_10 without replication took 1 ms 20/01/23 18:54:05.468 Thread-4 DEBUG Analyzer$ResolveReferences: Resolving 'words to words#38 20/01/23 18:54:05.468 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_10_piece0 stored as bytes in memory (estimated size 12.3 KiB, free 365.9 MiB) 20/01/23 18:54:05.469 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added broadcast_10_piece0 in memory on 192.168.1.225:35863 (size: 12.3 KiB, free: 366.2 MiB) 20/01/23 18:54:05.469 dag-scheduler-event-loop DEBUG BlockManagerMaster: Updated info of block broadcast_10_piece0 20/01/23 18:54:05.469 dag-scheduler-event-loop DEBUG BlockManager: Told master about block broadcast_10_piece0 20/01/23 18:54:05.469 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_10_piece0 locally took 1 ms 20/01/23 18:54:05.469 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_10_piece0 without replication took 1 ms 20/01/23 18:54:05.470 dag-scheduler-event-loop INFO SparkContext: Created broadcast 10 from broadcast at DAGScheduler.scala:1200 20/01/23 18:54:05.470 dag-scheduler-event-loop INFO DAGScheduler: Submitting 1 missing tasks from ResultStage 18 (MapPartitionsRDD[60] at map at ALS.scala:678) (first 15 tasks are for partitions Vector(0)) 20/01/23 18:54:05.470 dag-scheduler-event-loop INFO TaskSchedulerImpl: Adding task set 18.0 with 1 tasks 20/01/23 18:54:05.470 dag-scheduler-event-loop DEBUG TaskSetManager: Epoch for TaskSet 18.0: 5 20/01/23 18:54:05.476 dag-scheduler-event-loop DEBUG TaskSetManager: Adding pending tasks took 5 ms 20/01/23 18:54:05.476 dag-scheduler-event-loop DEBUG TaskSetManager: Valid locality levels for TaskSet 18.0: NO_PREF, ANY 20/01/23 18:54:05.477 dispatcher-event-loop-3 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_18.0, runningTasks: 0 20/01/23 18:54:05.477 dispatcher-event-loop-3 INFO TaskSetManager: Starting task 0.0 in stage 18.0 (TID 73, 192.168.1.225, executor driver, partition 0, PROCESS_LOCAL, 7373 bytes) 20/01/23 18:54:05.477 dispatcher-event-loop-3 DEBUG TaskSetManager: No tasks for locality level NO_PREF, so moving to locality level ANY 20/01/23 18:54:05.478 Executor task launch worker for task 73 INFO Executor: Running task 0.0 in stage 18.0 (TID 73) 20/01/23 18:54:05.478 Executor task launch worker for task 73 DEBUG ExecutorMetricsPoller: stageTCMP: (18, 0) -> 1 20/01/23 18:54:05.479 Executor task launch worker for task 73 DEBUG BlockManager: Getting local block broadcast_10 20/01/23 18:54:05.479 Executor task launch worker for task 73 DEBUG BlockManager: Level for block broadcast_10 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:05.501 Executor task launch worker for task 22 INFO PythonRunner: Times: total = 58, boot = -212, init = 270, finish = 0 20/01/23 18:54:05.528 Executor task launch worker for task 73 DEBUG GenerateSafeProjection: code for createexternalrow(input[0, int, false], input[1, int, false], input[2, float, true], StructField(UDF(user),IntegerType,false), StructField(UDF(item),IntegerType,false), StructField(rating,FloatType,true)): /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificSafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificSafeProjection extends org.apache.spark.sql.catalyst.expressions.codegen.BaseProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private InternalRow mutableRow; /* 009 */ /* 010 */ /* 011 */ public SpecificSafeProjection(Object[] references) { /* 012 */ this.references = references; /* 013 */ mutableRow = (InternalRow) references[references.length - 1]; /* 014 */ /* 015 */ } /* 016 */ /* 017 */ public void initialize(int partitionIndex) { /* 018 */ /* 019 */ } /* 020 */ /* 021 */ public java.lang.Object apply(java.lang.Object _i) { /* 022 */ InternalRow i = (InternalRow) _i; /* 023 */ Object[] values_0 = new Object[3]; /* 024 */ /* 025 */ int value_1 = i.getInt(0); /* 026 */ if (false) { /* 027 */ values_0[0] = null; /* 028 */ } else { /* 029 */ values_0[0] = value_1; /* 030 */ } /* 031 */ /* 032 */ int value_2 = i.getInt(1); /* 033 */ if (false) { /* 034 */ values_0[1] = null; /* 035 */ } else { /* 036 */ values_0[1] = value_2; /* 037 */ } /* 038 */ /* 039 */ boolean isNull_3 = i.isNullAt(2); /* 040 */ float value_3 = isNull_3 ? /* 041 */ -1.0f : (i.getFloat(2)); /* 042 */ if (isNull_3) { /* 043 */ values_0[2] = null; /* 044 */ } else { /* 045 */ values_0[2] = value_3; /* 046 */ } /* 047 */ /* 048 */ final org.apache.spark.sql.Row value_0 = new org.apache.spark.sql.catalyst.expressions.GenericRowWithSchema(values_0, ((org.apache.spark.sql.types.StructType) references[0] /* schema */)); /* 049 */ if (false) { /* 050 */ mutableRow.setNullAt(0); /* 051 */ } else { /* 052 */ /* 053 */ mutableRow.update(0, value_0); /* 054 */ } /* 055 */ /* 056 */ return mutableRow; /* 057 */ } /* 058 */ /* 059 */ /* 060 */ } 20/01/23 18:54:05.541 Thread-4 DEBUG WholeStageCodegenExec: /* 001 */ public Object generate(Object[] references) { /* 002 */ return new GeneratedIteratorForCodegenStage1(references); /* 003 */ } /* 004 */ /* 005 */ // codegenStageId=1 /* 006 */ final class GeneratedIteratorForCodegenStage1 extends org.apache.spark.sql.execution.BufferedRowIterator { /* 007 */ private Object[] references; /* 008 */ private scala.collection.Iterator[] inputs; /* 009 */ private scala.collection.Iterator rdd_input_0; /* 010 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[] rdd_mutableStateArray_0 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[3]; /* 011 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[] rdd_mutableStateArray_1 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[4]; /* 012 */ /* 013 */ public GeneratedIteratorForCodegenStage1(Object[] references) { /* 014 */ this.references = references; /* 015 */ } /* 016 */ /* 017 */ public void init(int index, scala.collection.Iterator[] inputs) { /* 018 */ partitionIndex = index; /* 019 */ this.inputs = inputs; /* 020 */ rdd_input_0 = inputs[0]; /* 021 */ rdd_mutableStateArray_0[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(3, 64); /* 022 */ rdd_mutableStateArray_1[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[0], 8); /* 023 */ rdd_mutableStateArray_0[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(rdd_mutableStateArray_0[0], 4); /* 024 */ rdd_mutableStateArray_1[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[1], 4); /* 025 */ rdd_mutableStateArray_1[2] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[1], 8); /* 026 */ rdd_mutableStateArray_0[2] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(1, 32); /* 027 */ rdd_mutableStateArray_1[3] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[2], 8); /* 028 */ /* 029 */ } /* 030 */ /* 031 */ protected void processNext() throws java.io.IOException { /* 032 */ while ( rdd_input_0.hasNext()) { /* 033 */ InternalRow rdd_row_0 = (InternalRow) rdd_input_0.next(); /* 034 */ ((org.apache.spark.sql.execution.metric.SQLMetric) references[0] /* numOutputRows */).add(1); /* 035 */ boolean rdd_isNull_1 = rdd_row_0.isNullAt(1); /* 036 */ ArrayData rdd_value_1 = rdd_isNull_1 ? /* 037 */ null : (rdd_row_0.getArray(1)); /* 038 */ rdd_mutableStateArray_0[2].reset(); /* 039 */ /* 040 */ rdd_mutableStateArray_0[2].zeroOutNullBytes(); /* 041 */ /* 042 */ if (rdd_isNull_1) { /* 043 */ rdd_mutableStateArray_0[2].setNullAt(0); /* 044 */ } else { /* 045 */ // Remember the current cursor so that we can calculate how many bytes are /* 046 */ // written later. /* 047 */ final int project_previousCursor_0 = rdd_mutableStateArray_0[2].cursor(); /* 048 */ /* 049 */ final ArrayData project_tmpInput_0 = rdd_value_1; /* 050 */ if (project_tmpInput_0 instanceof UnsafeArrayData) { /* 051 */ rdd_mutableStateArray_0[2].write((UnsafeArrayData) project_tmpInput_0); /* 052 */ } else { /* 053 */ final int project_numElements_0 = project_tmpInput_0.numElements(); /* 054 */ rdd_mutableStateArray_1[3].initialize(project_numElements_0); /* 055 */ /* 056 */ for (int project_index_0 = 0; project_index_0 < project_numElements_0; project_index_0++) { /* 057 */ if (project_tmpInput_0.isNullAt(project_index_0)) { /* 058 */ rdd_mutableStateArray_1[3].setNull8Bytes(project_index_0); /* 059 */ } else { /* 060 */ rdd_mutableStateArray_1[3].write(project_index_0, project_tmpInput_0.getUTF8String(project_index_0)); /* 061 */ } /* 062 */ /* 063 */ } /* 064 */ } /* 065 */ /* 066 */ rdd_mutableStateArray_0[2].setOffsetAndSizeFromPreviousCursor(0, project_previousCursor_0); /* 067 */ } /* 068 */ append((rdd_mutableStateArray_0[2].getRow())); /* 069 */ if (shouldStop()) return; /* 070 */ } /* 071 */ } /* 072 */ /* 073 */ } 20/01/23 18:54:05.542 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$doExecute$4$adapted 20/01/23 18:54:05.544 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$doExecute$4$adapted) is now cleaned +++ 20/01/23 18:54:05.547 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$rdd$1 20/01/23 18:54:05.504 Executor task launch worker for task 23 DEBUG GenerateSafeProjection: code for createexternalrow(input[0, double, true], input[1, double, true], input[2, double, false], StructField(prediction,DoubleType,true), StructField(label,DoubleType,true), StructField(1.0,DoubleType,false)): /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificSafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificSafeProjection extends org.apache.spark.sql.catalyst.expressions.codegen.BaseProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private InternalRow mutableRow; /* 009 */ /* 010 */ /* 011 */ public SpecificSafeProjection(Object[] references) { /* 012 */ this.references = references; /* 013 */ mutableRow = (InternalRow) references[references.length - 1]; /* 014 */ /* 015 */ } /* 016 */ /* 017 */ public void initialize(int partitionIndex) { /* 018 */ /* 019 */ } /* 020 */ /* 021 */ public java.lang.Object apply(java.lang.Object _i) { /* 022 */ InternalRow i = (InternalRow) _i; /* 023 */ Object[] values_0 = new Object[3]; /* 024 */ /* 025 */ boolean isNull_1 = i.isNullAt(0); /* 026 */ double value_1 = isNull_1 ? /* 027 */ -1.0 : (i.getDouble(0)); /* 028 */ if (isNull_1) { /* 029 */ values_0[0] = null; /* 030 */ } else { /* 031 */ values_0[0] = value_1; /* 032 */ } /* 033 */ /* 034 */ boolean isNull_2 = i.isNullAt(1); /* 035 */ double value_2 = isNull_2 ? /* 036 */ -1.0 : (i.getDouble(1)); /* 037 */ if (isNull_2) { /* 038 */ values_0[1] = null; /* 039 */ } else { /* 040 */ values_0[1] = value_2; /* 041 */ } /* 042 */ /* 043 */ double value_3 = i.getDouble(2); /* 044 */ if (false) { /* 045 */ values_0[2] = null; /* 046 */ } else { /* 047 */ values_0[2] = value_3; /* 048 */ } /* 049 */ /* 050 */ final org.apache.spark.sql.Row value_0 = new org.apache.spark.sql.catalyst.expressions.GenericRowWithSchema(values_0, ((org.apache.spark.sql.types.StructType) references[0] /* schema */)); /* 051 */ if (false) { /* 052 */ mutableRow.setNullAt(0); /* 053 */ } else { /* 054 */ /* 055 */ mutableRow.update(0, value_0); /* 056 */ } /* 057 */ /* 058 */ return mutableRow; /* 059 */ } /* 060 */ /* 061 */ /* 062 */ } 20/01/23 18:54:05.544 Executor task launch worker for task 25 DEBUG GenerateSafeProjection: code for createexternalrow(input[0, double, true], input[1, double, true], input[2, double, false], StructField(prediction,DoubleType,true), StructField(label,DoubleType,true), StructField(1.0,DoubleType,false)): /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificSafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificSafeProjection extends org.apache.spark.sql.catalyst.expressions.codegen.BaseProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private InternalRow mutableRow; /* 009 */ /* 010 */ /* 011 */ public SpecificSafeProjection(Object[] references) { /* 012 */ this.references = references; /* 013 */ mutableRow = (InternalRow) references[references.length - 1]; /* 014 */ /* 015 */ } /* 016 */ /* 017 */ public void initialize(int partitionIndex) { /* 018 */ /* 019 */ } /* 020 */ /* 021 */ public java.lang.Object apply(java.lang.Object _i) { /* 022 */ InternalRow i = (InternalRow) _i; /* 023 */ Object[] values_0 = new Object[3]; /* 024 */ /* 025 */ boolean isNull_1 = i.isNullAt(0); /* 026 */ double value_1 = isNull_1 ? /* 027 */ -1.0 : (i.getDouble(0)); /* 028 */ if (isNull_1) { /* 029 */ values_0[0] = null; /* 030 */ } else { /* 031 */ values_0[0] = value_1; /* 032 */ } /* 033 */ /* 034 */ boolean isNull_2 = i.isNullAt(1); /* 035 */ double value_2 = isNull_2 ? /* 036 */ -1.0 : (i.getDouble(1)); /* 037 */ if (isNull_2) { /* 038 */ values_0[1] = null; /* 039 */ } else { /* 040 */ values_0[1] = value_2; /* 041 */ } /* 042 */ /* 043 */ double value_3 = i.getDouble(2); /* 044 */ if (false) { /* 045 */ values_0[2] = null; /* 046 */ } else { /* 047 */ values_0[2] = value_3; /* 048 */ } /* 049 */ /* 050 */ final org.apache.spark.sql.Row value_0 = new org.apache.spark.sql.catalyst.expressions.GenericRowWithSchema(values_0, ((org.apache.spark.sql.types.StructType) references[0] /* schema */)); /* 051 */ if (false) { /* 052 */ mutableRow.setNullAt(0); /* 053 */ } else { /* 054 */ /* 055 */ mutableRow.update(0, value_0); /* 056 */ } /* 057 */ /* 058 */ return mutableRow; /* 059 */ } /* 060 */ /* 061 */ /* 062 */ } 20/01/23 18:54:05.549 Executor task launch worker for task 73 INFO Executor: Finished task 0.0 in stage 18.0 (TID 73). 2006 bytes result sent to driver 20/01/23 18:54:05.549 Executor task launch worker for task 23 INFO PythonRunner: Times: total = 94, boot = -211, init = 305, finish = 0 20/01/23 18:54:05.550 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$rdd$1) is now cleaned +++ 20/01/23 18:54:05.550 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$fit$2 20/01/23 18:54:05.551 Executor task launch worker for task 23 INFO Executor: Finished task 1.0 in stage 7.0 (TID 23). 2920 bytes result sent to driver 20/01/23 18:54:05.551 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$fit$2) is now cleaned +++ 20/01/23 18:54:05.551 Executor task launch worker for task 23 DEBUG ExecutorMetricsPoller: stageTCMP: (7, 0) -> 3 20/01/23 18:54:05.551 dispatcher-event-loop-3 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_7.0, runningTasks: 3 20/01/23 18:54:05.552 dispatcher-event-loop-3 DEBUG TaskSetManager: No tasks for locality level NO_PREF, so moving to locality level ANY 20/01/23 18:54:05.552 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$count$1$adapted 20/01/23 18:54:05.553 task-result-getter-0 INFO TaskSetManager: Finished task 1.0 in stage 7.0 (TID 23) in 123 ms on 192.168.1.225 (executor driver) (1/4) 20/01/23 18:54:05.529 Executor task launch worker for task 22 INFO Executor: Finished task 0.0 in stage 7.0 (TID 22). 2920 bytes result sent to driver 20/01/23 18:54:05.553 Executor task launch worker for task 22 DEBUG ExecutorMetricsPoller: stageTCMP: (7, 0) -> 2 20/01/23 18:54:05.528 Executor task launch worker for task 24 DEBUG GenerateSafeProjection: code for createexternalrow(input[0, double, true], input[1, double, true], input[2, double, false], StructField(prediction,DoubleType,true), StructField(label,DoubleType,true), StructField(1.0,DoubleType,false)): /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificSafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificSafeProjection extends org.apache.spark.sql.catalyst.expressions.codegen.BaseProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private InternalRow mutableRow; /* 009 */ /* 010 */ /* 011 */ public SpecificSafeProjection(Object[] references) { /* 012 */ this.references = references; /* 013 */ mutableRow = (InternalRow) references[references.length - 1]; /* 014 */ /* 015 */ } /* 016 */ /* 017 */ public void initialize(int partitionIndex) { /* 018 */ /* 019 */ } /* 020 */ /* 021 */ public java.lang.Object apply(java.lang.Object _i) { /* 022 */ InternalRow i = (InternalRow) _i; /* 023 */ Object[] values_0 = new Object[3]; /* 024 */ /* 025 */ boolean isNull_1 = i.isNullAt(0); /* 026 */ double value_1 = isNull_1 ? /* 027 */ -1.0 : (i.getDouble(0)); /* 028 */ if (isNull_1) { /* 029 */ values_0[0] = null; /* 030 */ } else { /* 031 */ values_0[0] = value_1; /* 032 */ } /* 033 */ /* 034 */ boolean isNull_2 = i.isNullAt(1); /* 035 */ double value_2 = isNull_2 ? /* 036 */ -1.0 : (i.getDouble(1)); /* 037 */ if (isNull_2) { /* 038 */ values_0[1] = null; /* 039 */ } else { /* 040 */ values_0[1] = value_2; /* 041 */ } /* 042 */ /* 043 */ double value_3 = i.getDouble(2); /* 044 */ if (false) { /* 045 */ values_0[2] = null; /* 046 */ } else { /* 047 */ values_0[2] = value_3; /* 048 */ } /* 049 */ /* 050 */ final org.apache.spark.sql.Row value_0 = new org.apache.spark.sql.catalyst.expressions.GenericRowWithSchema(values_0, ((org.apache.spark.sql.types.StructType) references[0] /* schema */)); /* 051 */ if (false) { /* 052 */ mutableRow.setNullAt(0); /* 053 */ } else { /* 054 */ /* 055 */ mutableRow.update(0, value_0); /* 056 */ } /* 057 */ /* 058 */ return mutableRow; /* 059 */ } /* 060 */ /* 061 */ /* 062 */ } 20/01/23 18:54:05.549 Executor task launch worker for task 25 INFO PythonRunner: Times: total = 73, boot = -176, init = 249, finish = 0 20/01/23 18:54:05.553 dispatcher-event-loop-1 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_7.0, runningTasks: 2 20/01/23 18:54:05.555 task-result-getter-3 INFO TaskSetManager: Finished task 0.0 in stage 7.0 (TID 22) in 126 ms on 192.168.1.225 (executor driver) (2/4) 20/01/23 18:54:05.555 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$count$1$adapted) is now cleaned +++ 20/01/23 18:54:05.555 Executor task launch worker for task 25 INFO Executor: Finished task 3.0 in stage 7.0 (TID 25). 3101 bytes result sent to driver 20/01/23 18:54:05.555 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$runJob$5 20/01/23 18:54:05.555 dispatcher-event-loop-0 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_7.0, runningTasks: 1 20/01/23 18:54:05.556 Executor task launch worker for task 25 DEBUG ExecutorMetricsPoller: stageTCMP: (7, 0) -> 1 20/01/23 18:54:05.556 task-result-getter-2 INFO TaskSetManager: Finished task 3.0 in stage 7.0 (TID 25) in 126 ms on 192.168.1.225 (executor driver) (3/4) 20/01/23 18:54:05.558 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$runJob$5) is now cleaned +++ 20/01/23 18:54:05.558 Thread-4 INFO SparkContext: Starting job: count at CountVectorizer.scala:197 20/01/23 18:54:05.549 dispatcher-event-loop-0 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_18.0, runningTasks: 0 20/01/23 18:54:05.558 Executor task launch worker for task 73 DEBUG ExecutorMetricsPoller: removing (18, 0) from stageTCMP 20/01/23 18:54:05.559 dag-scheduler-event-loop INFO DAGScheduler: Got job 8 (count at CountVectorizer.scala:197) with 4 output partitions 20/01/23 18:54:05.560 dag-scheduler-event-loop INFO DAGScheduler: Final stage: ResultStage 12 (count at CountVectorizer.scala:197) 20/01/23 18:54:05.560 dag-scheduler-event-loop INFO DAGScheduler: Parents of final stage: List() 20/01/23 18:54:05.561 dag-scheduler-event-loop INFO DAGScheduler: Missing parents: List() 20/01/23 18:54:05.561 dag-scheduler-event-loop DEBUG DAGScheduler: submitStage(ResultStage 12 (name=count at CountVectorizer.scala:197;jobs=8)) 20/01/23 18:54:05.561 dag-scheduler-event-loop DEBUG DAGScheduler: missing: List() 20/01/23 18:54:05.561 dag-scheduler-event-loop INFO DAGScheduler: Submitting ResultStage 12 (MapPartitionsRDD[46] at map at CountVectorizer.scala:191), which has no missing parents 20/01/23 18:54:05.561 dag-scheduler-event-loop DEBUG DAGScheduler: submitMissingTasks(ResultStage 12) 20/01/23 18:54:05.563 Executor task launch worker for task 24 INFO PythonRunner: Times: total = 124, boot = -190, init = 314, finish = 0 20/01/23 18:54:05.565 Executor task launch worker for task 24 INFO Executor: Finished task 2.0 in stage 7.0 (TID 24). 2920 bytes result sent to driver 20/01/23 18:54:05.565 dispatcher-event-loop-2 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_7.0, runningTasks: 0 20/01/23 18:54:05.566 Executor task launch worker for task 24 DEBUG ExecutorMetricsPoller: removing (7, 0) from stageTCMP 20/01/23 18:54:05.566 task-result-getter-1 INFO TaskSetManager: Finished task 2.0 in stage 7.0 (TID 24) in 136 ms on 192.168.1.225 (executor driver) (4/4) 20/01/23 18:54:05.566 task-result-getter-1 INFO TaskSchedulerImpl: Removed TaskSet 7.0, whose tasks have all completed, from pool 20/01/23 18:54:05.567 dag-scheduler-event-loop INFO DAGScheduler: ResultStage 7 (treeAggregate at Statistics.scala:58) finished in 0.154 s 20/01/23 18:54:05.567 dag-scheduler-event-loop DEBUG DAGScheduler: After removal of stage 7, remaining stages = 0 20/01/23 18:54:05.567 dag-scheduler-event-loop INFO DAGScheduler: Job 5 is finished. Cancelling potential speculative or zombie tasks for this job 20/01/23 18:54:05.567 dag-scheduler-event-loop INFO TaskSchedulerImpl: Killing all running tasks in stage 7: Stage finished 20/01/23 18:54:05.567 Thread-4 INFO DAGScheduler: Job 5 finished: treeAggregate at Statistics.scala:58, took 0.162974 s 20/01/23 18:54:05.568 task-result-getter-1 INFO TaskSetManager: Finished task 0.0 in stage 18.0 (TID 73) in 91 ms on 192.168.1.225 (executor driver) (1/1) 20/01/23 18:54:05.569 task-result-getter-1 INFO TaskSchedulerImpl: Removed TaskSet 18.0, whose tasks have all completed, from pool 20/01/23 18:54:05.570 dag-scheduler-event-loop INFO DAGScheduler: ResultStage 18 (isEmpty at ALS.scala:931) finished in 0.104 s 20/01/23 18:54:05.570 dag-scheduler-event-loop DEBUG DAGScheduler: After removal of stage 18, remaining stages = 0 20/01/23 18:54:05.570 dag-scheduler-event-loop INFO DAGScheduler: Job 5 is finished. Cancelling potential speculative or zombie tasks for this job 20/01/23 18:54:05.570 dag-scheduler-event-loop INFO TaskSchedulerImpl: Killing all running tasks in stage 18: Stage finished 20/01/23 18:54:05.570 Thread-4 INFO DAGScheduler: Job 5 finished: isEmpty at ALS.scala:931, took 0.112232 s 20/01/23 18:54:05.571 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$partitionRatings$1 20/01/23 18:54:05.572 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$partitionRatings$1) is now cleaned +++ 20/01/23 18:54:05.573 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$groupByKey$2 20/01/23 18:54:05.574 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$groupByKey$2) is now cleaned +++ 20/01/23 18:54:05.576 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$groupByKey$3 20/01/23 18:54:05.577 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$groupByKey$3) is now cleaned +++ 20/01/23 18:54:05.581 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$groupByKey$4 20/01/23 18:54:05.582 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$groupByKey$4) is now cleaned +++ 20/01/23 18:54:05.583 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$partitionRatings$7 20/01/23 18:54:05.584 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$partitionRatings$7) is now cleaned +++ 20/01/23 18:54:05.585 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$makeBlocks$1 20/01/23 18:54:05.585 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$makeBlocks$1) is now cleaned +++ 20/01/23 18:54:05.586 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$groupByKey$2 20/01/23 18:54:05.587 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$groupByKey$2) is now cleaned +++ 20/01/23 18:54:05.589 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$groupByKey$3 20/01/23 18:54:05.590 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_14 stored as values in memory (estimated size 20.3 KiB, free 366.1 MiB) 20/01/23 18:54:05.590 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_14 locally took 0 ms 20/01/23 18:54:05.590 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_14 without replication took 0 ms 20/01/23 18:54:05.591 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_14_piece0 stored as bytes in memory (estimated size 10.0 KiB, free 366.1 MiB) 20/01/23 18:54:05.592 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added broadcast_14_piece0 in memory on 192.168.1.225:41233 (size: 10.0 KiB, free: 366.2 MiB) 20/01/23 18:54:05.609 dag-scheduler-event-loop DEBUG BlockManagerMaster: Updated info of block broadcast_14_piece0 20/01/23 18:54:05.609 dag-scheduler-event-loop DEBUG BlockManager: Told master about block broadcast_14_piece0 20/01/23 18:54:05.609 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_14_piece0 locally took 18 ms 20/01/23 18:54:05.609 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_14_piece0 without replication took 18 ms 20/01/23 18:54:05.609 dag-scheduler-event-loop INFO SparkContext: Created broadcast 14 from broadcast at DAGScheduler.scala:1200 20/01/23 18:54:05.610 dag-scheduler-event-loop INFO DAGScheduler: Submitting 4 missing tasks from ResultStage 12 (MapPartitionsRDD[46] at map at CountVectorizer.scala:191) (first 15 tasks are for partitions Vector(0, 1, 2, 3)) 20/01/23 18:54:05.610 dag-scheduler-event-loop INFO TaskSchedulerImpl: Adding task set 12.0 with 4 tasks 20/01/23 18:54:05.610 dag-scheduler-event-loop DEBUG TaskSetManager: Epoch for TaskSet 12.0: 2 20/01/23 18:54:05.611 dag-scheduler-event-loop DEBUG TaskSetManager: Adding pending tasks took 0 ms 20/01/23 18:54:05.611 dag-scheduler-event-loop DEBUG TaskSetManager: Valid locality levels for TaskSet 12.0: NO_PREF, ANY 20/01/23 18:54:05.611 dispatcher-event-loop-3 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_12.0, runningTasks: 0 20/01/23 18:54:05.611 dispatcher-event-loop-3 INFO TaskSetManager: Starting task 0.0 in stage 12.0 (TID 40, 192.168.1.225, executor driver, partition 0, PROCESS_LOCAL, 7430 bytes) 20/01/23 18:54:05.612 dispatcher-event-loop-3 INFO TaskSetManager: Starting task 1.0 in stage 12.0 (TID 41, 192.168.1.225, executor driver, partition 1, PROCESS_LOCAL, 7415 bytes) 20/01/23 18:54:05.612 dispatcher-event-loop-3 INFO TaskSetManager: Starting task 2.0 in stage 12.0 (TID 42, 192.168.1.225, executor driver, partition 2, PROCESS_LOCAL, 7398 bytes) 20/01/23 18:54:05.612 dispatcher-event-loop-3 INFO TaskSetManager: Starting task 3.0 in stage 12.0 (TID 43, 192.168.1.225, executor driver, partition 3, PROCESS_LOCAL, 7380 bytes) 20/01/23 18:54:05.612 Executor task launch worker for task 41 INFO Executor: Running task 1.0 in stage 12.0 (TID 41) 20/01/23 18:54:05.612 Executor task launch worker for task 40 INFO Executor: Running task 0.0 in stage 12.0 (TID 40) 20/01/23 18:54:05.613 Executor task launch worker for task 43 INFO Executor: Running task 3.0 in stage 12.0 (TID 43) 20/01/23 18:54:05.613 Executor task launch worker for task 41 DEBUG ExecutorMetricsPoller: stageTCMP: (12, 0) -> 1 20/01/23 18:54:05.612 Executor task launch worker for task 42 INFO Executor: Running task 2.0 in stage 12.0 (TID 42) 20/01/23 18:54:05.613 Executor task launch worker for task 40 DEBUG ExecutorMetricsPoller: stageTCMP: (12, 0) -> 3 20/01/23 18:54:05.613 Executor task launch worker for task 42 DEBUG ExecutorMetricsPoller: stageTCMP: (12, 0) -> 4 20/01/23 18:54:05.614 Executor task launch worker for task 41 DEBUG BlockManager: Getting local block broadcast_14 20/01/23 18:54:05.614 Executor task launch worker for task 41 DEBUG BlockManager: Level for block broadcast_14 is StorageLevel(disk, memory, deserialized, 1 replicas) 20/01/23 18:54:05.613 Executor task launch worker for task 43 DEBUG ExecutorMetricsPoller: stageTCMP: (12, 0) -> 2 20/01/23 18:54:05.616 Executor task launch worker for task 41 DEBUG BlockManager: Getting local block rdd_46_1 20/01/23 18:54:05.616 Executor task launch worker for task 41 DEBUG BlockManager: Block rdd_46_1 was not found 20/01/23 18:54:05.616 Executor task launch worker for task 41 DEBUG BlockManager: Getting remote block rdd_46_1 20/01/23 18:54:05.616 Executor task launch worker for task 43 DEBUG BlockManager: Getting local block rdd_46_3 20/01/23 18:54:05.616 Executor task launch worker for task 43 DEBUG BlockManager: Block rdd_46_3 was not found 20/01/23 18:54:05.616 Executor task launch worker for task 43 DEBUG BlockManager: Getting remote block rdd_46_3 20/01/23 18:54:05.617 Executor task launch worker for task 41 DEBUG BlockManager: Block rdd_46_1 is unknown by block manager master 20/01/23 18:54:05.617 Executor task launch worker for task 43 DEBUG BlockManager: Block rdd_46_3 is unknown by block manager master 20/01/23 18:54:05.621 Executor task launch worker for task 40 DEBUG BlockManager: Getting local block rdd_46_0 20/01/23 18:54:05.621 Executor task launch worker for task 40 DEBUG BlockManager: Block rdd_46_0 was not found 20/01/23 18:54:05.621 Executor task launch worker for task 40 DEBUG BlockManager: Getting remote block rdd_46_0 20/01/23 18:54:05.623 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$groupByKey$3) is now cleaned +++ 20/01/23 18:54:05.623 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$groupByKey$4 20/01/23 18:54:05.624 Executor task launch worker for task 40 DEBUG BlockManager: Block rdd_46_0 is unknown by block manager master 20/01/23 18:54:05.624 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$groupByKey$4) is now cleaned +++ 20/01/23 18:54:05.625 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$makeBlocks$5 20/01/23 18:54:05.625 Executor task launch worker for task 42 DEBUG BlockManager: Getting local block rdd_46_2 20/01/23 18:54:05.625 Executor task launch worker for task 42 DEBUG BlockManager: Block rdd_46_2 was not found 20/01/23 18:54:05.625 Executor task launch worker for task 42 DEBUG BlockManager: Getting remote block rdd_46_2 20/01/23 18:54:05.626 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$makeBlocks$5) is now cleaned +++ 20/01/23 18:54:05.626 dispatcher-event-loop-0 INFO MapOutputTrackerMasterEndpoint: MapOutputTrackerMasterEndpoint stopped! 20/01/23 18:54:05.627 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$makeBlocks$7 20/01/23 18:54:05.628 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$makeBlocks$7) is now cleaned +++ 20/01/23 18:54:05.628 Thread-4 DEBUG ClosureCleaner: Cleaning lambda: $anonfun$count$1$adapted 20/01/23 18:54:05.630 Executor task launch worker for task 42 DEBUG BlockManager: Block rdd_46_2 is unknown by block manager master 20/01/23 18:54:05.633 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(154) 20/01/23 18:54:05.634 Thread-4 DEBUG ClosureCleaner: +++ Lambda closure ($anonfun$count$1$adapted) is now cleaned +++ 20/01/23 18:54:05.