21/01/17 17:42:53.225 main WARN Utils: Your hostname, spark-jenkins-arm-worker resolves to a loopback address: 127.0.1.1; using 192.168.1.225 instead (on interface eth0) 21/01/17 17:42:53.226 main WARN Utils: Set SPARK_LOCAL_IP if you need to bind to another address 21/01/17 17:42:53.257 main WARN Utils: Your hostname, spark-jenkins-arm-worker resolves to a loopback address: 127.0.1.1; using 192.168.1.225 instead (on interface eth0) 21/01/17 17:42:53.259 main WARN Utils: Set SPARK_LOCAL_IP if you need to bind to another address 21/01/17 17:42:53.265 main WARN Utils: Your hostname, spark-jenkins-arm-worker resolves to a loopback address: 127.0.1.1; using 192.168.1.225 instead (on interface eth0) 21/01/17 17:42:53.267 main WARN Utils: Set SPARK_LOCAL_IP if you need to bind to another address 21/01/17 17:42:53.265 main WARN Utils: Your hostname, spark-jenkins-arm-worker resolves to a loopback address: 127.0.1.1; using 192.168.1.225 instead (on interface eth0) 21/01/17 17:42:53.269 main WARN Utils: Set SPARK_LOCAL_IP if you need to bind to another address 21/01/17 17:42:53.856 main DEBUG ShutdownHookManager: Adding shutdown hook 21/01/17 17:42:53.884 main DEBUG ShutdownHookManager: Adding shutdown hook 21/01/17 17:42:53.898 main DEBUG ShutdownHookManager: Adding shutdown hook 21/01/17 17:42:53.908 main DEBUG ShutdownHookManager: Adding shutdown hook 21/01/17 17:42:53.934 main DEBUG PythonGatewayServer: Started PythonGatewayServer on port 43951 21/01/17 17:42:53.949 main DEBUG PythonGatewayServer: Started PythonGatewayServer on port 39539 21/01/17 17:42:53.963 main DEBUG PythonGatewayServer: Started PythonGatewayServer on port 34661 21/01/17 17:42:53.981 main DEBUG PythonGatewayServer: Started PythonGatewayServer on port 43025 21/01/17 17:42:54.078 Thread-3 INFO SparkContext: Running Spark version 3.2.0-SNAPSHOT 21/01/17 17:42:54.079 Thread-3 INFO SparkContext: Running Spark version 3.2.0-SNAPSHOT 21/01/17 17:42:54.080 Thread-3 INFO SparkContext: Running Spark version 3.2.0-SNAPSHOT 21/01/17 17:42:54.091 Thread-3 INFO SparkContext: Running Spark version 3.2.0-SNAPSHOT 21/01/17 17:42:54.146 Thread-3 DEBUG MutableMetricsFactory: field org.apache.hadoop.metrics2.lib.MutableRate org.apache.hadoop.security.UserGroupInformation$UgiMetrics.loginSuccess with annotation @org.apache.hadoop.metrics2.annotation.Metric(about=, sampleName=Ops, always=false, type=DEFAULT, valueName=Time, value=[Rate of successful kerberos logins and latency (milliseconds)]) 21/01/17 17:42:54.148 Thread-3 DEBUG MutableMetricsFactory: field org.apache.hadoop.metrics2.lib.MutableRate org.apache.hadoop.security.UserGroupInformation$UgiMetrics.loginSuccess with annotation @org.apache.hadoop.metrics2.annotation.Metric(about=, sampleName=Ops, always=false, type=DEFAULT, valueName=Time, value=[Rate of successful kerberos logins and latency (milliseconds)]) 21/01/17 17:42:54.158 Thread-3 DEBUG MutableMetricsFactory: field org.apache.hadoop.metrics2.lib.MutableRate org.apache.hadoop.security.UserGroupInformation$UgiMetrics.loginFailure with annotation @org.apache.hadoop.metrics2.annotation.Metric(about=, sampleName=Ops, always=false, type=DEFAULT, valueName=Time, value=[Rate of failed kerberos logins and latency (milliseconds)]) 21/01/17 17:42:54.158 Thread-3 DEBUG MutableMetricsFactory: field org.apache.hadoop.metrics2.lib.MutableRate org.apache.hadoop.security.UserGroupInformation$UgiMetrics.loginSuccess with annotation @org.apache.hadoop.metrics2.annotation.Metric(about=, sampleName=Ops, always=false, type=DEFAULT, valueName=Time, value=[Rate of successful kerberos logins and latency (milliseconds)]) 21/01/17 17:42:54.158 Thread-3 DEBUG MutableMetricsFactory: field org.apache.hadoop.metrics2.lib.MutableRate org.apache.hadoop.security.UserGroupInformation$UgiMetrics.loginSuccess with annotation @org.apache.hadoop.metrics2.annotation.Metric(about=, sampleName=Ops, always=false, type=DEFAULT, valueName=Time, value=[Rate of successful kerberos logins and latency (milliseconds)]) 21/01/17 17:42:54.159 Thread-3 DEBUG MutableMetricsFactory: field org.apache.hadoop.metrics2.lib.MutableRate org.apache.hadoop.security.UserGroupInformation$UgiMetrics.getGroups with annotation @org.apache.hadoop.metrics2.annotation.Metric(about=, sampleName=Ops, always=false, type=DEFAULT, valueName=Time, value=[GetGroups]) 21/01/17 17:42:54.160 Thread-3 DEBUG MutableMetricsFactory: field org.apache.hadoop.metrics2.lib.MutableRate org.apache.hadoop.security.UserGroupInformation$UgiMetrics.loginFailure with annotation @org.apache.hadoop.metrics2.annotation.Metric(about=, sampleName=Ops, always=false, type=DEFAULT, valueName=Time, value=[Rate of failed kerberos logins and latency (milliseconds)]) 21/01/17 17:42:54.160 Thread-3 DEBUG MutableMetricsFactory: field org.apache.hadoop.metrics2.lib.MutableRate org.apache.hadoop.security.UserGroupInformation$UgiMetrics.getGroups with annotation @org.apache.hadoop.metrics2.annotation.Metric(about=, sampleName=Ops, always=false, type=DEFAULT, valueName=Time, value=[GetGroups]) 21/01/17 17:42:54.160 Thread-3 DEBUG MetricsSystemImpl: UgiMetrics, User and group related metrics 21/01/17 17:42:54.162 Thread-3 DEBUG MetricsSystemImpl: UgiMetrics, User and group related metrics 21/01/17 17:42:54.171 Thread-3 DEBUG MutableMetricsFactory: field org.apache.hadoop.metrics2.lib.MutableRate org.apache.hadoop.security.UserGroupInformation$UgiMetrics.loginFailure with annotation @org.apache.hadoop.metrics2.annotation.Metric(about=, sampleName=Ops, always=false, type=DEFAULT, valueName=Time, value=[Rate of failed kerberos logins and latency (milliseconds)]) 21/01/17 17:42:54.171 Thread-3 DEBUG MutableMetricsFactory: field org.apache.hadoop.metrics2.lib.MutableRate org.apache.hadoop.security.UserGroupInformation$UgiMetrics.loginFailure with annotation @org.apache.hadoop.metrics2.annotation.Metric(about=, sampleName=Ops, always=false, type=DEFAULT, valueName=Time, value=[Rate of failed kerberos logins and latency (milliseconds)]) 21/01/17 17:42:54.171 Thread-3 DEBUG MutableMetricsFactory: field org.apache.hadoop.metrics2.lib.MutableRate org.apache.hadoop.security.UserGroupInformation$UgiMetrics.getGroups with annotation @org.apache.hadoop.metrics2.annotation.Metric(about=, sampleName=Ops, always=false, type=DEFAULT, valueName=Time, value=[GetGroups]) 21/01/17 17:42:54.171 Thread-3 DEBUG MutableMetricsFactory: field org.apache.hadoop.metrics2.lib.MutableRate org.apache.hadoop.security.UserGroupInformation$UgiMetrics.getGroups with annotation @org.apache.hadoop.metrics2.annotation.Metric(about=, sampleName=Ops, always=false, type=DEFAULT, valueName=Time, value=[GetGroups]) 21/01/17 17:42:54.173 Thread-3 DEBUG MetricsSystemImpl: UgiMetrics, User and group related metrics 21/01/17 17:42:54.173 Thread-3 DEBUG MetricsSystemImpl: UgiMetrics, User and group related metrics 21/01/17 17:42:54.246 Thread-3 DEBUG Shell: Failed to detect a valid hadoop home directory java.io.IOException: HADOOP_HOME or hadoop.home.dir are not set. at org.apache.hadoop.util.Shell.checkHadoopHome(Shell.java:329) at org.apache.hadoop.util.Shell.(Shell.java:354) at org.apache.hadoop.util.StringUtils.(StringUtils.java:80) at org.apache.hadoop.security.SecurityUtil.getAuthenticationMethod(SecurityUtil.java:611) at org.apache.hadoop.security.UserGroupInformation.initialize(UserGroupInformation.java:274) at org.apache.hadoop.security.UserGroupInformation.ensureInitialized(UserGroupInformation.java:262) at org.apache.hadoop.security.UserGroupInformation.loginUserFromSubject(UserGroupInformation.java:807) at org.apache.hadoop.security.UserGroupInformation.getLoginUser(UserGroupInformation.java:777) at org.apache.hadoop.security.UserGroupInformation.getCurrentUser(UserGroupInformation.java:650) at org.apache.spark.util.Utils$.$anonfun$getCurrentUserName$1(Utils.scala:2472) at scala.Option.getOrElse(Option.scala:189) at org.apache.spark.util.Utils$.getCurrentUserName(Utils.scala:2472) at org.apache.spark.SparkContext.(SparkContext.scala:314) at org.apache.spark.api.java.JavaSparkContext.(JavaSparkContext.scala:58) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:247) at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) at py4j.Gateway.invoke(Gateway.java:238) at py4j.commands.ConstructorCommand.invokeConstructor(ConstructorCommand.java:80) at py4j.commands.ConstructorCommand.execute(ConstructorCommand.java:69) at py4j.GatewayConnection.run(GatewayConnection.java:238) at java.lang.Thread.run(Thread.java:748) 21/01/17 17:42:54.260 Thread-3 DEBUG Shell: Failed to detect a valid hadoop home directory java.io.IOException: HADOOP_HOME or hadoop.home.dir are not set. at org.apache.hadoop.util.Shell.checkHadoopHome(Shell.java:329) at org.apache.hadoop.util.Shell.(Shell.java:354) at org.apache.hadoop.util.StringUtils.(StringUtils.java:80) at org.apache.hadoop.security.SecurityUtil.getAuthenticationMethod(SecurityUtil.java:611) at org.apache.hadoop.security.UserGroupInformation.initialize(UserGroupInformation.java:274) at org.apache.hadoop.security.UserGroupInformation.ensureInitialized(UserGroupInformation.java:262) at org.apache.hadoop.security.UserGroupInformation.loginUserFromSubject(UserGroupInformation.java:807) at org.apache.hadoop.security.UserGroupInformation.getLoginUser(UserGroupInformation.java:777) at org.apache.hadoop.security.UserGroupInformation.getCurrentUser(UserGroupInformation.java:650) at org.apache.spark.util.Utils$.$anonfun$getCurrentUserName$1(Utils.scala:2472) at scala.Option.getOrElse(Option.scala:189) at org.apache.spark.util.Utils$.getCurrentUserName(Utils.scala:2472) at org.apache.spark.SparkContext.(SparkContext.scala:314) at org.apache.spark.api.java.JavaSparkContext.(JavaSparkContext.scala:58) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:247) at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) at py4j.Gateway.invoke(Gateway.java:238) at py4j.commands.ConstructorCommand.invokeConstructor(ConstructorCommand.java:80) at py4j.commands.ConstructorCommand.execute(ConstructorCommand.java:69) at py4j.GatewayConnection.run(GatewayConnection.java:238) at java.lang.Thread.run(Thread.java:748) 21/01/17 17:42:54.281 Thread-3 DEBUG Shell: setsid exited with exit code 0 21/01/17 17:42:54.278 Thread-3 DEBUG Shell: Failed to detect a valid hadoop home directory java.io.IOException: HADOOP_HOME or hadoop.home.dir are not set. at org.apache.hadoop.util.Shell.checkHadoopHome(Shell.java:329) at org.apache.hadoop.util.Shell.(Shell.java:354) at org.apache.hadoop.util.StringUtils.(StringUtils.java:80) at org.apache.hadoop.security.SecurityUtil.getAuthenticationMethod(SecurityUtil.java:611) at org.apache.hadoop.security.UserGroupInformation.initialize(UserGroupInformation.java:274) at org.apache.hadoop.security.UserGroupInformation.ensureInitialized(UserGroupInformation.java:262) at org.apache.hadoop.security.UserGroupInformation.loginUserFromSubject(UserGroupInformation.java:807) at org.apache.hadoop.security.UserGroupInformation.getLoginUser(UserGroupInformation.java:777) at org.apache.hadoop.security.UserGroupInformation.getCurrentUser(UserGroupInformation.java:650) at org.apache.spark.util.Utils$.$anonfun$getCurrentUserName$1(Utils.scala:2472) at scala.Option.getOrElse(Option.scala:189) at org.apache.spark.util.Utils$.getCurrentUserName(Utils.scala:2472) at org.apache.spark.SparkContext.(SparkContext.scala:314) at org.apache.spark.api.java.JavaSparkContext.(JavaSparkContext.scala:58) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:247) at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) at py4j.Gateway.invoke(Gateway.java:238) at py4j.commands.ConstructorCommand.invokeConstructor(ConstructorCommand.java:80) at py4j.commands.ConstructorCommand.execute(ConstructorCommand.java:69) at py4j.GatewayConnection.run(GatewayConnection.java:238) at java.lang.Thread.run(Thread.java:748) 21/01/17 17:42:54.279 Thread-3 DEBUG Shell: Failed to detect a valid hadoop home directory java.io.IOException: HADOOP_HOME or hadoop.home.dir are not set. at org.apache.hadoop.util.Shell.checkHadoopHome(Shell.java:329) at org.apache.hadoop.util.Shell.(Shell.java:354) at org.apache.hadoop.util.StringUtils.(StringUtils.java:80) at org.apache.hadoop.security.SecurityUtil.getAuthenticationMethod(SecurityUtil.java:611) at org.apache.hadoop.security.UserGroupInformation.initialize(UserGroupInformation.java:274) at org.apache.hadoop.security.UserGroupInformation.ensureInitialized(UserGroupInformation.java:262) at org.apache.hadoop.security.UserGroupInformation.loginUserFromSubject(UserGroupInformation.java:807) at org.apache.hadoop.security.UserGroupInformation.getLoginUser(UserGroupInformation.java:777) at org.apache.hadoop.security.UserGroupInformation.getCurrentUser(UserGroupInformation.java:650) at org.apache.spark.util.Utils$.$anonfun$getCurrentUserName$1(Utils.scala:2472) at scala.Option.getOrElse(Option.scala:189) at org.apache.spark.util.Utils$.getCurrentUserName(Utils.scala:2472) at org.apache.spark.SparkContext.(SparkContext.scala:314) at org.apache.spark.api.java.JavaSparkContext.(JavaSparkContext.scala:58) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:247) at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) at py4j.Gateway.invoke(Gateway.java:238) at py4j.commands.ConstructorCommand.invokeConstructor(ConstructorCommand.java:80) at py4j.commands.ConstructorCommand.execute(ConstructorCommand.java:69) at py4j.GatewayConnection.run(GatewayConnection.java:238) at java.lang.Thread.run(Thread.java:748) 21/01/17 17:42:54.287 Thread-3 DEBUG KerberosName: Kerberos krb5 configuration not found, setting default realm to empty 21/01/17 17:42:54.292 Thread-3 DEBUG Shell: setsid exited with exit code 0 21/01/17 17:42:54.292 Thread-3 DEBUG Groups: Creating new Groups object 21/01/17 17:42:54.295 Thread-3 DEBUG Shell: setsid exited with exit code 0 21/01/17 17:42:54.296 Thread-3 DEBUG NativeCodeLoader: Trying to load the custom-built native-hadoop library... 21/01/17 17:42:54.296 Thread-3 DEBUG NativeCodeLoader: Failed to load native-hadoop with error: java.lang.UnsatisfiedLinkError: no hadoop in java.library.path 21/01/17 17:42:54.296 Thread-3 DEBUG NativeCodeLoader: java.library.path=/home/jenkins/deps/arrow/arrow-cpp-bin/lib:/usr/java/packages/lib/aarch64:/usr/lib/aarch64-linux-gnu/jni:/lib/aarch64-linux-gnu:/usr/lib/aarch64-linux-gnu:/usr/lib/jni:/lib:/usr/lib 21/01/17 17:42:54.297 Thread-3 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 21/01/17 17:42:54.297 Thread-3 DEBUG Shell: setsid exited with exit code 0 21/01/17 17:42:54.297 Thread-3 DEBUG PerformanceAdvisory: Falling back to shell based 21/01/17 17:42:54.297 Thread-3 DEBUG JniBasedUnixGroupsMappingWithFallback: Group mapping impl=org.apache.hadoop.security.ShellBasedUnixGroupsMapping 21/01/17 17:42:54.299 Thread-3 DEBUG KerberosName: Kerberos krb5 configuration not found, setting default realm to empty 21/01/17 17:42:54.302 Thread-3 DEBUG KerberosName: Kerberos krb5 configuration not found, setting default realm to empty 21/01/17 17:42:54.303 Thread-3 DEBUG KerberosName: Kerberos krb5 configuration not found, setting default realm to empty 21/01/17 17:42:54.303 Thread-3 DEBUG Groups: Creating new Groups object 21/01/17 17:42:54.308 Thread-3 DEBUG Groups: Creating new Groups object 21/01/17 17:42:54.312 Thread-3 DEBUG Groups: Creating new Groups object 21/01/17 17:42:54.312 Thread-3 DEBUG NativeCodeLoader: Trying to load the custom-built native-hadoop library... 21/01/17 17:42:54.312 Thread-3 DEBUG NativeCodeLoader: Trying to load the custom-built native-hadoop library... 21/01/17 17:42:54.313 Thread-3 DEBUG NativeCodeLoader: Failed to load native-hadoop with error: java.lang.UnsatisfiedLinkError: no hadoop in java.library.path 21/01/17 17:42:54.313 Thread-3 DEBUG NativeCodeLoader: java.library.path=/home/jenkins/deps/arrow/arrow-cpp-bin/lib:/usr/java/packages/lib/aarch64:/usr/lib/aarch64-linux-gnu/jni:/lib/aarch64-linux-gnu:/usr/lib/aarch64-linux-gnu:/usr/lib/jni:/lib:/usr/lib 21/01/17 17:42:54.313 Thread-3 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 21/01/17 17:42:54.313 Thread-3 DEBUG NativeCodeLoader: Failed to load native-hadoop with error: java.lang.UnsatisfiedLinkError: no hadoop in java.library.path 21/01/17 17:42:54.313 Thread-3 DEBUG NativeCodeLoader: java.library.path=/home/jenkins/deps/arrow/arrow-cpp-bin/lib:/usr/java/packages/lib/aarch64:/usr/lib/aarch64-linux-gnu/jni:/lib/aarch64-linux-gnu:/usr/lib/aarch64-linux-gnu:/usr/lib/jni:/lib:/usr/lib 21/01/17 17:42:54.313 Thread-3 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 21/01/17 17:42:54.313 Thread-3 DEBUG PerformanceAdvisory: Falling back to shell based 21/01/17 17:42:54.314 Thread-3 DEBUG JniBasedUnixGroupsMappingWithFallback: Group mapping impl=org.apache.hadoop.security.ShellBasedUnixGroupsMapping 21/01/17 17:42:54.314 Thread-3 DEBUG PerformanceAdvisory: Falling back to shell based 21/01/17 17:42:54.314 Thread-3 DEBUG JniBasedUnixGroupsMappingWithFallback: Group mapping impl=org.apache.hadoop.security.ShellBasedUnixGroupsMapping 21/01/17 17:42:54.316 Thread-3 DEBUG NativeCodeLoader: Trying to load the custom-built native-hadoop library... 21/01/17 17:42:54.316 Thread-3 DEBUG NativeCodeLoader: Failed to load native-hadoop with error: java.lang.UnsatisfiedLinkError: no hadoop in java.library.path 21/01/17 17:42:54.316 Thread-3 DEBUG NativeCodeLoader: java.library.path=/home/jenkins/deps/arrow/arrow-cpp-bin/lib:/usr/java/packages/lib/aarch64:/usr/lib/aarch64-linux-gnu/jni:/lib/aarch64-linux-gnu:/usr/lib/aarch64-linux-gnu:/usr/lib/jni:/lib:/usr/lib 21/01/17 17:42:54.316 Thread-3 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 21/01/17 17:42:54.317 Thread-3 DEBUG PerformanceAdvisory: Falling back to shell based 21/01/17 17:42:54.317 Thread-3 DEBUG JniBasedUnixGroupsMappingWithFallback: Group mapping impl=org.apache.hadoop.security.ShellBasedUnixGroupsMapping 21/01/17 17:42:54.457 Thread-3 DEBUG Groups: Group mapping impl=org.apache.hadoop.security.JniBasedUnixGroupsMappingWithFallback; cacheTimeout=300000; warningDeltaMs=5000 21/01/17 17:42:54.459 Thread-3 DEBUG UserGroupInformation: hadoop login 21/01/17 17:42:54.460 Thread-3 DEBUG UserGroupInformation: hadoop login commit 21/01/17 17:42:54.463 Thread-3 DEBUG UserGroupInformation: using local user:UnixPrincipal: jenkins 21/01/17 17:42:54.463 Thread-3 DEBUG UserGroupInformation: Using user: "UnixPrincipal: jenkins" with name jenkins 21/01/17 17:42:54.463 Thread-3 DEBUG UserGroupInformation: User entry: "jenkins" 21/01/17 17:42:54.463 Thread-3 DEBUG UserGroupInformation: Assuming keytab is managed externally since logged in from subject. 21/01/17 17:42:54.464 Thread-3 DEBUG UserGroupInformation: UGI loginUser:jenkins (auth:SIMPLE) 21/01/17 17:42:54.505 Thread-3 DEBUG Groups: Group mapping impl=org.apache.hadoop.security.JniBasedUnixGroupsMappingWithFallback; cacheTimeout=300000; warningDeltaMs=5000 21/01/17 17:42:54.508 Thread-3 DEBUG UserGroupInformation: hadoop login 21/01/17 17:42:54.509 Thread-3 DEBUG UserGroupInformation: hadoop login commit 21/01/17 17:42:54.509 Thread-3 DEBUG Groups: Group mapping impl=org.apache.hadoop.security.JniBasedUnixGroupsMappingWithFallback; cacheTimeout=300000; warningDeltaMs=5000 21/01/17 17:42:54.510 Thread-3 INFO ResourceUtils: ============================================================== 21/01/17 17:42:54.510 Thread-3 INFO ResourceUtils: No custom resources configured for spark.driver. 21/01/17 17:42:54.511 Thread-3 INFO ResourceUtils: ============================================================== 21/01/17 17:42:54.511 Thread-3 DEBUG UserGroupInformation: using local user:UnixPrincipal: jenkins 21/01/17 17:42:54.511 Thread-3 INFO SparkContext: Submitted application: ReusedPySparkTestCase 21/01/17 17:42:54.511 Thread-3 DEBUG UserGroupInformation: Using user: "UnixPrincipal: jenkins" with name jenkins 21/01/17 17:42:54.512 Thread-3 DEBUG UserGroupInformation: User entry: "jenkins" 21/01/17 17:42:54.512 Thread-3 DEBUG UserGroupInformation: Assuming keytab is managed externally since logged in from subject. 21/01/17 17:42:54.512 Thread-3 DEBUG UserGroupInformation: hadoop login 21/01/17 17:42:54.512 Thread-3 DEBUG UserGroupInformation: UGI loginUser:jenkins (auth:SIMPLE) 21/01/17 17:42:54.512 Thread-3 DEBUG UserGroupInformation: hadoop login commit 21/01/17 17:42:54.515 Thread-3 DEBUG UserGroupInformation: using local user:UnixPrincipal: jenkins 21/01/17 17:42:54.515 Thread-3 DEBUG UserGroupInformation: Using user: "UnixPrincipal: jenkins" with name jenkins 21/01/17 17:42:54.516 Thread-3 DEBUG UserGroupInformation: User entry: "jenkins" 21/01/17 17:42:54.516 Thread-3 DEBUG UserGroupInformation: Assuming keytab is managed externally since logged in from subject. 21/01/17 17:42:54.516 Thread-3 DEBUG UserGroupInformation: UGI loginUser:jenkins (auth:SIMPLE) 21/01/17 17:42:54.520 Thread-3 DEBUG Groups: Group mapping impl=org.apache.hadoop.security.JniBasedUnixGroupsMappingWithFallback; cacheTimeout=300000; warningDeltaMs=5000 21/01/17 17:42:54.523 Thread-3 DEBUG UserGroupInformation: hadoop login 21/01/17 17:42:54.523 Thread-3 DEBUG UserGroupInformation: hadoop login commit 21/01/17 17:42:54.526 Thread-3 DEBUG UserGroupInformation: using local user:UnixPrincipal: jenkins 21/01/17 17:42:54.526 Thread-3 DEBUG UserGroupInformation: Using user: "UnixPrincipal: jenkins" with name jenkins 21/01/17 17:42:54.526 Thread-3 DEBUG UserGroupInformation: User entry: "jenkins" 21/01/17 17:42:54.526 Thread-3 DEBUG UserGroupInformation: Assuming keytab is managed externally since logged in from subject. 21/01/17 17:42:54.527 Thread-3 DEBUG UserGroupInformation: UGI loginUser:jenkins (auth:SIMPLE) 21/01/17 17:42:54.549 Thread-3 INFO ResourceProfile: Default ResourceProfile created, executor resources: Map(cores -> name: cores, amount: 1, script: , vendor: , memory -> name: memory, amount: 1024, script: , vendor: , offHeap -> name: offHeap, amount: 0, script: , vendor: ), task resources: Map(cpus -> name: cpus, amount: 1.0) 21/01/17 17:42:54.557 Thread-3 INFO ResourceUtils: ============================================================== 21/01/17 17:42:54.557 Thread-3 INFO ResourceUtils: No custom resources configured for spark.driver. 21/01/17 17:42:54.558 Thread-3 INFO ResourceUtils: ============================================================== 21/01/17 17:42:54.558 Thread-3 INFO SparkContext: Submitted application: ReusedPySparkTestCase 21/01/17 17:42:54.560 Thread-3 INFO ResourceUtils: ============================================================== 21/01/17 17:42:54.560 Thread-3 INFO ResourceUtils: No custom resources configured for spark.driver. 21/01/17 17:42:54.561 Thread-3 INFO ResourceUtils: ============================================================== 21/01/17 17:42:54.561 Thread-3 INFO SparkContext: Submitted application: ReusedPySparkTestCase 21/01/17 17:42:54.569 Thread-3 INFO ResourceUtils: ============================================================== 21/01/17 17:42:54.570 Thread-3 INFO ResourceUtils: No custom resources configured for spark.driver. 21/01/17 17:42:54.571 Thread-3 INFO ResourceUtils: ============================================================== 21/01/17 17:42:54.571 Thread-3 INFO SparkContext: Submitted application: ReusedPySparkTestCase 21/01/17 17:42:54.572 Thread-3 INFO ResourceProfile: Limiting resource is cpu 21/01/17 17:42:54.573 Thread-3 INFO ResourceProfileManager: Added ResourceProfile id: 0 21/01/17 17:42:54.597 Thread-3 INFO ResourceProfile: Default ResourceProfile created, executor resources: Map(cores -> name: cores, amount: 1, script: , vendor: , memory -> name: memory, amount: 1024, script: , vendor: , offHeap -> name: offHeap, amount: 0, script: , vendor: ), task resources: Map(cpus -> name: cpus, amount: 1.0) 21/01/17 17:42:54.603 Thread-3 INFO ResourceProfile: Default ResourceProfile created, executor resources: Map(cores -> name: cores, amount: 1, script: , vendor: , memory -> name: memory, amount: 1024, script: , vendor: , offHeap -> name: offHeap, amount: 0, script: , vendor: ), task resources: Map(cpus -> name: cpus, amount: 1.0) 21/01/17 17:42:54.609 Thread-3 INFO ResourceProfile: Default ResourceProfile created, executor resources: Map(cores -> name: cores, amount: 1, script: , vendor: , memory -> name: memory, amount: 1024, script: , vendor: , offHeap -> name: offHeap, amount: 0, script: , vendor: ), task resources: Map(cpus -> name: cpus, amount: 1.0) 21/01/17 17:42:54.621 Thread-3 INFO ResourceProfile: Limiting resource is cpu 21/01/17 17:42:54.621 Thread-3 INFO ResourceProfileManager: Added ResourceProfile id: 0 21/01/17 17:42:54.627 Thread-3 INFO ResourceProfile: Limiting resource is cpu 21/01/17 17:42:54.627 Thread-3 INFO ResourceProfileManager: Added ResourceProfile id: 0 21/01/17 17:42:54.633 Thread-3 INFO ResourceProfile: Limiting resource is cpu 21/01/17 17:42:54.634 Thread-3 INFO ResourceProfileManager: Added ResourceProfile id: 0 21/01/17 17:42:54.646 Thread-3 INFO SecurityManager: Changing view acls to: jenkins 21/01/17 17:42:54.646 Thread-3 INFO SecurityManager: Changing modify acls to: jenkins 21/01/17 17:42:54.647 Thread-3 INFO SecurityManager: Changing view acls groups to: 21/01/17 17:42:54.648 Thread-3 INFO SecurityManager: Changing modify acls groups to: 21/01/17 17:42:54.648 Thread-3 INFO SecurityManager: SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(jenkins); groups with view permissions: Set(); users with modify permissions: Set(jenkins); groups with modify permissions: Set() 21/01/17 17:42:54.699 Thread-3 INFO SecurityManager: Changing view acls to: jenkins 21/01/17 17:42:54.699 Thread-3 INFO SecurityManager: Changing modify acls to: jenkins 21/01/17 17:42:54.700 Thread-3 INFO SecurityManager: Changing view acls groups to: 21/01/17 17:42:54.701 Thread-3 INFO SecurityManager: Changing modify acls groups to: 21/01/17 17:42:54.706 Thread-3 INFO SecurityManager: Changing view acls to: jenkins 21/01/17 17:42:54.706 Thread-3 INFO SecurityManager: SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(jenkins); groups with view permissions: Set(); users with modify permissions: Set(jenkins); groups with modify permissions: Set() 21/01/17 17:42:54.707 Thread-3 INFO SecurityManager: Changing modify acls to: jenkins 21/01/17 17:42:54.707 Thread-3 INFO SecurityManager: Changing view acls groups to: 21/01/17 17:42:54.708 Thread-3 INFO SecurityManager: Changing modify acls groups to: 21/01/17 17:42:54.708 Thread-3 INFO SecurityManager: SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(jenkins); groups with view permissions: Set(); users with modify permissions: Set(jenkins); groups with modify permissions: Set() 21/01/17 17:42:54.712 Thread-3 INFO SecurityManager: Changing view acls to: jenkins 21/01/17 17:42:54.712 Thread-3 INFO SecurityManager: Changing modify acls to: jenkins 21/01/17 17:42:54.713 Thread-3 INFO SecurityManager: Changing view acls groups to: 21/01/17 17:42:54.714 Thread-3 INFO SecurityManager: Changing modify acls groups to: 21/01/17 17:42:54.714 Thread-3 INFO SecurityManager: SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(jenkins); groups with view permissions: Set(); users with modify permissions: Set(jenkins); groups with modify permissions: Set() 21/01/17 17:42:55.220 Thread-3 DEBUG TransportServer: Shuffle server started on port: 42681 21/01/17 17:42:55.230 Thread-3 INFO Utils: Successfully started service 'sparkDriver' on port 42681. 21/01/17 17:42:55.231 Thread-3 DEBUG SparkEnv: Using serializer: class org.apache.spark.serializer.JavaSerializer 21/01/17 17:42:55.268 Thread-3 INFO SparkEnv: Registering MapOutputTracker 21/01/17 17:42:55.268 Thread-3 DEBUG MapOutputTrackerMasterEndpoint: init 21/01/17 17:42:55.289 Thread-3 DEBUG TransportServer: Shuffle server started on port: 41897 21/01/17 17:42:55.299 Thread-3 INFO Utils: Successfully started service 'sparkDriver' on port 41897. 21/01/17 17:42:55.300 Thread-3 DEBUG SparkEnv: Using serializer: class org.apache.spark.serializer.JavaSerializer 21/01/17 17:42:55.311 Thread-3 INFO SparkEnv: Registering BlockManagerMaster 21/01/17 17:42:55.319 Thread-3 INFO BlockManagerMasterEndpoint: Using org.apache.spark.storage.DefaultTopologyMapper for getting topology information 21/01/17 17:42:55.320 Thread-3 INFO BlockManagerMasterEndpoint: BlockManagerMasterEndpoint up 21/01/17 17:42:55.324 Thread-3 INFO SparkEnv: Registering BlockManagerMasterHeartbeat 21/01/17 17:42:55.335 Thread-3 DEBUG TransportServer: Shuffle server started on port: 37029 21/01/17 17:42:55.338 Thread-3 DEBUG TransportServer: Shuffle server started on port: 37329 21/01/17 17:42:55.342 Thread-3 INFO SparkEnv: Registering MapOutputTracker 21/01/17 17:42:55.343 Thread-3 DEBUG MapOutputTrackerMasterEndpoint: init 21/01/17 17:42:55.345 Thread-3 INFO DiskBlockManager: Created local directory at /home/jenkins/workspace/spark-master-test-python-arm/python/target/3dfd943d-5b3b-4196-84a7-4e3f79e40903/blockmgr-f21ee8f6-0d90-4434-9f4f-e3b2cdd47bb5 21/01/17 17:42:55.346 Thread-3 DEBUG DiskBlockManager: Adding shutdown hook 21/01/17 17:42:55.350 Thread-3 INFO Utils: Successfully started service 'sparkDriver' on port 37029. 21/01/17 17:42:55.352 Thread-3 DEBUG SparkEnv: Using serializer: class org.apache.spark.serializer.JavaSerializer 21/01/17 17:42:55.353 Thread-3 INFO Utils: Successfully started service 'sparkDriver' on port 37329. 21/01/17 17:42:55.355 Thread-3 DEBUG SparkEnv: Using serializer: class org.apache.spark.serializer.JavaSerializer 21/01/17 17:42:55.382 Thread-3 INFO MemoryStore: MemoryStore started with capacity 366.3 MiB 21/01/17 17:42:55.387 Thread-3 INFO SparkEnv: Registering BlockManagerMaster 21/01/17 17:42:55.391 Thread-3 INFO SparkEnv: Registering MapOutputTracker 21/01/17 17:42:55.392 Thread-3 DEBUG MapOutputTrackerMasterEndpoint: init 21/01/17 17:42:55.395 Thread-3 INFO BlockManagerMasterEndpoint: Using org.apache.spark.storage.DefaultTopologyMapper for getting topology information 21/01/17 17:42:55.396 Thread-3 INFO BlockManagerMasterEndpoint: BlockManagerMasterEndpoint up 21/01/17 17:42:55.400 Thread-3 INFO SparkEnv: Registering BlockManagerMasterHeartbeat 21/01/17 17:42:55.402 Thread-3 INFO SparkEnv: Registering MapOutputTracker 21/01/17 17:42:55.403 Thread-3 DEBUG MapOutputTrackerMasterEndpoint: init 21/01/17 17:42:55.404 Thread-3 INFO SparkEnv: Registering OutputCommitCoordinator 21/01/17 17:42:55.404 Thread-3 DEBUG OutputCommitCoordinator$OutputCommitCoordinatorEndpoint: init 21/01/17 17:42:55.420 Thread-3 INFO DiskBlockManager: Created local directory at /home/jenkins/workspace/spark-master-test-python-arm/python/target/5853e670-f5ec-412d-a984-ccfa23493aff/blockmgr-c4fbffee-9a5c-454c-9c5f-2e460e4852c2 21/01/17 17:42:55.421 Thread-3 DEBUG DiskBlockManager: Adding shutdown hook 21/01/17 17:42:55.434 Thread-3 INFO SparkEnv: Registering BlockManagerMaster 21/01/17 17:42:55.442 Thread-3 INFO BlockManagerMasterEndpoint: Using org.apache.spark.storage.DefaultTopologyMapper for getting topology information 21/01/17 17:42:55.443 Thread-3 INFO BlockManagerMasterEndpoint: BlockManagerMasterEndpoint up 21/01/17 17:42:55.444 Thread-3 INFO SparkEnv: Registering BlockManagerMaster 21/01/17 17:42:55.452 Thread-3 INFO BlockManagerMasterEndpoint: Using org.apache.spark.storage.DefaultTopologyMapper for getting topology information 21/01/17 17:42:55.453 Thread-3 INFO BlockManagerMasterEndpoint: BlockManagerMasterEndpoint up 21/01/17 17:42:55.453 Thread-3 INFO SparkEnv: Registering BlockManagerMasterHeartbeat 21/01/17 17:42:55.458 Thread-3 INFO MemoryStore: MemoryStore started with capacity 366.3 MiB 21/01/17 17:42:55.467 Thread-3 INFO SparkEnv: Registering BlockManagerMasterHeartbeat 21/01/17 17:42:55.475 Thread-3 INFO DiskBlockManager: Created local directory at /home/jenkins/workspace/spark-master-test-python-arm/python/target/b3a5329f-5bcf-493d-9700-75740e75e18a/blockmgr-04a0f1df-fc64-4a41-97d2-017bd71decd1 21/01/17 17:42:55.476 Thread-3 DEBUG DiskBlockManager: Adding shutdown hook 21/01/17 17:42:55.482 Thread-3 INFO DiskBlockManager: Created local directory at /home/jenkins/workspace/spark-master-test-python-arm/python/target/f344877f-8415-4e7f-bb44-a4c9c9d59bb8/blockmgr-a6e647f6-49e4-4701-8ef7-9c10990055d3 21/01/17 17:42:55.483 Thread-3 DEBUG DiskBlockManager: Adding shutdown hook 21/01/17 17:42:55.490 Thread-3 INFO SparkEnv: Registering OutputCommitCoordinator 21/01/17 17:42:55.491 Thread-3 DEBUG OutputCommitCoordinator$OutputCommitCoordinatorEndpoint: init 21/01/17 17:42:55.513 Thread-3 INFO MemoryStore: MemoryStore started with capacity 366.3 MiB 21/01/17 17:42:55.520 Thread-3 INFO MemoryStore: MemoryStore started with capacity 366.3 MiB 21/01/17 17:42:55.542 Thread-3 INFO SparkEnv: Registering OutputCommitCoordinator 21/01/17 17:42:55.543 Thread-3 DEBUG OutputCommitCoordinator$OutputCommitCoordinatorEndpoint: init 21/01/17 17:42:55.557 Thread-3 INFO SparkEnv: Registering OutputCommitCoordinator 21/01/17 17:42:55.557 Thread-3 DEBUG OutputCommitCoordinator$OutputCommitCoordinatorEndpoint: init 21/01/17 17:42:55.664 Thread-3 INFO Executor: Starting executor ID driver on host 192.168.1.225 21/01/17 17:42:55.702 Thread-3 DEBUG TransportServer: Shuffle server started on port: 46295 21/01/17 17:42:55.703 Thread-3 INFO Utils: Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 46295. 21/01/17 17:42:55.703 Thread-3 INFO NettyBlockTransferService: Server created on 192.168.1.225:46295 21/01/17 17:42:55.705 Thread-3 INFO BlockManager: Using org.apache.spark.storage.RandomBlockReplicationPolicy for block replication policy 21/01/17 17:42:55.715 Thread-3 INFO BlockManagerMaster: Registering BlockManager BlockManagerId(driver, 192.168.1.225, 46295, None) 21/01/17 17:42:55.717 dispatcher-BlockManagerMaster DEBUG DefaultTopologyMapper: Got a request for 192.168.1.225 21/01/17 17:42:55.718 dispatcher-BlockManagerMaster INFO BlockManagerMasterEndpoint: Registering block manager 192.168.1.225:46295 with 366.3 MiB RAM, BlockManagerId(driver, 192.168.1.225, 46295, None) 21/01/17 17:42:55.723 Thread-3 INFO BlockManagerMaster: Registered BlockManager BlockManagerId(driver, 192.168.1.225, 46295, None) 21/01/17 17:42:55.724 Thread-3 INFO BlockManager: Initialized BlockManager: BlockManagerId(driver, 192.168.1.225, 46295, None) 21/01/17 17:42:55.749 Thread-3 INFO Executor: Starting executor ID driver on host 192.168.1.225 21/01/17 17:42:55.785 Thread-3 DEBUG TransportServer: Shuffle server started on port: 40985 21/01/17 17:42:55.785 Thread-3 INFO Utils: Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 40985. 21/01/17 17:42:55.785 Thread-3 INFO NettyBlockTransferService: Server created on 192.168.1.225:40985 21/01/17 17:42:55.787 Thread-3 INFO BlockManager: Using org.apache.spark.storage.RandomBlockReplicationPolicy for block replication policy 21/01/17 17:42:55.790 Thread-3 INFO Executor: Starting executor ID driver on host 192.168.1.225 21/01/17 17:42:55.793 Thread-3 INFO Executor: Starting executor ID driver on host 192.168.1.225 21/01/17 17:42:55.798 Thread-3 INFO BlockManagerMaster: Registering BlockManager BlockManagerId(driver, 192.168.1.225, 40985, None) 21/01/17 17:42:55.801 dispatcher-BlockManagerMaster DEBUG DefaultTopologyMapper: Got a request for 192.168.1.225 21/01/17 17:42:55.802 dispatcher-BlockManagerMaster INFO BlockManagerMasterEndpoint: Registering block manager 192.168.1.225:40985 with 366.3 MiB RAM, BlockManagerId(driver, 192.168.1.225, 40985, None) 21/01/17 17:42:55.807 Thread-3 INFO BlockManagerMaster: Registered BlockManager BlockManagerId(driver, 192.168.1.225, 40985, None) 21/01/17 17:42:55.810 Thread-3 INFO BlockManager: Initialized BlockManager: BlockManagerId(driver, 192.168.1.225, 40985, None) 21/01/17 17:42:55.829 Thread-3 DEBUG TransportServer: Shuffle server started on port: 33787 21/01/17 17:42:55.829 Thread-3 INFO Utils: Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 33787. 21/01/17 17:42:55.829 Thread-3 INFO NettyBlockTransferService: Server created on 192.168.1.225:33787 21/01/17 17:42:55.830 Thread-3 DEBUG TransportServer: Shuffle server started on port: 45289 21/01/17 17:42:55.830 Thread-3 INFO Utils: Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 45289. 21/01/17 17:42:55.830 Thread-3 INFO NettyBlockTransferService: Server created on 192.168.1.225:45289 21/01/17 17:42:55.831 Thread-3 INFO BlockManager: Using org.apache.spark.storage.RandomBlockReplicationPolicy for block replication policy 21/01/17 17:42:55.832 Thread-3 INFO BlockManager: Using org.apache.spark.storage.RandomBlockReplicationPolicy for block replication policy 21/01/17 17:42:55.841 Thread-3 INFO BlockManagerMaster: Registering BlockManager BlockManagerId(driver, 192.168.1.225, 33787, None) 21/01/17 17:42:55.842 Thread-3 INFO BlockManagerMaster: Registering BlockManager BlockManagerId(driver, 192.168.1.225, 45289, None) 21/01/17 17:42:55.844 dispatcher-BlockManagerMaster DEBUG DefaultTopologyMapper: Got a request for 192.168.1.225 21/01/17 17:42:55.844 dispatcher-BlockManagerMaster DEBUG DefaultTopologyMapper: Got a request for 192.168.1.225 21/01/17 17:42:55.845 dispatcher-BlockManagerMaster INFO BlockManagerMasterEndpoint: Registering block manager 192.168.1.225:33787 with 366.3 MiB RAM, BlockManagerId(driver, 192.168.1.225, 33787, None) 21/01/17 17:42:55.846 dispatcher-BlockManagerMaster INFO BlockManagerMasterEndpoint: Registering block manager 192.168.1.225:45289 with 366.3 MiB RAM, BlockManagerId(driver, 192.168.1.225, 45289, None) 21/01/17 17:42:55.849 Thread-3 INFO BlockManagerMaster: Registered BlockManager BlockManagerId(driver, 192.168.1.225, 33787, None) 21/01/17 17:42:55.850 Thread-3 INFO BlockManagerMaster: Registered BlockManager BlockManagerId(driver, 192.168.1.225, 45289, None) 21/01/17 17:42:55.851 Thread-3 INFO BlockManager: Initialized BlockManager: BlockManagerId(driver, 192.168.1.225, 33787, None) 21/01/17 17:42:55.852 Thread-3 INFO BlockManager: Initialized BlockManager: BlockManagerId(driver, 192.168.1.225, 45289, None) 21/01/17 17:42:56.134 Thread-3 DEBUG log: Logging to org.slf4j.impl.Log4jLoggerAdapter(org.eclipse.jetty.util.log) via org.eclipse.jetty.util.log.Slf4jLog 21/01/17 17:42:56.137 Thread-3 INFO log: Logging initialized @4405ms to org.eclipse.jetty.util.log.Slf4jLog 21/01/17 17:42:56.186 Thread-3 DEBUG DecoratedObjectFactory: Adding Decorator: org.eclipse.jetty.util.DeprecationWarning@15f32ec 21/01/17 17:42:56.198 Thread-3 DEBUG ContainerLifeCycle: o.e.j.s.ServletContextHandler@6b84deb8{/,null,STOPPED} added {ServletHandler@3ccb055d{STOPPED},MANAGED} 21/01/17 17:42:56.206 Thread-3 DEBUG ContainerLifeCycle: ServletHandler@3ccb055d{STOPPED} added {org.apache.spark.ui.JettyUtils$$anon$1-733ed85c==org.apache.spark.ui.JettyUtils$$anon$1@d05e793e{jsp=null,order=-1,inst=false,async=true,src=EMBEDDED:null},AUTO} 21/01/17 17:42:56.207 Thread-3 DEBUG ContainerLifeCycle: ServletHandler@3ccb055d{STOPPED} added {[/]=>org.apache.spark.ui.JettyUtils$$anon$1-733ed85c,POJO} 21/01/17 17:42:56.299 Thread-3 DEBUG log: Logging to org.slf4j.impl.Log4jLoggerAdapter(org.eclipse.jetty.util.log) via org.eclipse.jetty.util.log.Slf4jLog 21/01/17 17:42:56.301 Thread-3 INFO log: Logging initialized @4569ms to org.eclipse.jetty.util.log.Slf4jLog 21/01/17 17:42:56.330 Thread-3 DEBUG SparkContext: Adding shutdown hook 21/01/17 17:42:56.344 Thread-3 DEBUG log: Logging to org.slf4j.impl.Log4jLoggerAdapter(org.eclipse.jetty.util.log) via org.eclipse.jetty.util.log.Slf4jLog 21/01/17 17:42:56.347 Thread-3 INFO log: Logging initialized @4621ms to org.eclipse.jetty.util.log.Slf4jLog 21/01/17 17:42:56.347 Thread-3 DEBUG log: Logging to org.slf4j.impl.Log4jLoggerAdapter(org.eclipse.jetty.util.log) via org.eclipse.jetty.util.log.Slf4jLog 21/01/17 17:42:56.350 Thread-3 INFO log: Logging initialized @4630ms to org.eclipse.jetty.util.log.Slf4jLog 21/01/17 17:42:56.355 Thread-3 DEBUG DecoratedObjectFactory: Adding Decorator: org.eclipse.jetty.util.DeprecationWarning@5581a11d 21/01/17 17:42:56.368 Thread-3 DEBUG ContainerLifeCycle: o.e.j.s.ServletContextHandler@53171b6c{/,null,STOPPED} added {ServletHandler@3af0ef8d{STOPPED},MANAGED} 21/01/17 17:42:56.377 Thread-3 DEBUG ContainerLifeCycle: ServletHandler@3af0ef8d{STOPPED} added {org.apache.spark.ui.JettyUtils$$anon$1-2cf234bd==org.apache.spark.ui.JettyUtils$$anon$1@845de4e{jsp=null,order=-1,inst=false,async=true,src=EMBEDDED:null},AUTO} 21/01/17 17:42:56.384 Thread-3 DEBUG DecoratedObjectFactory: Adding Decorator: org.eclipse.jetty.util.DeprecationWarning@69ab8835 21/01/17 17:42:56.387 Thread-3 DEBUG ContainerLifeCycle: ServletHandler@3af0ef8d{STOPPED} added {[/]=>org.apache.spark.ui.JettyUtils$$anon$1-2cf234bd,POJO} 21/01/17 17:42:56.403 Thread-3 DEBUG ContainerLifeCycle: o.e.j.s.ServletContextHandler@393a5269{/,null,STOPPED} added {ServletHandler@437501{STOPPED},MANAGED} 21/01/17 17:42:56.411 Thread-3 DEBUG ContainerLifeCycle: ServletHandler@437501{STOPPED} added {org.apache.spark.ui.JettyUtils$$anon$1-72187ebd==org.apache.spark.ui.JettyUtils$$anon$1@9580a4c0{jsp=null,order=-1,inst=false,async=true,src=EMBEDDED:null},AUTO} 21/01/17 17:42:56.412 Thread-3 DEBUG ContainerLifeCycle: ServletHandler@437501{STOPPED} added {[/]=>org.apache.spark.ui.JettyUtils$$anon$1-72187ebd,POJO} 21/01/17 17:42:56.424 Thread-3 DEBUG DecoratedObjectFactory: Adding Decorator: org.eclipse.jetty.util.DeprecationWarning@26ad9bb6 21/01/17 17:42:56.440 Thread-3 DEBUG ContainerLifeCycle: o.e.j.s.ServletContextHandler@7d3d47bf{/,null,STOPPED} added {ServletHandler@35421bf9{STOPPED},MANAGED} 21/01/17 17:42:56.448 Thread-3 DEBUG ContainerLifeCycle: ServletHandler@35421bf9{STOPPED} added {org.apache.spark.ui.JettyUtils$$anon$1-3c7fb671==org.apache.spark.ui.JettyUtils$$anon$1@22e56bbb{jsp=null,order=-1,inst=false,async=true,src=EMBEDDED:null},AUTO} 21/01/17 17:42:56.449 Thread-3 DEBUG ContainerLifeCycle: ServletHandler@35421bf9{STOPPED} added {[/]=>org.apache.spark.ui.JettyUtils$$anon$1-3c7fb671,POJO} 21/01/17 17:42:56.492 Thread-3 DEBUG SparkContext: Adding shutdown hook 21/01/17 17:42:56.529 Thread-3 DEBUG SparkContext: Adding shutdown hook 21/01/17 17:42:56.611 Thread-3 DEBUG SparkContext: Adding shutdown hook 21/01/17 17:42:57.960 Thread-3 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$pythonToJava$1 21/01/17 17:42:57.986 Thread-3 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$pythonToJava$1) is now cleaned +++ 21/01/17 17:42:57.987 Thread-3 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$pythonToJava$1 21/01/17 17:42:57.987 Thread-3 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$pythonToJava$1 21/01/17 17:42:58.001 Thread-3 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$toJavaArray$1 21/01/17 17:42:58.011 Thread-3 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$toJavaArray$1) is now cleaned +++ 21/01/17 17:42:58.012 Thread-3 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$pythonToJava$1) is now cleaned +++ 21/01/17 17:42:58.012 Thread-3 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$pythonToJava$1) is now cleaned +++ 21/01/17 17:42:58.037 Thread-3 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$toJavaArray$1 21/01/17 17:42:58.040 Thread-3 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$toJavaArray$1 21/01/17 17:42:58.041 Thread-3 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$toJavaArray$1) is now cleaned +++ 21/01/17 17:42:58.043 Thread-3 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$toJavaArray$1) is now cleaned +++ 21/01/17 17:42:58.156 Thread-3 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$applySchemaToPythonRDD$1 21/01/17 17:42:58.175 Thread-3 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$applySchemaToPythonRDD$1) is now cleaned +++ 21/01/17 17:42:58.238 Thread-3 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$applySchemaToPythonRDD$1 21/01/17 17:42:58.253 Thread-3 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$applySchemaToPythonRDD$1 21/01/17 17:42:58.276 Thread-3 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$applySchemaToPythonRDD$1) is now cleaned +++ 21/01/17 17:42:58.280 Thread-3 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$applySchemaToPythonRDD$1) is now cleaned +++ 21/01/17 17:42:58.797 Thread-3 INFO SharedState: loading hive config file: java.io.BufferedInputStream@1bc9b6fe 21/01/17 17:42:58.833 Thread-3 INFO SharedState: loading hive config file: java.io.BufferedInputStream@3d84a3ce 21/01/17 17:42:58.883 Thread-3 INFO SharedState: loading hive config file: java.io.BufferedInputStream@72627afc 21/01/17 17:42:58.962 Thread-3 INFO SharedState: Setting hive.metastore.warehouse.dir ('null') to the value of spark.sql.warehouse.dir ('/home/jenkins/workspace/spark-master-test-python-arm/python/target/b3a5329f-5bcf-493d-9700-75740e75e18a/acc5ed01-ff87-4d6d-b7f9-a66f321186e1'). 21/01/17 17:42:58.962 Thread-3 INFO SharedState: Warehouse path is '/home/jenkins/workspace/spark-master-test-python-arm/python/target/b3a5329f-5bcf-493d-9700-75740e75e18a/acc5ed01-ff87-4d6d-b7f9-a66f321186e1'. 21/01/17 17:42:58.993 Thread-3 INFO SharedState: Setting hive.metastore.warehouse.dir ('null') to the value of spark.sql.warehouse.dir ('/home/jenkins/workspace/spark-master-test-python-arm/python/target/5853e670-f5ec-412d-a984-ccfa23493aff/e697c829-443e-4232-80ed-e91b3f1131c0'). 21/01/17 17:42:58.994 Thread-3 INFO SharedState: Warehouse path is '/home/jenkins/workspace/spark-master-test-python-arm/python/target/5853e670-f5ec-412d-a984-ccfa23493aff/e697c829-443e-4232-80ed-e91b3f1131c0'. 21/01/17 17:42:59.019 Thread-3 INFO SharedState: Setting hive.metastore.warehouse.dir ('null') to the value of spark.sql.warehouse.dir ('/home/jenkins/workspace/spark-master-test-python-arm/python/target/3dfd943d-5b3b-4196-84a7-4e3f79e40903/9fe49051-0e28-4902-8182-23cd4a46e858'). 21/01/17 17:42:59.020 Thread-3 INFO SharedState: Warehouse path is '/home/jenkins/workspace/spark-master-test-python-arm/python/target/3dfd943d-5b3b-4196-84a7-4e3f79e40903/9fe49051-0e28-4902-8182-23cd4a46e858'. 21/01/17 17:42:59.854 Thread-3 INFO SharedState: loading hive config file: java.io.BufferedInputStream@7486adfe 21/01/17 17:42:59.975 Thread-3 INFO SharedState: Setting hive.metastore.warehouse.dir ('null') to the value of spark.sql.warehouse.dir ('/home/jenkins/workspace/spark-master-test-python-arm/python/target/f344877f-8415-4e7f-bb44-a4c9c9d59bb8/ff8a52ca-4e27-4893-b801-e35b1b346e8f'). 21/01/17 17:42:59.976 Thread-3 INFO SharedState: Warehouse path is '/home/jenkins/workspace/spark-master-test-python-arm/python/target/f344877f-8415-4e7f-bb44-a4c9c9d59bb8/ff8a52ca-4e27-4893-b801-e35b1b346e8f'. 21/01/17 17:43:01.403 Thread-3 DEBUG CatalystSqlParser: Parsing command: double 21/01/17 17:43:01.805 Thread-3 DEBUG Analyzer$ResolveReferences: Resolving 'input to input#4 21/01/17 17:43:01.806 Thread-3 DEBUG Analyzer$ResolveReferences: Resolving 'output to output#7 21/01/17 17:43:02.257 Thread-3 DEBUG CountVectorizerModel: Input schema: {"type":"struct","fields":[{"name":"id","type":"long","nullable":true,"metadata":{}},{"name":"words","type":{"type":"array","elementType":"string","containsNull":true},"nullable":true,"metadata":{}},{"name":"expected","type":{"type":"udt","class":"org.apache.spark.ml.linalg.VectorUDT","pyClass":"pyspark.ml.linalg.VectorUDT","sqlType":{"type":"struct","fields":[{"name":"type","type":"byte","nullable":false,"metadata":{}},{"name":"size","type":"integer","nullable":true,"metadata":{}},{"name":"indices","type":{"type":"array","elementType":"integer","containsNull":false},"nullable":true,"metadata":{}},{"name":"values","type":{"type":"array","elementType":"double","containsNull":false},"nullable":true,"metadata":{}}]}},"nullable":true,"metadata":{}}]} 21/01/17 17:43:02.274 Thread-3 DEBUG CountVectorizerModel: Expected output schema: {"type":"struct","fields":[{"name":"id","type":"long","nullable":true,"metadata":{}},{"name":"words","type":{"type":"array","elementType":"string","containsNull":true},"nullable":true,"metadata":{}},{"name":"expected","type":{"type":"udt","class":"org.apache.spark.ml.linalg.VectorUDT","pyClass":"pyspark.ml.linalg.VectorUDT","sqlType":{"type":"struct","fields":[{"name":"type","type":"byte","nullable":false,"metadata":{}},{"name":"size","type":"integer","nullable":true,"metadata":{}},{"name":"indices","type":{"type":"array","elementType":"integer","containsNull":false},"nullable":true,"metadata":{}},{"name":"values","type":{"type":"array","elementType":"double","containsNull":false},"nullable":true,"metadata":{}}]}},"nullable":true,"metadata":{}},{"name":"features","type":{"type":"udt","class":"org.apache.spark.ml.linalg.VectorUDT","pyClass":"pyspark.ml.linalg.VectorUDT","sqlType":{"type":"struct","fields":[{"name":"type","type":"byte","nullable":false,"metadata":{}},{"name":"size","type":"integer","nullable":true,"metadata":{}},{"name":"indices","type":{"type":"array","elementType":"integer","containsNull":false},"nullable":true,"metadata":{}},{"name":"values","type":{"type":"array","elementType":"double","containsNull":false},"nullable":true,"metadata":{}}]}},"nullable":false,"metadata":{"ml_attr":{"attrs":{},"num_attrs":3}}}]} 21/01/17 17:43:02.323 Thread-3 INFO MemoryStore: Block broadcast_0 stored as values in memory (estimated size 256.0 B, free 366.3 MiB) 21/01/17 17:43:02.325 Thread-3 DEBUG BlockManager: Put block broadcast_0 locally took 36 ms 21/01/17 17:43:02.328 Thread-3 DEBUG BlockManager: Putting block broadcast_0 without replication took 38 ms 21/01/17 17:43:02.370 Thread-3 INFO MemoryStore: Block broadcast_0_piece0 stored as bytes in memory (estimated size 264.0 B, free 366.3 MiB) 21/01/17 17:43:02.372 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master broadcast_0_piece0 for BlockManagerId(driver, 192.168.1.225, 46295, None) 21/01/17 17:43:02.375 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added broadcast_0_piece0 in memory on 192.168.1.225:46295 (size: 264.0 B, free: 366.3 MiB) 21/01/17 17:43:02.376 Thread-3 DEBUG BlockManagerMaster: Updated info of block broadcast_0_piece0 21/01/17 17:43:02.377 Thread-3 DEBUG BlockManager: Told master about block broadcast_0_piece0 21/01/17 17:43:02.378 Thread-3 DEBUG BlockManager: Put block broadcast_0_piece0 locally took 9 ms 21/01/17 17:43:02.378 Thread-3 DEBUG BlockManager: Putting block broadcast_0_piece0 without replication took 9 ms 21/01/17 17:43:02.379 Thread-3 INFO SparkContext: Created broadcast 0 from broadcast at CountVectorizer.scala:309 21/01/17 17:43:02.542 Thread-3 DEBUG Analyzer$ResolveReferences: Resolving 'prediction to prediction#1 21/01/17 17:43:02.543 Thread-3 DEBUG Analyzer$ResolveReferences: Resolving 'features to features#0 21/01/17 17:43:02.609 Thread-3 DEBUG Analyzer$ResolveReferences: Resolving 'user to user#0L 21/01/17 17:43:02.610 Thread-3 DEBUG Analyzer$ResolveReferences: Resolving 'item to item#1L 21/01/17 17:43:02.611 Thread-3 DEBUG Analyzer$ResolveReferences: Resolving 'rating to rating#2 21/01/17 17:43:02.708 Thread-3 DEBUG Analyzer$ResolveReferences: Resolving 'features to features#4 21/01/17 17:43:02.824 Thread-3 DEBUG Analyzer$ResolveReferences: Resolving 'features to features#4 21/01/17 17:43:02.942 Thread-3 DEBUG Analyzer$ResolveReferences: Resolving 'words to words#1 21/01/17 17:43:03.066 Thread-3 DEBUG WholeStageCodegenExec: /* 001 */ public Object generate(Object[] references) { /* 002 */ return new GeneratedIteratorForCodegenStage2(references); /* 003 */ } /* 004 */ /* 005 */ // codegenStageId=2 /* 006 */ final class GeneratedIteratorForCodegenStage2 extends org.apache.spark.sql.execution.BufferedRowIterator { /* 007 */ private Object[] references; /* 008 */ private scala.collection.Iterator[] inputs; /* 009 */ private scala.collection.Iterator inputadapter_input_0; /* 010 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[] project_mutableStateArray_0 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[1]; /* 011 */ /* 012 */ public GeneratedIteratorForCodegenStage2(Object[] references) { /* 013 */ this.references = references; /* 014 */ } /* 015 */ /* 016 */ public void init(int index, scala.collection.Iterator[] inputs) { /* 017 */ partitionIndex = index; /* 018 */ this.inputs = inputs; /* 019 */ inputadapter_input_0 = inputs[0]; /* 020 */ project_mutableStateArray_0[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(2, 0); /* 021 */ /* 022 */ } /* 023 */ /* 024 */ protected void processNext() throws java.io.IOException { /* 025 */ while ( inputadapter_input_0.hasNext()) { /* 026 */ InternalRow inputadapter_row_0 = (InternalRow) inputadapter_input_0.next(); /* 027 */ /* 028 */ // common sub-expressions /* 029 */ /* 030 */ long inputadapter_value_0 = inputadapter_row_0.getLong(0); /* 031 */ boolean project_isNull_0 = false; /* 032 */ double project_value_0 = -1.0; /* 033 */ if (!false) { /* 034 */ project_value_0 = (double) inputadapter_value_0; /* 035 */ } /* 036 */ boolean inputadapter_isNull_1 = inputadapter_row_0.isNullAt(1); /* 037 */ double inputadapter_value_1 = inputadapter_isNull_1 ? /* 038 */ -1.0 : (inputadapter_row_0.getDouble(1)); /* 039 */ project_mutableStateArray_0[0].reset(); /* 040 */ /* 041 */ project_mutableStateArray_0[0].zeroOutNullBytes(); /* 042 */ /* 043 */ project_mutableStateArray_0[0].write(0, project_value_0); /* 044 */ /* 045 */ if (inputadapter_isNull_1) { /* 046 */ project_mutableStateArray_0[0].setNullAt(1); /* 047 */ } else { /* 048 */ project_mutableStateArray_0[0].write(1, inputadapter_value_1); /* 049 */ } /* 050 */ append((project_mutableStateArray_0[0].getRow())); /* 051 */ if (shouldStop()) return; /* 052 */ } /* 053 */ } /* 054 */ /* 055 */ } 21/01/17 17:43:03.117 Thread-3 DEBUG CodeGenerator: /* 001 */ public Object generate(Object[] references) { /* 002 */ return new GeneratedIteratorForCodegenStage2(references); /* 003 */ } /* 004 */ /* 005 */ // codegenStageId=2 /* 006 */ final class GeneratedIteratorForCodegenStage2 extends org.apache.spark.sql.execution.BufferedRowIterator { /* 007 */ private Object[] references; /* 008 */ private scala.collection.Iterator[] inputs; /* 009 */ private scala.collection.Iterator inputadapter_input_0; /* 010 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[] project_mutableStateArray_0 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[1]; /* 011 */ /* 012 */ public GeneratedIteratorForCodegenStage2(Object[] references) { /* 013 */ this.references = references; /* 014 */ } /* 015 */ /* 016 */ public void init(int index, scala.collection.Iterator[] inputs) { /* 017 */ partitionIndex = index; /* 018 */ this.inputs = inputs; /* 019 */ inputadapter_input_0 = inputs[0]; /* 020 */ project_mutableStateArray_0[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(2, 0); /* 021 */ /* 022 */ } /* 023 */ /* 024 */ protected void processNext() throws java.io.IOException { /* 025 */ while ( inputadapter_input_0.hasNext()) { /* 026 */ InternalRow inputadapter_row_0 = (InternalRow) inputadapter_input_0.next(); /* 027 */ /* 028 */ // common sub-expressions /* 029 */ /* 030 */ long inputadapter_value_0 = inputadapter_row_0.getLong(0); /* 031 */ boolean project_isNull_0 = false; /* 032 */ double project_value_0 = -1.0; /* 033 */ if (!false) { /* 034 */ project_value_0 = (double) inputadapter_value_0; /* 035 */ } /* 036 */ boolean inputadapter_isNull_1 = inputadapter_row_0.isNullAt(1); /* 037 */ double inputadapter_value_1 = inputadapter_isNull_1 ? /* 038 */ -1.0 : (inputadapter_row_0.getDouble(1)); /* 039 */ project_mutableStateArray_0[0].reset(); /* 040 */ /* 041 */ project_mutableStateArray_0[0].zeroOutNullBytes(); /* 042 */ /* 043 */ project_mutableStateArray_0[0].write(0, project_value_0); /* 044 */ /* 045 */ if (inputadapter_isNull_1) { /* 046 */ project_mutableStateArray_0[0].setNullAt(1); /* 047 */ } else { /* 048 */ project_mutableStateArray_0[0].write(1, inputadapter_value_1); /* 049 */ } /* 050 */ append((project_mutableStateArray_0[0].getRow())); /* 051 */ if (shouldStop()) return; /* 052 */ } /* 053 */ } /* 054 */ /* 055 */ } 21/01/17 17:43:03.134 Thread-3 DEBUG Analyzer$ResolveReferences: Resolving 'features to features#9 21/01/17 17:43:03.135 Thread-3 DEBUG Analyzer$ResolveReferences: Resolving 'expected to expected#2 21/01/17 17:43:03.434 Thread-3 INFO CodeGenerator: Code generated in 366.91564 ms 21/01/17 17:43:03.439 Thread-3 DEBUG WholeStageCodegenExec: /* 001 */ public Object generate(Object[] references) { /* 002 */ return new GeneratedIteratorForCodegenStage1(references); /* 003 */ } /* 004 */ /* 005 */ // codegenStageId=1 /* 006 */ final class GeneratedIteratorForCodegenStage1 extends org.apache.spark.sql.execution.BufferedRowIterator { /* 007 */ private Object[] references; /* 008 */ private scala.collection.Iterator[] inputs; /* 009 */ private scala.collection.Iterator rdd_input_0; /* 010 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[] rdd_mutableStateArray_0 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[2]; /* 011 */ /* 012 */ public GeneratedIteratorForCodegenStage1(Object[] references) { /* 013 */ this.references = references; /* 014 */ } /* 015 */ /* 016 */ public void init(int index, scala.collection.Iterator[] inputs) { /* 017 */ partitionIndex = index; /* 018 */ this.inputs = inputs; /* 019 */ rdd_input_0 = inputs[0]; /* 020 */ rdd_mutableStateArray_0[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(3, 0); /* 021 */ rdd_mutableStateArray_0[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(3, 0); /* 022 */ /* 023 */ } /* 024 */ /* 025 */ protected void processNext() throws java.io.IOException { /* 026 */ while ( rdd_input_0.hasNext()) { /* 027 */ InternalRow rdd_row_0 = (InternalRow) rdd_input_0.next(); /* 028 */ ((org.apache.spark.sql.execution.metric.SQLMetric) references[0] /* numOutputRows */).add(1); /* 029 */ // common sub-expressions /* 030 */ /* 031 */ boolean rdd_isNull_0 = rdd_row_0.isNullAt(0); /* 032 */ long rdd_value_0 = rdd_isNull_0 ? /* 033 */ -1L : (rdd_row_0.getLong(0)); /* 034 */ Object project_arg_0 = rdd_isNull_0 ? null : ((scala.Function1[]) references[1] /* converters */)[0].apply(rdd_value_0); /* 035 */ /* 036 */ Integer project_result_0 = null; /* 037 */ try { /* 038 */ project_result_0 = (Integer)((scala.Function1) references[3] /* udf */).apply(project_arg_0); /* 039 */ } catch (Exception e) { /* 040 */ throw new org.apache.spark.SparkException(((java.lang.String) references[2] /* errMsg */), e); /* 041 */ } /* 042 */ /* 043 */ boolean project_isNull_0 = project_result_0 == null; /* 044 */ int project_value_0 = -1; /* 045 */ if (!project_isNull_0) { /* 046 */ project_value_0 = project_result_0; /* 047 */ } /* 048 */ boolean rdd_isNull_1 = rdd_row_0.isNullAt(1); /* 049 */ long rdd_value_1 = rdd_isNull_1 ? /* 050 */ -1L : (rdd_row_0.getLong(1)); /* 051 */ Object project_arg_1 = rdd_isNull_1 ? null : ((scala.Function1[]) references[4] /* converters */)[0].apply(rdd_value_1); /* 052 */ /* 053 */ Integer project_result_1 = null; /* 054 */ try { /* 055 */ project_result_1 = (Integer)((scala.Function1) references[6] /* udf */).apply(project_arg_1); /* 056 */ } catch (Exception e) { /* 057 */ throw new org.apache.spark.SparkException(((java.lang.String) references[5] /* errMsg */), e); /* 058 */ } /* 059 */ /* 060 */ boolean project_isNull_2 = project_result_1 == null; /* 061 */ int project_value_2 = -1; /* 062 */ if (!project_isNull_2) { /* 063 */ project_value_2 = project_result_1; /* 064 */ } /* 065 */ boolean rdd_isNull_2 = rdd_row_0.isNullAt(2); /* 066 */ double rdd_value_2 = rdd_isNull_2 ? /* 067 */ -1.0 : (rdd_row_0.getDouble(2)); /* 068 */ boolean project_isNull_4 = rdd_isNull_2; /* 069 */ float project_value_4 = -1.0f; /* 070 */ if (!rdd_isNull_2) { /* 071 */ project_value_4 = (float) rdd_value_2; /* 072 */ } /* 073 */ rdd_mutableStateArray_0[1].reset(); /* 074 */ /* 075 */ rdd_mutableStateArray_0[1].zeroOutNullBytes(); /* 076 */ /* 077 */ rdd_mutableStateArray_0[1].write(0, project_value_0); /* 078 */ /* 079 */ rdd_mutableStateArray_0[1].write(1, project_value_2); /* 080 */ /* 081 */ if (project_isNull_4) { /* 082 */ rdd_mutableStateArray_0[1].setNullAt(2); /* 083 */ } else { /* 084 */ rdd_mutableStateArray_0[1].write(2, project_value_4); /* 085 */ } /* 086 */ append((rdd_mutableStateArray_0[1].getRow())); /* 087 */ if (shouldStop()) return; /* 088 */ } /* 089 */ } /* 090 */ /* 091 */ } 21/01/17 17:43:03.458 Thread-3 DEBUG WholeStageCodegenExec: /* 001 */ public Object generate(Object[] references) { /* 002 */ return new GeneratedIteratorForCodegenStage1(references); /* 003 */ } /* 004 */ /* 005 */ // codegenStageId=1 /* 006 */ final class GeneratedIteratorForCodegenStage1 extends org.apache.spark.sql.execution.BufferedRowIterator { /* 007 */ private Object[] references; /* 008 */ private scala.collection.Iterator[] inputs; /* 009 */ private boolean range_initRange_0; /* 010 */ private long range_nextIndex_0; /* 011 */ private TaskContext range_taskContext_0; /* 012 */ private InputMetrics range_inputMetrics_0; /* 013 */ private long range_batchEnd_0; /* 014 */ private long range_numElementsTodo_0; /* 015 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[] range_mutableStateArray_0 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[1]; /* 016 */ /* 017 */ public GeneratedIteratorForCodegenStage1(Object[] references) { /* 018 */ this.references = references; /* 019 */ } /* 020 */ /* 021 */ public void init(int index, scala.collection.Iterator[] inputs) { /* 022 */ partitionIndex = index; /* 023 */ this.inputs = inputs; /* 024 */ /* 025 */ range_taskContext_0 = TaskContext.get(); /* 026 */ range_inputMetrics_0 = range_taskContext_0.taskMetrics().inputMetrics(); /* 027 */ range_mutableStateArray_0[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(1, 0); /* 028 */ /* 029 */ } /* 030 */ /* 031 */ private void initRange(int idx) { /* 032 */ java.math.BigInteger index = java.math.BigInteger.valueOf(idx); /* 033 */ java.math.BigInteger numSlice = java.math.BigInteger.valueOf(4L); /* 034 */ java.math.BigInteger numElement = java.math.BigInteger.valueOf(10L); /* 035 */ java.math.BigInteger step = java.math.BigInteger.valueOf(1L); /* 036 */ java.math.BigInteger start = java.math.BigInteger.valueOf(0L); /* 037 */ long partitionEnd; /* 038 */ /* 039 */ java.math.BigInteger st = index.multiply(numElement).divide(numSlice).multiply(step).add(start); /* 040 */ if (st.compareTo(java.math.BigInteger.valueOf(Long.MAX_VALUE)) > 0) { /* 041 */ range_nextIndex_0 = Long.MAX_VALUE; /* 042 */ } else if (st.compareTo(java.math.BigInteger.valueOf(Long.MIN_VALUE)) < 0) { /* 043 */ range_nextIndex_0 = Long.MIN_VALUE; /* 044 */ } else { /* 045 */ range_nextIndex_0 = st.longValue(); /* 046 */ } /* 047 */ range_batchEnd_0 = range_nextIndex_0; /* 048 */ /* 049 */ java.math.BigInteger end = index.add(java.math.BigInteger.ONE).multiply(numElement).divide(numSlice) /* 050 */ .multiply(step).add(start); /* 051 */ if (end.compareTo(java.math.BigInteger.valueOf(Long.MAX_VALUE)) > 0) { /* 052 */ partitionEnd = Long.MAX_VALUE; /* 053 */ } else if (end.compareTo(java.math.BigInteger.valueOf(Long.MIN_VALUE)) < 0) { /* 054 */ partitionEnd = Long.MIN_VALUE; /* 055 */ } else { /* 056 */ partitionEnd = end.longValue(); /* 057 */ } /* 058 */ /* 059 */ java.math.BigInteger startToEnd = java.math.BigInteger.valueOf(partitionEnd).subtract( /* 060 */ java.math.BigInteger.valueOf(range_nextIndex_0)); /* 061 */ range_numElementsTodo_0 = startToEnd.divide(step).longValue(); /* 062 */ if (range_numElementsTodo_0 < 0) { /* 063 */ range_numElementsTodo_0 = 0; /* 064 */ } else if (startToEnd.remainder(step).compareTo(java.math.BigInteger.valueOf(0L)) != 0) { /* 065 */ range_numElementsTodo_0++; /* 066 */ } /* 067 */ } /* 068 */ /* 069 */ protected void processNext() throws java.io.IOException { /* 070 */ // initialize Range /* 071 */ if (!range_initRange_0) { /* 072 */ range_initRange_0 = true; /* 073 */ initRange(partitionIndex); /* 074 */ } /* 075 */ /* 076 */ while (true) { /* 077 */ if (range_nextIndex_0 == range_batchEnd_0) { /* 078 */ long range_nextBatchTodo_0; /* 079 */ if (range_numElementsTodo_0 > 1000L) { /* 080 */ range_nextBatchTodo_0 = 1000L; /* 081 */ range_numElementsTodo_0 -= 1000L; /* 082 */ } else { /* 083 */ range_nextBatchTodo_0 = range_numElementsTodo_0; /* 084 */ range_numElementsTodo_0 = 0; /* 085 */ if (range_nextBatchTodo_0 == 0) break; /* 086 */ } /* 087 */ range_batchEnd_0 += range_nextBatchTodo_0 * 1L; /* 088 */ } /* 089 */ /* 090 */ int range_localEnd_0 = (int)((range_batchEnd_0 - range_nextIndex_0) / 1L); /* 091 */ for (int range_localIdx_0 = 0; range_localIdx_0 < range_localEnd_0; range_localIdx_0++) { /* 092 */ long range_value_0 = ((long)range_localIdx_0 * 1L) + range_nextIndex_0; /* 093 */ /* 094 */ range_mutableStateArray_0[0].reset(); /* 095 */ /* 096 */ range_mutableStateArray_0[0].write(0, range_value_0); /* 097 */ append((range_mutableStateArray_0[0].getRow())); /* 098 */ /* 099 */ if (shouldStop()) { /* 100 */ range_nextIndex_0 = range_value_0 + 1L; /* 101 */ ((org.apache.spark.sql.execution.metric.SQLMetric) references[0] /* numOutputRows */).add(range_localIdx_0 + 1); /* 102 */ range_inputMetrics_0.incRecordsRead(range_localIdx_0 + 1); /* 103 */ return; /* 104 */ } /* 105 */ /* 106 */ } /* 107 */ range_nextIndex_0 = range_batchEnd_0; /* 108 */ ((org.apache.spark.sql.execution.metric.SQLMetric) references[0] /* numOutputRows */).add(range_localEnd_0); /* 109 */ range_inputMetrics_0.incRecordsRead(range_localEnd_0); /* 110 */ range_taskContext_0.killTaskIfInterrupted(); /* 111 */ } /* 112 */ } /* 113 */ /* 114 */ } 21/01/17 17:43:03.466 Thread-3 DEBUG CodeGenerator: /* 001 */ public Object generate(Object[] references) { /* 002 */ return new GeneratedIteratorForCodegenStage1(references); /* 003 */ } /* 004 */ /* 005 */ // codegenStageId=1 /* 006 */ final class GeneratedIteratorForCodegenStage1 extends org.apache.spark.sql.execution.BufferedRowIterator { /* 007 */ private Object[] references; /* 008 */ private scala.collection.Iterator[] inputs; /* 009 */ private boolean range_initRange_0; /* 010 */ private long range_nextIndex_0; /* 011 */ private TaskContext range_taskContext_0; /* 012 */ private InputMetrics range_inputMetrics_0; /* 013 */ private long range_batchEnd_0; /* 014 */ private long range_numElementsTodo_0; /* 015 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[] range_mutableStateArray_0 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[1]; /* 016 */ /* 017 */ public GeneratedIteratorForCodegenStage1(Object[] references) { /* 018 */ this.references = references; /* 019 */ } /* 020 */ /* 021 */ public void init(int index, scala.collection.Iterator[] inputs) { /* 022 */ partitionIndex = index; /* 023 */ this.inputs = inputs; /* 024 */ /* 025 */ range_taskContext_0 = TaskContext.get(); /* 026 */ range_inputMetrics_0 = range_taskContext_0.taskMetrics().inputMetrics(); /* 027 */ range_mutableStateArray_0[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(1, 0); /* 028 */ /* 029 */ } /* 030 */ /* 031 */ private void initRange(int idx) { /* 032 */ java.math.BigInteger index = java.math.BigInteger.valueOf(idx); /* 033 */ java.math.BigInteger numSlice = java.math.BigInteger.valueOf(4L); /* 034 */ java.math.BigInteger numElement = java.math.BigInteger.valueOf(10L); /* 035 */ java.math.BigInteger step = java.math.BigInteger.valueOf(1L); /* 036 */ java.math.BigInteger start = java.math.BigInteger.valueOf(0L); /* 037 */ long partitionEnd; /* 038 */ /* 039 */ java.math.BigInteger st = index.multiply(numElement).divide(numSlice).multiply(step).add(start); /* 040 */ if (st.compareTo(java.math.BigInteger.valueOf(Long.MAX_VALUE)) > 0) { /* 041 */ range_nextIndex_0 = Long.MAX_VALUE; /* 042 */ } else if (st.compareTo(java.math.BigInteger.valueOf(Long.MIN_VALUE)) < 0) { /* 043 */ range_nextIndex_0 = Long.MIN_VALUE; /* 044 */ } else { /* 045 */ range_nextIndex_0 = st.longValue(); /* 046 */ } /* 047 */ range_batchEnd_0 = range_nextIndex_0; /* 048 */ /* 049 */ java.math.BigInteger end = index.add(java.math.BigInteger.ONE).multiply(numElement).divide(numSlice) /* 050 */ .multiply(step).add(start); /* 051 */ if (end.compareTo(java.math.BigInteger.valueOf(Long.MAX_VALUE)) > 0) { /* 052 */ partitionEnd = Long.MAX_VALUE; /* 053 */ } else if (end.compareTo(java.math.BigInteger.valueOf(Long.MIN_VALUE)) < 0) { /* 054 */ partitionEnd = Long.MIN_VALUE; /* 055 */ } else { /* 056 */ partitionEnd = end.longValue(); /* 057 */ } /* 058 */ /* 059 */ java.math.BigInteger startToEnd = java.math.BigInteger.valueOf(partitionEnd).subtract( /* 060 */ java.math.BigInteger.valueOf(range_nextIndex_0)); /* 061 */ range_numElementsTodo_0 = startToEnd.divide(step).longValue(); /* 062 */ if (range_numElementsTodo_0 < 0) { /* 063 */ range_numElementsTodo_0 = 0; /* 064 */ } else if (startToEnd.remainder(step).compareTo(java.math.BigInteger.valueOf(0L)) != 0) { /* 065 */ range_numElementsTodo_0++; /* 066 */ } /* 067 */ } /* 068 */ /* 069 */ protected void processNext() throws java.io.IOException { /* 070 */ // initialize Range /* 071 */ if (!range_initRange_0) { /* 072 */ range_initRange_0 = true; /* 073 */ initRange(partitionIndex); /* 074 */ } /* 075 */ /* 076 */ while (true) { /* 077 */ if (range_nextIndex_0 == range_batchEnd_0) { /* 078 */ long range_nextBatchTodo_0; /* 079 */ if (range_numElementsTodo_0 > 1000L) { /* 080 */ range_nextBatchTodo_0 = 1000L; /* 081 */ range_numElementsTodo_0 -= 1000L; /* 082 */ } else { /* 083 */ range_nextBatchTodo_0 = range_numElementsTodo_0; /* 084 */ range_numElementsTodo_0 = 0; /* 085 */ if (range_nextBatchTodo_0 == 0) break; /* 086 */ } /* 087 */ range_batchEnd_0 += range_nextBatchTodo_0 * 1L; /* 088 */ } /* 089 */ /* 090 */ int range_localEnd_0 = (int)((range_batchEnd_0 - range_nextIndex_0) / 1L); /* 091 */ for (int range_localIdx_0 = 0; range_localIdx_0 < range_localEnd_0; range_localIdx_0++) { /* 092 */ long range_value_0 = ((long)range_localIdx_0 * 1L) + range_nextIndex_0; /* 093 */ /* 094 */ range_mutableStateArray_0[0].reset(); /* 095 */ /* 096 */ range_mutableStateArray_0[0].write(0, range_value_0); /* 097 */ append((range_mutableStateArray_0[0].getRow())); /* 098 */ /* 099 */ if (shouldStop()) { /* 100 */ range_nextIndex_0 = range_value_0 + 1L; /* 101 */ ((org.apache.spark.sql.execution.metric.SQLMetric) references[0] /* numOutputRows */).add(range_localIdx_0 + 1); /* 102 */ range_inputMetrics_0.incRecordsRead(range_localIdx_0 + 1); /* 103 */ return; /* 104 */ } /* 105 */ /* 106 */ } /* 107 */ range_nextIndex_0 = range_batchEnd_0; /* 108 */ ((org.apache.spark.sql.execution.metric.SQLMetric) references[0] /* numOutputRows */).add(range_localEnd_0); /* 109 */ range_inputMetrics_0.incRecordsRead(range_localEnd_0); /* 110 */ range_taskContext_0.killTaskIfInterrupted(); /* 111 */ } /* 112 */ } /* 113 */ /* 114 */ } 21/01/17 17:43:03.491 Thread-3 DEBUG CodeGenerator: /* 001 */ public Object generate(Object[] references) { /* 002 */ return new GeneratedIteratorForCodegenStage1(references); /* 003 */ } /* 004 */ /* 005 */ // codegenStageId=1 /* 006 */ final class GeneratedIteratorForCodegenStage1 extends org.apache.spark.sql.execution.BufferedRowIterator { /* 007 */ private Object[] references; /* 008 */ private scala.collection.Iterator[] inputs; /* 009 */ private scala.collection.Iterator rdd_input_0; /* 010 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[] rdd_mutableStateArray_0 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[2]; /* 011 */ /* 012 */ public GeneratedIteratorForCodegenStage1(Object[] references) { /* 013 */ this.references = references; /* 014 */ } /* 015 */ /* 016 */ public void init(int index, scala.collection.Iterator[] inputs) { /* 017 */ partitionIndex = index; /* 018 */ this.inputs = inputs; /* 019 */ rdd_input_0 = inputs[0]; /* 020 */ rdd_mutableStateArray_0[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(3, 0); /* 021 */ rdd_mutableStateArray_0[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(3, 0); /* 022 */ /* 023 */ } /* 024 */ /* 025 */ protected void processNext() throws java.io.IOException { /* 026 */ while ( rdd_input_0.hasNext()) { /* 027 */ InternalRow rdd_row_0 = (InternalRow) rdd_input_0.next(); /* 028 */ ((org.apache.spark.sql.execution.metric.SQLMetric) references[0] /* numOutputRows */).add(1); /* 029 */ // common sub-expressions /* 030 */ /* 031 */ boolean rdd_isNull_0 = rdd_row_0.isNullAt(0); /* 032 */ long rdd_value_0 = rdd_isNull_0 ? /* 033 */ -1L : (rdd_row_0.getLong(0)); /* 034 */ Object project_arg_0 = rdd_isNull_0 ? null : ((scala.Function1[]) references[1] /* converters */)[0].apply(rdd_value_0); /* 035 */ /* 036 */ Integer project_result_0 = null; /* 037 */ try { /* 038 */ project_result_0 = (Integer)((scala.Function1) references[3] /* udf */).apply(project_arg_0); /* 039 */ } catch (Exception e) { /* 040 */ throw new org.apache.spark.SparkException(((java.lang.String) references[2] /* errMsg */), e); /* 041 */ } /* 042 */ /* 043 */ boolean project_isNull_0 = project_result_0 == null; /* 044 */ int project_value_0 = -1; /* 045 */ if (!project_isNull_0) { /* 046 */ project_value_0 = project_result_0; /* 047 */ } /* 048 */ boolean rdd_isNull_1 = rdd_row_0.isNullAt(1); /* 049 */ long rdd_value_1 = rdd_isNull_1 ? /* 050 */ -1L : (rdd_row_0.getLong(1)); /* 051 */ Object project_arg_1 = rdd_isNull_1 ? null : ((scala.Function1[]) references[4] /* converters */)[0].apply(rdd_value_1); /* 052 */ /* 053 */ Integer project_result_1 = null; /* 054 */ try { /* 055 */ project_result_1 = (Integer)((scala.Function1) references[6] /* udf */).apply(project_arg_1); /* 056 */ } catch (Exception e) { /* 057 */ throw new org.apache.spark.SparkException(((java.lang.String) references[5] /* errMsg */), e); /* 058 */ } /* 059 */ /* 060 */ boolean project_isNull_2 = project_result_1 == null; /* 061 */ int project_value_2 = -1; /* 062 */ if (!project_isNull_2) { /* 063 */ project_value_2 = project_result_1; /* 064 */ } /* 065 */ boolean rdd_isNull_2 = rdd_row_0.isNullAt(2); /* 066 */ double rdd_value_2 = rdd_isNull_2 ? /* 067 */ -1.0 : (rdd_row_0.getDouble(2)); /* 068 */ boolean project_isNull_4 = rdd_isNull_2; /* 069 */ float project_value_4 = -1.0f; /* 070 */ if (!rdd_isNull_2) { /* 071 */ project_value_4 = (float) rdd_value_2; /* 072 */ } /* 073 */ rdd_mutableStateArray_0[1].reset(); /* 074 */ /* 075 */ rdd_mutableStateArray_0[1].zeroOutNullBytes(); /* 076 */ /* 077 */ rdd_mutableStateArray_0[1].write(0, project_value_0); /* 078 */ /* 079 */ rdd_mutableStateArray_0[1].write(1, project_value_2); /* 080 */ /* 081 */ if (project_isNull_4) { /* 082 */ rdd_mutableStateArray_0[1].setNullAt(2); /* 083 */ } else { /* 084 */ rdd_mutableStateArray_0[1].write(2, project_value_4); /* 085 */ } /* 086 */ append((rdd_mutableStateArray_0[1].getRow())); /* 087 */ if (shouldStop()) return; /* 088 */ } /* 089 */ } /* 090 */ /* 091 */ } 21/01/17 17:43:03.513 Thread-3 INFO CodeGenerator: Code generated in 54.809781 ms 21/01/17 17:43:03.546 Thread-3 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$inputRDDs$1$adapted 21/01/17 17:43:03.567 Thread-3 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$inputRDDs$1$adapted) is now cleaned +++ 21/01/17 17:43:03.586 Thread-3 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$doExecute$4$adapted 21/01/17 17:43:03.595 Thread-3 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$doExecute$4$adapted) is now cleaned +++ 21/01/17 17:43:03.616 Thread-3 DEBUG WholeStageCodegenExec: /* 001 */ public Object generate(Object[] references) { /* 002 */ return new GeneratedIteratorForCodegenStage1(references); /* 003 */ } /* 004 */ /* 005 */ // codegenStageId=1 /* 006 */ final class GeneratedIteratorForCodegenStage1 extends org.apache.spark.sql.execution.BufferedRowIterator { /* 007 */ private Object[] references; /* 008 */ private scala.collection.Iterator[] inputs; /* 009 */ private scala.collection.Iterator rdd_input_0; /* 010 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[] rdd_mutableStateArray_0 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[4]; /* 011 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[] rdd_mutableStateArray_1 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[4]; /* 012 */ /* 013 */ public GeneratedIteratorForCodegenStage1(Object[] references) { /* 014 */ this.references = references; /* 015 */ } /* 016 */ /* 017 */ public void init(int index, scala.collection.Iterator[] inputs) { /* 018 */ partitionIndex = index; /* 019 */ this.inputs = inputs; /* 020 */ rdd_input_0 = inputs[0]; /* 021 */ rdd_mutableStateArray_0[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(2, 32); /* 022 */ rdd_mutableStateArray_0[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(rdd_mutableStateArray_0[0], 4); /* 023 */ rdd_mutableStateArray_1[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[1], 4); /* 024 */ rdd_mutableStateArray_1[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[1], 8); /* 025 */ rdd_mutableStateArray_0[2] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(1, 32); /* 026 */ rdd_mutableStateArray_0[3] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(rdd_mutableStateArray_0[2], 4); /* 027 */ rdd_mutableStateArray_1[2] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[3], 4); /* 028 */ rdd_mutableStateArray_1[3] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[3], 8); /* 029 */ /* 030 */ } /* 031 */ /* 032 */ private void project_writeFields_0_1(InternalRow project_tmpInput_0) { /* 033 */ if ((project_tmpInput_0.isNullAt(3))) { /* 034 */ rdd_mutableStateArray_0[3].setNullAt(3); /* 035 */ } else { /* 036 */ // Remember the current cursor so that we can calculate how many bytes are /* 037 */ // written later. /* 038 */ final int project_previousCursor_2 = rdd_mutableStateArray_0[3].cursor(); /* 039 */ /* 040 */ final ArrayData project_tmpInput_2 = (project_tmpInput_0.getArray(3)); /* 041 */ if (project_tmpInput_2 instanceof UnsafeArrayData) { /* 042 */ rdd_mutableStateArray_0[3].write((UnsafeArrayData) project_tmpInput_2); /* 043 */ } else { /* 044 */ final int project_numElements_1 = project_tmpInput_2.numElements(); /* 045 */ rdd_mutableStateArray_1[3].initialize(project_numElements_1); /* 046 */ /* 047 */ for (int project_index_1 = 0; project_index_1 < project_numElements_1; project_index_1++) { /* 048 */ rdd_mutableStateArray_1[3].write(project_index_1, project_tmpInput_2.getDouble(project_index_1)); /* 049 */ } /* 050 */ } /* 051 */ /* 052 */ rdd_mutableStateArray_0[3].setOffsetAndSizeFromPreviousCursor(3, project_previousCursor_2); /* 053 */ } /* 054 */ /* 055 */ } /* 056 */ /* 057 */ private void project_writeFields_0_0(InternalRow project_tmpInput_0) { /* 058 */ rdd_mutableStateArray_0[3].write(0, (project_tmpInput_0.getByte(0))); /* 059 */ /* 060 */ if ((project_tmpInput_0.isNullAt(1))) { /* 061 */ rdd_mutableStateArray_0[3].setNullAt(1); /* 062 */ } else { /* 063 */ rdd_mutableStateArray_0[3].write(1, (project_tmpInput_0.getInt(1))); /* 064 */ } /* 065 */ /* 066 */ if ((project_tmpInput_0.isNullAt(2))) { /* 067 */ rdd_mutableStateArray_0[3].setNullAt(2); /* 068 */ } else { /* 069 */ // Remember the current cursor so that we can calculate how many bytes are /* 070 */ // written later. /* 071 */ final int project_previousCursor_1 = rdd_mutableStateArray_0[3].cursor(); /* 072 */ /* 073 */ final ArrayData project_tmpInput_1 = (project_tmpInput_0.getArray(2)); /* 074 */ if (project_tmpInput_1 instanceof UnsafeArrayData) { /* 075 */ rdd_mutableStateArray_0[3].write((UnsafeArrayData) project_tmpInput_1); /* 076 */ } else { /* 077 */ final int project_numElements_0 = project_tmpInput_1.numElements(); /* 078 */ rdd_mutableStateArray_1[2].initialize(project_numElements_0); /* 079 */ /* 080 */ for (int project_index_0 = 0; project_index_0 < project_numElements_0; project_index_0++) { /* 081 */ rdd_mutableStateArray_1[2].write(project_index_0, project_tmpInput_1.getInt(project_index_0)); /* 082 */ } /* 083 */ } /* 084 */ /* 085 */ rdd_mutableStateArray_0[3].setOffsetAndSizeFromPreviousCursor(2, project_previousCursor_1); /* 086 */ } /* 087 */ /* 088 */ } /* 089 */ /* 090 */ protected void processNext() throws java.io.IOException { /* 091 */ while ( rdd_input_0.hasNext()) { /* 092 */ InternalRow rdd_row_0 = (InternalRow) rdd_input_0.next(); /* 093 */ ((org.apache.spark.sql.execution.metric.SQLMetric) references[0] /* numOutputRows */).add(1); /* 094 */ // common sub-expressions /* 095 */ /* 096 */ boolean rdd_isNull_0 = rdd_row_0.isNullAt(0); /* 097 */ InternalRow rdd_value_0 = rdd_isNull_0 ? /* 098 */ null : (rdd_row_0.getStruct(0, 4)); /* 099 */ rdd_mutableStateArray_0[2].reset(); /* 100 */ /* 101 */ rdd_mutableStateArray_0[2].zeroOutNullBytes(); /* 102 */ /* 103 */ if (rdd_isNull_0) { /* 104 */ rdd_mutableStateArray_0[2].setNullAt(0); /* 105 */ } else { /* 106 */ final InternalRow project_tmpInput_0 = rdd_value_0; /* 107 */ if (project_tmpInput_0 instanceof UnsafeRow) { /* 108 */ rdd_mutableStateArray_0[2].write(0, (UnsafeRow) project_tmpInput_0); /* 109 */ } else { /* 110 */ // Remember the current cursor so that we can calculate how many bytes are /* 111 */ // written later. /* 112 */ final int project_previousCursor_0 = rdd_mutableStateArray_0[2].cursor(); /* 113 */ /* 114 */ rdd_mutableStateArray_0[3].resetRowWriter(); /* 115 */ project_writeFields_0_0(project_tmpInput_0); /* 116 */ project_writeFields_0_1(project_tmpInput_0); /* 117 */ /* 118 */ rdd_mutableStateArray_0[2].setOffsetAndSizeFromPreviousCursor(0, project_previousCursor_0); /* 119 */ } /* 120 */ } /* 121 */ append((rdd_mutableStateArray_0[2].getRow())); /* 122 */ if (shouldStop()) return; /* 123 */ } /* 124 */ } /* 125 */ /* 126 */ } 21/01/17 17:43:03.642 Thread-3 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$doExecute$1 21/01/17 17:43:03.654 Thread-3 DEBUG CodeGenerator: /* 001 */ public Object generate(Object[] references) { /* 002 */ return new GeneratedIteratorForCodegenStage1(references); /* 003 */ } /* 004 */ /* 005 */ // codegenStageId=1 /* 006 */ final class GeneratedIteratorForCodegenStage1 extends org.apache.spark.sql.execution.BufferedRowIterator { /* 007 */ private Object[] references; /* 008 */ private scala.collection.Iterator[] inputs; /* 009 */ private scala.collection.Iterator rdd_input_0; /* 010 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[] rdd_mutableStateArray_0 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[4]; /* 011 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[] rdd_mutableStateArray_1 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[4]; /* 012 */ /* 013 */ public GeneratedIteratorForCodegenStage1(Object[] references) { /* 014 */ this.references = references; /* 015 */ } /* 016 */ /* 017 */ public void init(int index, scala.collection.Iterator[] inputs) { /* 018 */ partitionIndex = index; /* 019 */ this.inputs = inputs; /* 020 */ rdd_input_0 = inputs[0]; /* 021 */ rdd_mutableStateArray_0[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(2, 32); /* 022 */ rdd_mutableStateArray_0[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(rdd_mutableStateArray_0[0], 4); /* 023 */ rdd_mutableStateArray_1[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[1], 4); /* 024 */ rdd_mutableStateArray_1[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[1], 8); /* 025 */ rdd_mutableStateArray_0[2] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(1, 32); /* 026 */ rdd_mutableStateArray_0[3] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(rdd_mutableStateArray_0[2], 4); /* 027 */ rdd_mutableStateArray_1[2] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[3], 4); /* 028 */ rdd_mutableStateArray_1[3] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[3], 8); /* 029 */ /* 030 */ } /* 031 */ /* 032 */ private void project_writeFields_0_1(InternalRow project_tmpInput_0) { /* 033 */ if ((project_tmpInput_0.isNullAt(3))) { /* 034 */ rdd_mutableStateArray_0[3].setNullAt(3); /* 035 */ } else { /* 036 */ // Remember the current cursor so that we can calculate how many bytes are /* 037 */ // written later. /* 038 */ final int project_previousCursor_2 = rdd_mutableStateArray_0[3].cursor(); /* 039 */ /* 040 */ final ArrayData project_tmpInput_2 = (project_tmpInput_0.getArray(3)); /* 041 */ if (project_tmpInput_2 instanceof UnsafeArrayData) { /* 042 */ rdd_mutableStateArray_0[3].write((UnsafeArrayData) project_tmpInput_2); /* 043 */ } else { /* 044 */ final int project_numElements_1 = project_tmpInput_2.numElements(); /* 045 */ rdd_mutableStateArray_1[3].initialize(project_numElements_1); /* 046 */ /* 047 */ for (int project_index_1 = 0; project_index_1 < project_numElements_1; project_index_1++) { /* 048 */ rdd_mutableStateArray_1[3].write(project_index_1, project_tmpInput_2.getDouble(project_index_1)); /* 049 */ } /* 050 */ } /* 051 */ /* 052 */ rdd_mutableStateArray_0[3].setOffsetAndSizeFromPreviousCursor(3, project_previousCursor_2); /* 053 */ } /* 054 */ /* 055 */ } /* 056 */ /* 057 */ private void project_writeFields_0_0(InternalRow project_tmpInput_0) { /* 058 */ rdd_mutableStateArray_0[3].write(0, (project_tmpInput_0.getByte(0))); /* 059 */ /* 060 */ if ((project_tmpInput_0.isNullAt(1))) { /* 061 */ rdd_mutableStateArray_0[3].setNullAt(1); /* 062 */ } else { /* 063 */ rdd_mutableStateArray_0[3].write(1, (project_tmpInput_0.getInt(1))); /* 064 */ } /* 065 */ /* 066 */ if ((project_tmpInput_0.isNullAt(2))) { /* 067 */ rdd_mutableStateArray_0[3].setNullAt(2); /* 068 */ } else { /* 069 */ // Remember the current cursor so that we can calculate how many bytes are /* 070 */ // written later. /* 071 */ final int project_previousCursor_1 = rdd_mutableStateArray_0[3].cursor(); /* 072 */ /* 073 */ final ArrayData project_tmpInput_1 = (project_tmpInput_0.getArray(2)); /* 074 */ if (project_tmpInput_1 instanceof UnsafeArrayData) { /* 075 */ rdd_mutableStateArray_0[3].write((UnsafeArrayData) project_tmpInput_1); /* 076 */ } else { /* 077 */ final int project_numElements_0 = project_tmpInput_1.numElements(); /* 078 */ rdd_mutableStateArray_1[2].initialize(project_numElements_0); /* 079 */ /* 080 */ for (int project_index_0 = 0; project_index_0 < project_numElements_0; project_index_0++) { /* 081 */ rdd_mutableStateArray_1[2].write(project_index_0, project_tmpInput_1.getInt(project_index_0)); /* 082 */ } /* 083 */ } /* 084 */ /* 085 */ rdd_mutableStateArray_0[3].setOffsetAndSizeFromPreviousCursor(2, project_previousCursor_1); /* 086 */ } /* 087 */ /* 088 */ } /* 089 */ /* 090 */ protected void processNext() throws java.io.IOException { /* 091 */ while ( rdd_input_0.hasNext()) { /* 092 */ InternalRow rdd_row_0 = (InternalRow) rdd_input_0.next(); /* 093 */ ((org.apache.spark.sql.execution.metric.SQLMetric) references[0] /* numOutputRows */).add(1); /* 094 */ // common sub-expressions /* 095 */ /* 096 */ boolean rdd_isNull_0 = rdd_row_0.isNullAt(0); /* 097 */ InternalRow rdd_value_0 = rdd_isNull_0 ? /* 098 */ null : (rdd_row_0.getStruct(0, 4)); /* 099 */ rdd_mutableStateArray_0[2].reset(); /* 100 */ /* 101 */ rdd_mutableStateArray_0[2].zeroOutNullBytes(); /* 102 */ /* 103 */ if (rdd_isNull_0) { /* 104 */ rdd_mutableStateArray_0[2].setNullAt(0); /* 105 */ } else { /* 106 */ final InternalRow project_tmpInput_0 = rdd_value_0; /* 107 */ if (project_tmpInput_0 instanceof UnsafeRow) { /* 108 */ rdd_mutableStateArray_0[2].write(0, (UnsafeRow) project_tmpInput_0); /* 109 */ } else { /* 110 */ // Remember the current cursor so that we can calculate how many bytes are /* 111 */ // written later. /* 112 */ final int project_previousCursor_0 = rdd_mutableStateArray_0[2].cursor(); /* 113 */ /* 114 */ rdd_mutableStateArray_0[3].resetRowWriter(); /* 115 */ project_writeFields_0_0(project_tmpInput_0); /* 116 */ project_writeFields_0_1(project_tmpInput_0); /* 117 */ /* 118 */ rdd_mutableStateArray_0[2].setOffsetAndSizeFromPreviousCursor(0, project_previousCursor_0); /* 119 */ } /* 120 */ } /* 121 */ append((rdd_mutableStateArray_0[2].getRow())); /* 122 */ if (shouldStop()) return; /* 123 */ } /* 124 */ } /* 125 */ /* 126 */ } 21/01/17 17:43:03.667 Thread-3 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$doExecute$1) is now cleaned +++ 21/01/17 17:43:03.671 Thread-3 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$doExecute$2 21/01/17 17:43:03.672 Thread-3 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$doExecute$2) is now cleaned +++ 21/01/17 17:43:03.748 Thread-3 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$doExecute$4$adapted 21/01/17 17:43:03.750 Thread-3 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$doExecute$4$adapted) is now cleaned +++ 21/01/17 17:43:03.782 Thread-3 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$collect$2 21/01/17 17:43:03.798 Thread-3 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$collect$2) is now cleaned +++ 21/01/17 17:43:03.861 Thread-3 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$runJob$5 21/01/17 17:43:03.866 Thread-3 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$runJob$5) is now cleaned +++ 21/01/17 17:43:03.871 Thread-3 INFO SparkContext: Starting job: at /usr/lib/python3.6/runpy.py:85 21/01/17 17:43:03.872 Thread-3 INFO CodeGenerator: Code generated in 432.109871 ms 21/01/17 17:43:03.874 Thread-3 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$doExecute$4$adapted 21/01/17 17:43:03.878 Thread-3 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$doExecute$4$adapted) is now cleaned +++ 21/01/17 17:43:03.885 dag-scheduler-event-loop DEBUG DAGScheduler: Merging stage rdd profiles: Set() 21/01/17 17:43:03.928 dag-scheduler-event-loop INFO DAGScheduler: Got job 0 ( at /usr/lib/python3.6/runpy.py:85) with 4 output partitions 21/01/17 17:43:03.934 dag-scheduler-event-loop INFO DAGScheduler: Final stage: ResultStage 0 ( at /usr/lib/python3.6/runpy.py:85) 21/01/17 17:43:03.935 dag-scheduler-event-loop INFO DAGScheduler: Parents of final stage: List() 21/01/17 17:43:03.937 dag-scheduler-event-loop INFO DAGScheduler: Missing parents: List() 21/01/17 17:43:03.949 dag-scheduler-event-loop DEBUG DAGScheduler: submitStage(ResultStage 0 (name= at /usr/lib/python3.6/runpy.py:85;jobs=0)) 21/01/17 17:43:03.952 dag-scheduler-event-loop DEBUG DAGScheduler: missing: List() 21/01/17 17:43:03.956 dag-scheduler-event-loop INFO DAGScheduler: Submitting ResultStage 0 (MapPartitionsRDD[6] at at /usr/lib/python3.6/runpy.py:85), which has no missing parents 21/01/17 17:43:03.956 dag-scheduler-event-loop DEBUG DAGScheduler: submitMissingTasks(ResultStage 0) 21/01/17 17:43:04.013 Thread-3 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$rdd$1 21/01/17 17:43:04.018 Thread-3 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$rdd$1) is now cleaned +++ 21/01/17 17:43:04.040 Thread-3 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$fit$2 21/01/17 17:43:04.040 Thread-3 INFO CodeGenerator: Code generated in 422.548956 ms 21/01/17 17:43:04.041 Thread-3 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$fit$2) is now cleaned +++ 21/01/17 17:43:04.042 Thread-3 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$doExecute$4$adapted 21/01/17 17:43:04.043 Thread-3 INFO Instrumentation: [51e91e47] Stage class: ALS 21/01/17 17:43:04.044 Thread-3 INFO Instrumentation: [51e91e47] Stage uid: ALS_6b34a42ac296 21/01/17 17:43:04.045 Thread-3 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$doExecute$4$adapted) is now cleaned +++ 21/01/17 17:43:04.066 Thread-3 DEBUG WholeStageCodegenExec: /* 001 */ public Object generate(Object[] references) { /* 002 */ return new GeneratedIteratorForCodegenStage1(references); /* 003 */ } /* 004 */ /* 005 */ // codegenStageId=1 /* 006 */ final class GeneratedIteratorForCodegenStage1 extends org.apache.spark.sql.execution.BufferedRowIterator { /* 007 */ private Object[] references; /* 008 */ private scala.collection.Iterator[] inputs; /* 009 */ private scala.collection.Iterator rdd_input_0; /* 010 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[] rdd_mutableStateArray_0 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[5]; /* 011 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[] rdd_mutableStateArray_1 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[7]; /* 012 */ /* 013 */ public GeneratedIteratorForCodegenStage1(Object[] references) { /* 014 */ this.references = references; /* 015 */ } /* 016 */ /* 017 */ public void init(int index, scala.collection.Iterator[] inputs) { /* 018 */ partitionIndex = index; /* 019 */ this.inputs = inputs; /* 020 */ wholestagecodegen_init_0_0(); /* 021 */ wholestagecodegen_init_0_1(); /* 022 */ /* 023 */ } /* 024 */ /* 025 */ private void project_writeFields_1_1(InternalRow project_tmpInput_3) { /* 026 */ if ((project_tmpInput_3.isNullAt(3))) { /* 027 */ rdd_mutableStateArray_0[4].setNullAt(3); /* 028 */ } else { /* 029 */ // Remember the current cursor so that we can calculate how many bytes are /* 030 */ // written later. /* 031 */ final int project_previousCursor_5 = rdd_mutableStateArray_0[4].cursor(); /* 032 */ /* 033 */ final ArrayData project_tmpInput_5 = (project_tmpInput_3.getArray(3)); /* 034 */ if (project_tmpInput_5 instanceof UnsafeArrayData) { /* 035 */ rdd_mutableStateArray_0[4].write((UnsafeArrayData) project_tmpInput_5); /* 036 */ } else { /* 037 */ final int project_numElements_3 = project_tmpInput_5.numElements(); /* 038 */ rdd_mutableStateArray_1[6].initialize(project_numElements_3); /* 039 */ /* 040 */ for (int project_index_3 = 0; project_index_3 < project_numElements_3; project_index_3++) { /* 041 */ rdd_mutableStateArray_1[6].write(project_index_3, project_tmpInput_5.getDouble(project_index_3)); /* 042 */ } /* 043 */ } /* 044 */ /* 045 */ rdd_mutableStateArray_0[4].setOffsetAndSizeFromPreviousCursor(3, project_previousCursor_5); /* 046 */ } /* 047 */ /* 048 */ } /* 049 */ /* 050 */ private void project_writeFields_1_0(InternalRow project_tmpInput_3) { /* 051 */ rdd_mutableStateArray_0[4].write(0, (project_tmpInput_3.getByte(0))); /* 052 */ /* 053 */ if ((project_tmpInput_3.isNullAt(1))) { /* 054 */ rdd_mutableStateArray_0[4].setNullAt(1); /* 055 */ } else { /* 056 */ rdd_mutableStateArray_0[4].write(1, (project_tmpInput_3.getInt(1))); /* 057 */ } /* 058 */ /* 059 */ if ((project_tmpInput_3.isNullAt(2))) { /* 060 */ rdd_mutableStateArray_0[4].setNullAt(2); /* 061 */ } else { /* 062 */ // Remember the current cursor so that we can calculate how many bytes are /* 063 */ // written later. /* 064 */ final int project_previousCursor_4 = rdd_mutableStateArray_0[4].cursor(); /* 065 */ /* 066 */ final ArrayData project_tmpInput_4 = (project_tmpInput_3.getArray(2)); /* 067 */ if (project_tmpInput_4 instanceof UnsafeArrayData) { /* 068 */ rdd_mutableStateArray_0[4].write((UnsafeArrayData) project_tmpInput_4); /* 069 */ } else { /* 070 */ final int project_numElements_2 = project_tmpInput_4.numElements(); /* 071 */ rdd_mutableStateArray_1[5].initialize(project_numElements_2); /* 072 */ /* 073 */ for (int project_index_2 = 0; project_index_2 < project_numElements_2; project_index_2++) { /* 074 */ rdd_mutableStateArray_1[5].write(project_index_2, project_tmpInput_4.getInt(project_index_2)); /* 075 */ } /* 076 */ } /* 077 */ /* 078 */ rdd_mutableStateArray_0[4].setOffsetAndSizeFromPreviousCursor(2, project_previousCursor_4); /* 079 */ } /* 080 */ /* 081 */ } /* 082 */ /* 083 */ private void project_writeFields_0_1(InternalRow project_tmpInput_0) { /* 084 */ if ((project_tmpInput_0.isNullAt(3))) { /* 085 */ rdd_mutableStateArray_0[3].setNullAt(3); /* 086 */ } else { /* 087 */ // Remember the current cursor so that we can calculate how many bytes are /* 088 */ // written later. /* 089 */ final int project_previousCursor_2 = rdd_mutableStateArray_0[3].cursor(); /* 090 */ /* 091 */ final ArrayData project_tmpInput_2 = (project_tmpInput_0.getArray(3)); /* 092 */ if (project_tmpInput_2 instanceof UnsafeArrayData) { /* 093 */ rdd_mutableStateArray_0[3].write((UnsafeArrayData) project_tmpInput_2); /* 094 */ } else { /* 095 */ final int project_numElements_1 = project_tmpInput_2.numElements(); /* 096 */ rdd_mutableStateArray_1[4].initialize(project_numElements_1); /* 097 */ /* 098 */ for (int project_index_1 = 0; project_index_1 < project_numElements_1; project_index_1++) { /* 099 */ rdd_mutableStateArray_1[4].write(project_index_1, project_tmpInput_2.getDouble(project_index_1)); /* 100 */ } /* 101 */ } /* 102 */ /* 103 */ rdd_mutableStateArray_0[3].setOffsetAndSizeFromPreviousCursor(3, project_previousCursor_2); /* 104 */ } /* 105 */ /* 106 */ } /* 107 */ /* 108 */ private void wholestagecodegen_init_0_1() { /* 109 */ rdd_mutableStateArray_1[4] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[3], 8); /* 110 */ rdd_mutableStateArray_0[4] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(rdd_mutableStateArray_0[2], 4); /* 111 */ rdd_mutableStateArray_1[5] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[4], 4); /* 112 */ rdd_mutableStateArray_1[6] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[4], 8); /* 113 */ /* 114 */ } /* 115 */ /* 116 */ private void project_writeFields_0_0(InternalRow project_tmpInput_0) { /* 117 */ rdd_mutableStateArray_0[3].write(0, (project_tmpInput_0.getByte(0))); /* 118 */ /* 119 */ if ((project_tmpInput_0.isNullAt(1))) { /* 120 */ rdd_mutableStateArray_0[3].setNullAt(1); /* 121 */ } else { /* 122 */ rdd_mutableStateArray_0[3].write(1, (project_tmpInput_0.getInt(1))); /* 123 */ } /* 124 */ /* 125 */ if ((project_tmpInput_0.isNullAt(2))) { /* 126 */ rdd_mutableStateArray_0[3].setNullAt(2); /* 127 */ } else { /* 128 */ // Remember the current cursor so that we can calculate how many bytes are /* 129 */ // written later. /* 130 */ final int project_previousCursor_1 = rdd_mutableStateArray_0[3].cursor(); /* 131 */ /* 132 */ final ArrayData project_tmpInput_1 = (project_tmpInput_0.getArray(2)); /* 133 */ if (project_tmpInput_1 instanceof UnsafeArrayData) { /* 134 */ rdd_mutableStateArray_0[3].write((UnsafeArrayData) project_tmpInput_1); /* 135 */ } else { /* 136 */ final int project_numElements_0 = project_tmpInput_1.numElements(); /* 137 */ rdd_mutableStateArray_1[3].initialize(project_numElements_0); /* 138 */ /* 139 */ for (int project_index_0 = 0; project_index_0 < project_numElements_0; project_index_0++) { /* 140 */ rdd_mutableStateArray_1[3].write(project_index_0, project_tmpInput_1.getInt(project_index_0)); /* 141 */ } /* 142 */ } /* 143 */ /* 144 */ rdd_mutableStateArray_0[3].setOffsetAndSizeFromPreviousCursor(2, project_previousCursor_1); /* 145 */ } /* 146 */ /* 147 */ } /* 148 */ /* 149 */ protected void processNext() throws java.io.IOException { /* 150 */ while ( rdd_input_0.hasNext()) { /* 151 */ InternalRow rdd_row_0 = (InternalRow) rdd_input_0.next(); /* 152 */ ((org.apache.spark.sql.execution.metric.SQLMetric) references[0] /* numOutputRows */).add(1); /* 153 */ // common sub-expressions /* 154 */ /* 155 */ boolean rdd_isNull_1 = rdd_row_0.isNullAt(1); /* 156 */ ArrayData rdd_value_1 = rdd_isNull_1 ? /* 157 */ null : (rdd_row_0.getArray(1)); /* 158 */ /* 159 */ Object project_arg_0 = null; /* 160 */ if (rdd_isNull_1) { /* 161 */ project_arg_0 = ((scala.Function1[]) references[1] /* converters */)[0].apply(null); /* 162 */ } else { /* 163 */ project_arg_0 = ((scala.Function1[]) references[1] /* converters */)[0].apply(rdd_value_1); /* 164 */ } /* 165 */ /* 166 */ InternalRow project_result_0 = null; /* 167 */ try { /* 168 */ project_result_0 = (InternalRow)((scala.Function1[]) references[1] /* converters */)[1].apply(((scala.Function1) references[3] /* udf */).apply(project_arg_0)); /* 169 */ } catch (Exception e) { /* 170 */ throw new org.apache.spark.SparkException(((java.lang.String) references[2] /* errMsg */), e); /* 171 */ } /* 172 */ /* 173 */ boolean project_isNull_0 = project_result_0 == null; /* 174 */ InternalRow project_value_0 = null; /* 175 */ if (!project_isNull_0) { /* 176 */ project_value_0 = project_result_0; /* 177 */ } /* 178 */ boolean rdd_isNull_2 = rdd_row_0.isNullAt(2); /* 179 */ InternalRow rdd_value_2 = rdd_isNull_2 ? /* 180 */ null : (rdd_row_0.getStruct(2, 4)); /* 181 */ rdd_mutableStateArray_0[2].reset(); /* 182 */ /* 183 */ rdd_mutableStateArray_0[2].zeroOutNullBytes(); /* 184 */ /* 185 */ if (project_isNull_0) { /* 186 */ rdd_mutableStateArray_0[2].setNullAt(0); /* 187 */ } else { /* 188 */ final InternalRow project_tmpInput_0 = project_value_0; /* 189 */ if (project_tmpInput_0 instanceof UnsafeRow) { /* 190 */ rdd_mutableStateArray_0[2].write(0, (UnsafeRow) project_tmpInput_0); /* 191 */ } else { /* 192 */ // Remember the current cursor so that we can calculate how many bytes are /* 193 */ // written later. /* 194 */ final int project_previousCursor_0 = rdd_mutableStateArray_0[2].cursor(); /* 195 */ /* 196 */ rdd_mutableStateArray_0[3].resetRowWriter(); /* 197 */ project_writeFields_0_0(project_tmpInput_0); /* 198 */ project_writeFields_0_1(project_tmpInput_0); /* 199 */ /* 200 */ rdd_mutableStateArray_0[2].setOffsetAndSizeFromPreviousCursor(0, project_previousCursor_0); /* 201 */ } /* 202 */ } /* 203 */ /* 204 */ if (rdd_isNull_2) { /* 205 */ rdd_mutableStateArray_0[2].setNullAt(1); /* 206 */ } else { /* 207 */ final InternalRow project_tmpInput_3 = rdd_value_2; /* 208 */ if (project_tmpInput_3 instanceof UnsafeRow) { /* 209 */ rdd_mutableStateArray_0[2].write(1, (UnsafeRow) project_tmpInput_3); /* 210 */ } else { /* 211 */ // Remember the current cursor so that we can calculate how many bytes are /* 212 */ // written later. /* 213 */ final int project_previousCursor_3 = rdd_mutableStateArray_0[2].cursor(); /* 214 */ /* 215 */ rdd_mutableStateArray_0[4].resetRowWriter(); /* 216 */ project_writeFields_1_0(project_tmpInput_3); /* 217 */ project_writeFields_1_1(project_tmpInput_3); /* 218 */ /* 219 */ rdd_mutableStateArray_0[2].setOffsetAndSizeFromPreviousCursor(1, project_previousCursor_3); /* 220 */ } /* 221 */ } /* 222 */ append((rdd_mutableStateArray_0[2].getRow())); /* 223 */ if (shouldStop()) return; /* 224 */ } /* 225 */ } /* 226 */ /* 227 */ private void wholestagecodegen_init_0_0() { /* 228 */ rdd_input_0 = inputs[0]; /* 229 */ rdd_mutableStateArray_0[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(3, 64); /* 230 */ rdd_mutableStateArray_1[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[0], 8); /* 231 */ rdd_mutableStateArray_0[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(rdd_mutableStateArray_0[0], 4); /* 232 */ rdd_mutableStateArray_1[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[1], 4); /* 233 */ rdd_mutableStateArray_1[2] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[1], 8); /* 234 */ rdd_mutableStateArray_0[2] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(2, 64); /* 235 */ rdd_mutableStateArray_0[3] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(rdd_mutableStateArray_0[2], 4); /* 236 */ rdd_mutableStateArray_1[3] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[3], 4); /* 237 */ /* 238 */ } /* 239 */ /* 240 */ } 21/01/17 17:43:04.082 Thread-3 DEBUG WholeStageCodegenExec: /* 001 */ public Object generate(Object[] references) { /* 002 */ return new GeneratedIteratorForCodegenStage1(references); /* 003 */ } /* 004 */ /* 005 */ // codegenStageId=1 /* 006 */ final class GeneratedIteratorForCodegenStage1 extends org.apache.spark.sql.execution.BufferedRowIterator { /* 007 */ private Object[] references; /* 008 */ private scala.collection.Iterator[] inputs; /* 009 */ private scala.collection.Iterator rdd_input_0; /* 010 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[] rdd_mutableStateArray_0 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[1]; /* 011 */ /* 012 */ public GeneratedIteratorForCodegenStage1(Object[] references) { /* 013 */ this.references = references; /* 014 */ } /* 015 */ /* 016 */ public void init(int index, scala.collection.Iterator[] inputs) { /* 017 */ partitionIndex = index; /* 018 */ this.inputs = inputs; /* 019 */ rdd_input_0 = inputs[0]; /* 020 */ rdd_mutableStateArray_0[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(3, 0); /* 021 */ /* 022 */ } /* 023 */ /* 024 */ protected void processNext() throws java.io.IOException { /* 025 */ while ( rdd_input_0.hasNext()) { /* 026 */ InternalRow rdd_row_0 = (InternalRow) rdd_input_0.next(); /* 027 */ ((org.apache.spark.sql.execution.metric.SQLMetric) references[0] /* numOutputRows */).add(1); /* 028 */ boolean rdd_isNull_0 = rdd_row_0.isNullAt(0); /* 029 */ long rdd_value_0 = rdd_isNull_0 ? /* 030 */ -1L : (rdd_row_0.getLong(0)); /* 031 */ boolean rdd_isNull_1 = rdd_row_0.isNullAt(1); /* 032 */ long rdd_value_1 = rdd_isNull_1 ? /* 033 */ -1L : (rdd_row_0.getLong(1)); /* 034 */ boolean rdd_isNull_2 = rdd_row_0.isNullAt(2); /* 035 */ double rdd_value_2 = rdd_isNull_2 ? /* 036 */ -1.0 : (rdd_row_0.getDouble(2)); /* 037 */ rdd_mutableStateArray_0[0].reset(); /* 038 */ /* 039 */ rdd_mutableStateArray_0[0].zeroOutNullBytes(); /* 040 */ /* 041 */ if (rdd_isNull_0) { /* 042 */ rdd_mutableStateArray_0[0].setNullAt(0); /* 043 */ } else { /* 044 */ rdd_mutableStateArray_0[0].write(0, rdd_value_0); /* 045 */ } /* 046 */ /* 047 */ if (rdd_isNull_1) { /* 048 */ rdd_mutableStateArray_0[0].setNullAt(1); /* 049 */ } else { /* 050 */ rdd_mutableStateArray_0[0].write(1, rdd_value_1); /* 051 */ } /* 052 */ /* 053 */ if (rdd_isNull_2) { /* 054 */ rdd_mutableStateArray_0[0].setNullAt(2); /* 055 */ } else { /* 056 */ rdd_mutableStateArray_0[0].write(2, rdd_value_2); /* 057 */ } /* 058 */ append((rdd_mutableStateArray_0[0].getRow())); /* 059 */ if (shouldStop()) return; /* 060 */ } /* 061 */ } /* 062 */ /* 063 */ } 21/01/17 17:43:04.087 Thread-3 DEBUG CodeGenerator: /* 001 */ public Object generate(Object[] references) { /* 002 */ return new GeneratedIteratorForCodegenStage1(references); /* 003 */ } /* 004 */ /* 005 */ // codegenStageId=1 /* 006 */ final class GeneratedIteratorForCodegenStage1 extends org.apache.spark.sql.execution.BufferedRowIterator { /* 007 */ private Object[] references; /* 008 */ private scala.collection.Iterator[] inputs; /* 009 */ private scala.collection.Iterator rdd_input_0; /* 010 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[] rdd_mutableStateArray_0 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[1]; /* 011 */ /* 012 */ public GeneratedIteratorForCodegenStage1(Object[] references) { /* 013 */ this.references = references; /* 014 */ } /* 015 */ /* 016 */ public void init(int index, scala.collection.Iterator[] inputs) { /* 017 */ partitionIndex = index; /* 018 */ this.inputs = inputs; /* 019 */ rdd_input_0 = inputs[0]; /* 020 */ rdd_mutableStateArray_0[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(3, 0); /* 021 */ /* 022 */ } /* 023 */ /* 024 */ protected void processNext() throws java.io.IOException { /* 025 */ while ( rdd_input_0.hasNext()) { /* 026 */ InternalRow rdd_row_0 = (InternalRow) rdd_input_0.next(); /* 027 */ ((org.apache.spark.sql.execution.metric.SQLMetric) references[0] /* numOutputRows */).add(1); /* 028 */ boolean rdd_isNull_0 = rdd_row_0.isNullAt(0); /* 029 */ long rdd_value_0 = rdd_isNull_0 ? /* 030 */ -1L : (rdd_row_0.getLong(0)); /* 031 */ boolean rdd_isNull_1 = rdd_row_0.isNullAt(1); /* 032 */ long rdd_value_1 = rdd_isNull_1 ? /* 033 */ -1L : (rdd_row_0.getLong(1)); /* 034 */ boolean rdd_isNull_2 = rdd_row_0.isNullAt(2); /* 035 */ double rdd_value_2 = rdd_isNull_2 ? /* 036 */ -1.0 : (rdd_row_0.getDouble(2)); /* 037 */ rdd_mutableStateArray_0[0].reset(); /* 038 */ /* 039 */ rdd_mutableStateArray_0[0].zeroOutNullBytes(); /* 040 */ /* 041 */ if (rdd_isNull_0) { /* 042 */ rdd_mutableStateArray_0[0].setNullAt(0); /* 043 */ } else { /* 044 */ rdd_mutableStateArray_0[0].write(0, rdd_value_0); /* 045 */ } /* 046 */ /* 047 */ if (rdd_isNull_1) { /* 048 */ rdd_mutableStateArray_0[0].setNullAt(1); /* 049 */ } else { /* 050 */ rdd_mutableStateArray_0[0].write(1, rdd_value_1); /* 051 */ } /* 052 */ /* 053 */ if (rdd_isNull_2) { /* 054 */ rdd_mutableStateArray_0[0].setNullAt(2); /* 055 */ } else { /* 056 */ rdd_mutableStateArray_0[0].write(2, rdd_value_2); /* 057 */ } /* 058 */ append((rdd_mutableStateArray_0[0].getRow())); /* 059 */ if (shouldStop()) return; /* 060 */ } /* 061 */ } /* 062 */ /* 063 */ } 21/01/17 17:43:04.094 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_0 stored as values in memory (estimated size 16.8 KiB, free 366.3 MiB) 21/01/17 17:43:04.096 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_0 locally took 90 ms 21/01/17 17:43:04.099 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_0 without replication took 93 ms 21/01/17 17:43:04.105 Thread-3 INFO CodeGenerator: Code generated in 22.21173 ms 21/01/17 17:43:04.106 Thread-3 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$doExecute$4$adapted 21/01/17 17:43:04.107 Thread-3 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$doExecute$4$adapted) is now cleaned +++ 21/01/17 17:43:04.111 Thread-3 DEBUG CodeGenerator: /* 001 */ public Object generate(Object[] references) { /* 002 */ return new GeneratedIteratorForCodegenStage1(references); /* 003 */ } /* 004 */ /* 005 */ // codegenStageId=1 /* 006 */ final class GeneratedIteratorForCodegenStage1 extends org.apache.spark.sql.execution.BufferedRowIterator { /* 007 */ private Object[] references; /* 008 */ private scala.collection.Iterator[] inputs; /* 009 */ private scala.collection.Iterator rdd_input_0; /* 010 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[] rdd_mutableStateArray_0 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[5]; /* 011 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[] rdd_mutableStateArray_1 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[7]; /* 012 */ /* 013 */ public GeneratedIteratorForCodegenStage1(Object[] references) { /* 014 */ this.references = references; /* 015 */ } /* 016 */ /* 017 */ public void init(int index, scala.collection.Iterator[] inputs) { /* 018 */ partitionIndex = index; /* 019 */ this.inputs = inputs; /* 020 */ wholestagecodegen_init_0_0(); /* 021 */ wholestagecodegen_init_0_1(); /* 022 */ /* 023 */ } /* 024 */ /* 025 */ private void project_writeFields_1_1(InternalRow project_tmpInput_3) { /* 026 */ if ((project_tmpInput_3.isNullAt(3))) { /* 027 */ rdd_mutableStateArray_0[4].setNullAt(3); /* 028 */ } else { /* 029 */ // Remember the current cursor so that we can calculate how many bytes are /* 030 */ // written later. /* 031 */ final int project_previousCursor_5 = rdd_mutableStateArray_0[4].cursor(); /* 032 */ /* 033 */ final ArrayData project_tmpInput_5 = (project_tmpInput_3.getArray(3)); /* 034 */ if (project_tmpInput_5 instanceof UnsafeArrayData) { /* 035 */ rdd_mutableStateArray_0[4].write((UnsafeArrayData) project_tmpInput_5); /* 036 */ } else { /* 037 */ final int project_numElements_3 = project_tmpInput_5.numElements(); /* 038 */ rdd_mutableStateArray_1[6].initialize(project_numElements_3); /* 039 */ /* 040 */ for (int project_index_3 = 0; project_index_3 < project_numElements_3; project_index_3++) { /* 041 */ rdd_mutableStateArray_1[6].write(project_index_3, project_tmpInput_5.getDouble(project_index_3)); /* 042 */ } /* 043 */ } /* 044 */ /* 045 */ rdd_mutableStateArray_0[4].setOffsetAndSizeFromPreviousCursor(3, project_previousCursor_5); /* 046 */ } /* 047 */ /* 048 */ } /* 049 */ /* 050 */ private void project_writeFields_1_0(InternalRow project_tmpInput_3) { /* 051 */ rdd_mutableStateArray_0[4].write(0, (project_tmpInput_3.getByte(0))); /* 052 */ /* 053 */ if ((project_tmpInput_3.isNullAt(1))) { /* 054 */ rdd_mutableStateArray_0[4].setNullAt(1); /* 055 */ } else { /* 056 */ rdd_mutableStateArray_0[4].write(1, (project_tmpInput_3.getInt(1))); /* 057 */ } /* 058 */ /* 059 */ if ((project_tmpInput_3.isNullAt(2))) { /* 060 */ rdd_mutableStateArray_0[4].setNullAt(2); /* 061 */ } else { /* 062 */ // Remember the current cursor so that we can calculate how many bytes are /* 063 */ // written later. /* 064 */ final int project_previousCursor_4 = rdd_mutableStateArray_0[4].cursor(); /* 065 */ /* 066 */ final ArrayData project_tmpInput_4 = (project_tmpInput_3.getArray(2)); /* 067 */ if (project_tmpInput_4 instanceof UnsafeArrayData) { /* 068 */ rdd_mutableStateArray_0[4].write((UnsafeArrayData) project_tmpInput_4); /* 069 */ } else { /* 070 */ final int project_numElements_2 = project_tmpInput_4.numElements(); /* 071 */ rdd_mutableStateArray_1[5].initialize(project_numElements_2); /* 072 */ /* 073 */ for (int project_index_2 = 0; project_index_2 < project_numElements_2; project_index_2++) { /* 074 */ rdd_mutableStateArray_1[5].write(project_index_2, project_tmpInput_4.getInt(project_index_2)); /* 075 */ } /* 076 */ } /* 077 */ /* 078 */ rdd_mutableStateArray_0[4].setOffsetAndSizeFromPreviousCursor(2, project_previousCursor_4); /* 079 */ } /* 080 */ /* 081 */ } /* 082 */ /* 083 */ private void project_writeFields_0_1(InternalRow project_tmpInput_0) { /* 084 */ if ((project_tmpInput_0.isNullAt(3))) { /* 085 */ rdd_mutableStateArray_0[3].setNullAt(3); /* 086 */ } else { /* 087 */ // Remember the current cursor so that we can calculate how many bytes are /* 088 */ // written later. /* 089 */ final int project_previousCursor_2 = rdd_mutableStateArray_0[3].cursor(); /* 090 */ /* 091 */ final ArrayData project_tmpInput_2 = (project_tmpInput_0.getArray(3)); /* 092 */ if (project_tmpInput_2 instanceof UnsafeArrayData) { /* 093 */ rdd_mutableStateArray_0[3].write((UnsafeArrayData) project_tmpInput_2); /* 094 */ } else { /* 095 */ final int project_numElements_1 = project_tmpInput_2.numElements(); /* 096 */ rdd_mutableStateArray_1[4].initialize(project_numElements_1); /* 097 */ /* 098 */ for (int project_index_1 = 0; project_index_1 < project_numElements_1; project_index_1++) { /* 099 */ rdd_mutableStateArray_1[4].write(project_index_1, project_tmpInput_2.getDouble(project_index_1)); /* 100 */ } /* 101 */ } /* 102 */ /* 103 */ rdd_mutableStateArray_0[3].setOffsetAndSizeFromPreviousCursor(3, project_previousCursor_2); /* 104 */ } /* 105 */ /* 106 */ } /* 107 */ /* 108 */ private void wholestagecodegen_init_0_1() { /* 109 */ rdd_mutableStateArray_1[4] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[3], 8); /* 110 */ rdd_mutableStateArray_0[4] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(rdd_mutableStateArray_0[2], 4); /* 111 */ rdd_mutableStateArray_1[5] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[4], 4); /* 112 */ rdd_mutableStateArray_1[6] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[4], 8); /* 113 */ /* 114 */ } /* 115 */ /* 116 */ private void project_writeFields_0_0(InternalRow project_tmpInput_0) { /* 117 */ rdd_mutableStateArray_0[3].write(0, (project_tmpInput_0.getByte(0))); /* 118 */ /* 119 */ if ((project_tmpInput_0.isNullAt(1))) { /* 120 */ rdd_mutableStateArray_0[3].setNullAt(1); /* 121 */ } else { /* 122 */ rdd_mutableStateArray_0[3].write(1, (project_tmpInput_0.getInt(1))); /* 123 */ } /* 124 */ /* 125 */ if ((project_tmpInput_0.isNullAt(2))) { /* 126 */ rdd_mutableStateArray_0[3].setNullAt(2); /* 127 */ } else { /* 128 */ // Remember the current cursor so that we can calculate how many bytes are /* 129 */ // written later. /* 130 */ final int project_previousCursor_1 = rdd_mutableStateArray_0[3].cursor(); /* 131 */ /* 132 */ final ArrayData project_tmpInput_1 = (project_tmpInput_0.getArray(2)); /* 133 */ if (project_tmpInput_1 instanceof UnsafeArrayData) { /* 134 */ rdd_mutableStateArray_0[3].write((UnsafeArrayData) project_tmpInput_1); /* 135 */ } else { /* 136 */ final int project_numElements_0 = project_tmpInput_1.numElements(); /* 137 */ rdd_mutableStateArray_1[3].initialize(project_numElements_0); /* 138 */ /* 139 */ for (int project_index_0 = 0; project_index_0 < project_numElements_0; project_index_0++) { /* 140 */ rdd_mutableStateArray_1[3].write(project_index_0, project_tmpInput_1.getInt(project_index_0)); /* 141 */ } /* 142 */ } /* 143 */ /* 144 */ rdd_mutableStateArray_0[3].setOffsetAndSizeFromPreviousCursor(2, project_previousCursor_1); /* 145 */ } /* 146 */ /* 147 */ } /* 148 */ /* 149 */ protected void processNext() throws java.io.IOException { /* 150 */ while ( rdd_input_0.hasNext()) { /* 151 */ InternalRow rdd_row_0 = (InternalRow) rdd_input_0.next(); /* 152 */ ((org.apache.spark.sql.execution.metric.SQLMetric) references[0] /* numOutputRows */).add(1); /* 153 */ // common sub-expressions /* 154 */ /* 155 */ boolean rdd_isNull_1 = rdd_row_0.isNullAt(1); /* 156 */ ArrayData rdd_value_1 = rdd_isNull_1 ? /* 157 */ null : (rdd_row_0.getArray(1)); /* 158 */ /* 159 */ Object project_arg_0 = null; /* 160 */ if (rdd_isNull_1) { /* 161 */ project_arg_0 = ((scala.Function1[]) references[1] /* converters */)[0].apply(null); /* 162 */ } else { /* 163 */ project_arg_0 = ((scala.Function1[]) references[1] /* converters */)[0].apply(rdd_value_1); /* 164 */ } /* 165 */ /* 166 */ InternalRow project_result_0 = null; /* 167 */ try { /* 168 */ project_result_0 = (InternalRow)((scala.Function1[]) references[1] /* converters */)[1].apply(((scala.Function1) references[3] /* udf */).apply(project_arg_0)); /* 169 */ } catch (Exception e) { /* 170 */ throw new org.apache.spark.SparkException(((java.lang.String) references[2] /* errMsg */), e); /* 171 */ } /* 172 */ /* 173 */ boolean project_isNull_0 = project_result_0 == null; /* 174 */ InternalRow project_value_0 = null; /* 175 */ if (!project_isNull_0) { /* 176 */ project_value_0 = project_result_0; /* 177 */ } /* 178 */ boolean rdd_isNull_2 = rdd_row_0.isNullAt(2); /* 179 */ InternalRow rdd_value_2 = rdd_isNull_2 ? /* 180 */ null : (rdd_row_0.getStruct(2, 4)); /* 181 */ rdd_mutableStateArray_0[2].reset(); /* 182 */ /* 183 */ rdd_mutableStateArray_0[2].zeroOutNullBytes(); /* 184 */ /* 185 */ if (project_isNull_0) { /* 186 */ rdd_mutableStateArray_0[2].setNullAt(0); /* 187 */ } else { /* 188 */ final InternalRow project_tmpInput_0 = project_value_0; /* 189 */ if (project_tmpInput_0 instanceof UnsafeRow) { /* 190 */ rdd_mutableStateArray_0[2].write(0, (UnsafeRow) project_tmpInput_0); /* 191 */ } else { /* 192 */ // Remember the current cursor so that we can calculate how many bytes are /* 193 */ // written later. /* 194 */ final int project_previousCursor_0 = rdd_mutableStateArray_0[2].cursor(); /* 195 */ /* 196 */ rdd_mutableStateArray_0[3].resetRowWriter(); /* 197 */ project_writeFields_0_0(project_tmpInput_0); /* 198 */ project_writeFields_0_1(project_tmpInput_0); /* 199 */ /* 200 */ rdd_mutableStateArray_0[2].setOffsetAndSizeFromPreviousCursor(0, project_previousCursor_0); /* 201 */ } /* 202 */ } /* 203 */ /* 204 */ if (rdd_isNull_2) { /* 205 */ rdd_mutableStateArray_0[2].setNullAt(1); /* 206 */ } else { /* 207 */ final InternalRow project_tmpInput_3 = rdd_value_2; /* 208 */ if (project_tmpInput_3 instanceof UnsafeRow) { /* 209 */ rdd_mutableStateArray_0[2].write(1, (UnsafeRow) project_tmpInput_3); /* 210 */ } else { /* 211 */ // Remember the current cursor so that we can calculate how many bytes are /* 212 */ // written later. /* 213 */ final int project_previousCursor_3 = rdd_mutableStateArray_0[2].cursor(); /* 214 */ /* 215 */ rdd_mutableStateArray_0[4].resetRowWriter(); /* 216 */ project_writeFields_1_0(project_tmpInput_3); /* 217 */ project_writeFields_1_1(project_tmpInput_3); /* 218 */ /* 219 */ rdd_mutableStateArray_0[2].setOffsetAndSizeFromPreviousCursor(1, project_previousCursor_3); /* 220 */ } /* 221 */ } /* 222 */ append((rdd_mutableStateArray_0[2].getRow())); /* 223 */ if (shouldStop()) return; /* 224 */ } /* 225 */ } /* 226 */ /* 227 */ private void wholestagecodegen_init_0_0() { /* 228 */ rdd_input_0 = inputs[0]; /* 229 */ rdd_mutableStateArray_0[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(3, 64); /* 230 */ rdd_mutableStateArray_1[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[0], 8); /* 231 */ rdd_mutableStateArray_0[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(rdd_mutableStateArray_0[0], 4); /* 232 */ rdd_mutableStateArray_1[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[1], 4); /* 233 */ rdd_mutableStateArray_1[2] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[1], 8); /* 234 */ rdd_mutableStateArray_0[2] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(2, 64); /* 235 */ rdd_mutableStateArray_0[3] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(rdd_mutableStateArray_0[2], 4); /* 236 */ rdd_mutableStateArray_1[3] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[3], 4); /* 237 */ /* 238 */ } /* 239 */ /* 240 */ } 21/01/17 17:43:04.118 Thread-3 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$rdd$1 21/01/17 17:43:04.125 Thread-3 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$rdd$1) is now cleaned +++ 21/01/17 17:43:04.142 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_0_piece0 stored as bytes in memory (estimated size 7.9 KiB, free 366.3 MiB) 21/01/17 17:43:04.143 Thread-3 INFO Instrumentation: [51e91e47] training: numPartitions=4 storageLevel=StorageLevel(1 replicas) 21/01/17 17:43:04.144 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master broadcast_0_piece0 for BlockManagerId(driver, 192.168.1.225, 45289, None) 21/01/17 17:43:04.147 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added broadcast_0_piece0 in memory on 192.168.1.225:45289 (size: 7.9 KiB, free: 366.3 MiB) 21/01/17 17:43:04.150 dag-scheduler-event-loop DEBUG BlockManagerMaster: Updated info of block broadcast_0_piece0 21/01/17 17:43:04.151 dag-scheduler-event-loop DEBUG BlockManager: Told master about block broadcast_0_piece0 21/01/17 17:43:04.152 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_0_piece0 locally took 12 ms 21/01/17 17:43:04.152 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_0_piece0 without replication took 13 ms 21/01/17 17:43:04.153 dag-scheduler-event-loop INFO SparkContext: Created broadcast 0 from broadcast at DAGScheduler.scala:1383 21/01/17 17:43:04.170 Thread-3 INFO Instrumentation: [51e91e47] {"rank":1,"maxIter":1} 21/01/17 17:43:04.171 dag-scheduler-event-loop INFO DAGScheduler: Submitting 4 missing tasks from ResultStage 0 (MapPartitionsRDD[6] at at /usr/lib/python3.6/runpy.py:85) (first 15 tasks are for partitions Vector(0, 1, 2, 3)) 21/01/17 17:43:04.172 dag-scheduler-event-loop INFO TaskSchedulerImpl: Adding task set 0.0 with 4 tasks resource profile 0 21/01/17 17:43:04.174 Thread-3 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$executeTake$2 21/01/17 17:43:04.177 Thread-3 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$executeTake$2) is now cleaned +++ 21/01/17 17:43:04.179 Thread-3 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$take$2 21/01/17 17:43:04.179 Thread-3 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$runJob$5 21/01/17 17:43:04.183 Thread-3 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$take$2) is now cleaned +++ 21/01/17 17:43:04.185 Thread-3 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$runJob$5) is now cleaned +++ 21/01/17 17:43:04.187 Thread-3 INFO SparkContext: Starting job: head at MetadataUtils.scala:51 21/01/17 17:43:04.197 dag-scheduler-event-loop DEBUG DAGScheduler: Merging stage rdd profiles: Set() 21/01/17 17:43:04.217 dag-scheduler-event-loop DEBUG TaskSetManager: Epoch for TaskSet 0.0: 0 21/01/17 17:43:04.225 dag-scheduler-event-loop DEBUG TaskSetManager: Adding pending tasks took 6 ms 21/01/17 17:43:04.228 dag-scheduler-event-loop DEBUG TaskSetManager: Valid locality levels for TaskSet 0.0: NO_PREF, ANY 21/01/17 17:43:04.236 dag-scheduler-event-loop INFO DAGScheduler: Got job 0 (head at MetadataUtils.scala:51) with 1 output partitions 21/01/17 17:43:04.237 dag-scheduler-event-loop INFO DAGScheduler: Final stage: ResultStage 0 (head at MetadataUtils.scala:51) 21/01/17 17:43:04.238 dag-scheduler-event-loop INFO DAGScheduler: Parents of final stage: List() 21/01/17 17:43:04.246 dag-scheduler-event-loop INFO DAGScheduler: Missing parents: List() 21/01/17 17:43:04.254 dag-scheduler-event-loop DEBUG DAGScheduler: submitStage(ResultStage 0 (name=head at MetadataUtils.scala:51;jobs=0)) 21/01/17 17:43:04.256 dag-scheduler-event-loop DEBUG DAGScheduler: missing: List() 21/01/17 17:43:04.256 dag-scheduler-event-loop INFO DAGScheduler: Submitting ResultStage 0 (MapPartitionsRDD[6] at head at MetadataUtils.scala:51), which has no missing parents 21/01/17 17:43:04.257 dag-scheduler-event-loop DEBUG DAGScheduler: submitMissingTasks(ResultStage 0) 21/01/17 17:43:04.249 dispatcher-event-loop-2 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_0.0, runningTasks: 0 21/01/17 17:43:04.258 dispatcher-event-loop-2 DEBUG TaskSetManager: Valid locality levels for TaskSet 0.0: NO_PREF, ANY 21/01/17 17:43:04.277 Thread-3 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$runJob$5 21/01/17 17:43:04.297 Thread-3 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$runJob$5) is now cleaned +++ 21/01/17 17:43:04.301 Thread-3 INFO SparkContext: Starting job: isEmpty at ALS.scala:960 21/01/17 17:43:04.314 dag-scheduler-event-loop DEBUG DAGScheduler: Merging stage rdd profiles: Set() 21/01/17 17:43:04.324 dispatcher-event-loop-2 INFO TaskSetManager: Starting task 0.0 in stage 0.0 (TID 0) (192.168.1.225, executor driver, partition 0, PROCESS_LOCAL, 7450 bytes) taskResourceAssignments Map() 21/01/17 17:43:04.337 dag-scheduler-event-loop INFO DAGScheduler: Got job 0 (isEmpty at ALS.scala:960) with 1 output partitions 21/01/17 17:43:04.338 dag-scheduler-event-loop INFO DAGScheduler: Final stage: ResultStage 0 (isEmpty at ALS.scala:960) 21/01/17 17:43:04.338 dag-scheduler-event-loop INFO DAGScheduler: Parents of final stage: List() 21/01/17 17:43:04.341 dispatcher-event-loop-2 INFO TaskSetManager: Starting task 1.0 in stage 0.0 (TID 1) (192.168.1.225, executor driver, partition 1, PROCESS_LOCAL, 7450 bytes) taskResourceAssignments Map() 21/01/17 17:43:04.341 dispatcher-event-loop-2 INFO TaskSetManager: Starting task 2.0 in stage 0.0 (TID 2) (192.168.1.225, executor driver, partition 2, PROCESS_LOCAL, 7450 bytes) taskResourceAssignments Map() 21/01/17 17:43:04.342 dispatcher-event-loop-2 INFO TaskSetManager: Starting task 3.0 in stage 0.0 (TID 3) (192.168.1.225, executor driver, partition 3, PROCESS_LOCAL, 7450 bytes) taskResourceAssignments Map() 21/01/17 17:43:04.344 dag-scheduler-event-loop INFO DAGScheduler: Missing parents: List() 21/01/17 17:43:04.376 Executor task launch worker for task 0.0 in stage 0.0 (TID 0) INFO Executor: Running task 0.0 in stage 0.0 (TID 0) 21/01/17 17:43:04.391 Executor task launch worker for task 1.0 in stage 0.0 (TID 1) INFO Executor: Running task 1.0 in stage 0.0 (TID 1) 21/01/17 17:43:04.392 dag-scheduler-event-loop DEBUG DAGScheduler: submitStage(ResultStage 0 (name=isEmpty at ALS.scala:960;jobs=0)) 21/01/17 17:43:04.393 dag-scheduler-event-loop DEBUG DAGScheduler: missing: List() 21/01/17 17:43:04.398 Executor task launch worker for task 3.0 in stage 0.0 (TID 3) INFO Executor: Running task 3.0 in stage 0.0 (TID 3) 21/01/17 17:43:04.403 Executor task launch worker for task 2.0 in stage 0.0 (TID 2) INFO Executor: Running task 2.0 in stage 0.0 (TID 2) 21/01/17 17:43:04.406 dag-scheduler-event-loop INFO DAGScheduler: Submitting ResultStage 0 (MapPartitionsRDD[9] at map at ALS.scala:712), which has no missing parents 21/01/17 17:43:04.406 dag-scheduler-event-loop DEBUG DAGScheduler: submitMissingTasks(ResultStage 0) 21/01/17 17:43:04.409 Executor task launch worker for task 2.0 in stage 0.0 (TID 2) DEBUG ExecutorMetricsPoller: stageTCMP: (0, 0) -> 3 21/01/17 17:43:04.412 Executor task launch worker for task 3.0 in stage 0.0 (TID 3) DEBUG ExecutorMetricsPoller: stageTCMP: (0, 0) -> 4 21/01/17 17:43:04.412 Executor task launch worker for task 0.0 in stage 0.0 (TID 0) DEBUG ExecutorMetricsPoller: stageTCMP: (0, 0) -> 1 21/01/17 17:43:04.412 Executor task launch worker for task 1.0 in stage 0.0 (TID 1) DEBUG ExecutorMetricsPoller: stageTCMP: (0, 0) -> 2 21/01/17 17:43:04.463 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_0 stored as values in memory (estimated size 15.1 KiB, free 366.3 MiB) 21/01/17 17:43:04.465 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_0 locally took 40 ms 21/01/17 17:43:04.468 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_0 without replication took 43 ms 21/01/17 17:43:04.492 Executor task launch worker for task 1.0 in stage 0.0 (TID 1) DEBUG BlockManager: Getting local block broadcast_0 21/01/17 17:43:04.494 Executor task launch worker for task 1.0 in stage 0.0 (TID 1) DEBUG BlockManager: Level for block broadcast_0 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:04.528 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_0_piece0 stored as bytes in memory (estimated size 7.0 KiB, free 366.3 MiB) 21/01/17 17:43:04.530 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master broadcast_0_piece0 for BlockManagerId(driver, 192.168.1.225, 33787, None) 21/01/17 17:43:04.532 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added broadcast_0_piece0 in memory on 192.168.1.225:33787 (size: 7.0 KiB, free: 366.3 MiB) 21/01/17 17:43:04.535 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_0 stored as values in memory (estimated size 28.8 KiB, free 366.3 MiB) 21/01/17 17:43:04.537 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_0 locally took 38 ms 21/01/17 17:43:04.539 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_0 without replication took 40 ms 21/01/17 17:43:04.534 dag-scheduler-event-loop DEBUG BlockManagerMaster: Updated info of block broadcast_0_piece0 21/01/17 17:43:04.541 dag-scheduler-event-loop DEBUG BlockManager: Told master about block broadcast_0_piece0 21/01/17 17:43:04.542 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_0_piece0 locally took 17 ms 21/01/17 17:43:04.542 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_0_piece0 without replication took 17 ms 21/01/17 17:43:04.543 dag-scheduler-event-loop INFO SparkContext: Created broadcast 0 from broadcast at DAGScheduler.scala:1383 21/01/17 17:43:04.578 dag-scheduler-event-loop INFO DAGScheduler: Submitting 1 missing tasks from ResultStage 0 (MapPartitionsRDD[6] at head at MetadataUtils.scala:51) (first 15 tasks are for partitions Vector(0)) 21/01/17 17:43:04.579 dag-scheduler-event-loop INFO TaskSchedulerImpl: Adding task set 0.0 with 1 tasks resource profile 0 21/01/17 17:43:04.582 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.6 KiB, free 366.3 MiB) 21/01/17 17:43:04.584 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master broadcast_0_piece0 for BlockManagerId(driver, 192.168.1.225, 40985, None) 21/01/17 17:43:04.585 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added broadcast_0_piece0 in memory on 192.168.1.225:40985 (size: 13.6 KiB, free: 366.3 MiB) 21/01/17 17:43:04.590 Thread-3 INFO CodeGenerator: Code generated in 523.420607 ms 21/01/17 17:43:04.593 Thread-3 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$doExecute$4$adapted 21/01/17 17:43:04.594 dag-scheduler-event-loop DEBUG BlockManagerMaster: Updated info of block broadcast_0_piece0 21/01/17 17:43:04.594 dag-scheduler-event-loop DEBUG BlockManager: Told master about block broadcast_0_piece0 21/01/17 17:43:04.595 Thread-3 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$doExecute$4$adapted) is now cleaned +++ 21/01/17 17:43:04.597 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_0_piece0 locally took 17 ms 21/01/17 17:43:04.597 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_0_piece0 without replication took 17 ms 21/01/17 17:43:04.598 dag-scheduler-event-loop INFO SparkContext: Created broadcast 0 from broadcast at DAGScheduler.scala:1383 21/01/17 17:43:04.616 dag-scheduler-event-loop DEBUG TaskSetManager: Epoch for TaskSet 0.0: 0 21/01/17 17:43:04.617 dag-scheduler-event-loop INFO DAGScheduler: Submitting 1 missing tasks from ResultStage 0 (MapPartitionsRDD[9] at map at ALS.scala:712) (first 15 tasks are for partitions Vector(0)) 21/01/17 17:43:04.619 dag-scheduler-event-loop INFO TaskSchedulerImpl: Adding task set 0.0 with 1 tasks resource profile 0 21/01/17 17:43:04.620 dag-scheduler-event-loop DEBUG TaskSetManager: Adding pending tasks took 3 ms 21/01/17 17:43:04.624 dag-scheduler-event-loop DEBUG TaskSetManager: Valid locality levels for TaskSet 0.0: NO_PREF, ANY 21/01/17 17:43:04.647 dag-scheduler-event-loop DEBUG TaskSetManager: Epoch for TaskSet 0.0: 0 21/01/17 17:43:04.652 dag-scheduler-event-loop DEBUG TaskSetManager: Adding pending tasks took 3 ms 21/01/17 17:43:04.655 dag-scheduler-event-loop DEBUG TaskSetManager: Valid locality levels for TaskSet 0.0: NO_PREF, ANY 21/01/17 17:43:04.658 dispatcher-event-loop-2 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_0.0, runningTasks: 0 21/01/17 17:43:04.672 dispatcher-event-loop-2 DEBUG TaskSetManager: Valid locality levels for TaskSet 0.0: NO_PREF, ANY 21/01/17 17:43:04.672 dispatcher-event-loop-2 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_0.0, runningTasks: 0 21/01/17 17:43:04.675 dispatcher-event-loop-2 DEBUG TaskSetManager: Valid locality levels for TaskSet 0.0: NO_PREF, ANY 21/01/17 17:43:04.706 dispatcher-event-loop-2 INFO TaskSetManager: Starting task 0.0 in stage 0.0 (TID 0) (192.168.1.225, executor driver, partition 0, PROCESS_LOCAL, 7373 bytes) taskResourceAssignments Map() 21/01/17 17:43:04.712 dispatcher-event-loop-2 DEBUG TaskSetManager: No tasks for locality level NO_PREF, so moving to locality level ANY 21/01/17 17:43:04.721 dispatcher-event-loop-2 INFO TaskSetManager: Starting task 0.0 in stage 0.0 (TID 0) (192.168.1.225, executor driver, partition 0, PROCESS_LOCAL, 7398 bytes) taskResourceAssignments Map() 21/01/17 17:43:04.756 Thread-3 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$collect$2 21/01/17 17:43:04.764 Thread-3 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$collect$2) is now cleaned +++ 21/01/17 17:43:04.768 Executor task launch worker for task 0.0 in stage 0.0 (TID 0) INFO Executor: Running task 0.0 in stage 0.0 (TID 0) 21/01/17 17:43:04.793 Executor task launch worker for task 0.0 in stage 0.0 (TID 0) DEBUG ExecutorMetricsPoller: stageTCMP: (0, 0) -> 1 21/01/17 17:43:04.802 dispatcher-event-loop-2 DEBUG TaskSetManager: No tasks for locality level NO_PREF, so moving to locality level ANY 21/01/17 17:43:04.843 Executor task launch worker for task 0.0 in stage 0.0 (TID 0) INFO Executor: Running task 0.0 in stage 0.0 (TID 0) 21/01/17 17:43:04.847 Thread-3 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$runJob$5 21/01/17 17:43:04.852 Thread-3 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$runJob$5) is now cleaned +++ 21/01/17 17:43:04.852 Executor task launch worker for task 0.0 in stage 0.0 (TID 0) DEBUG ExecutorMetricsPoller: stageTCMP: (0, 0) -> 1 21/01/17 17:43:04.865 Thread-3 INFO SparkContext: Starting job: at /usr/lib/python3.6/runpy.py:85 21/01/17 17:43:04.871 dag-scheduler-event-loop DEBUG DAGScheduler: Merging stage rdd profiles: Set() 21/01/17 17:43:04.885 dag-scheduler-event-loop INFO DAGScheduler: Got job 0 ( at /usr/lib/python3.6/runpy.py:85) with 4 output partitions 21/01/17 17:43:04.886 dag-scheduler-event-loop INFO DAGScheduler: Final stage: ResultStage 0 ( at /usr/lib/python3.6/runpy.py:85) 21/01/17 17:43:04.886 dag-scheduler-event-loop INFO DAGScheduler: Parents of final stage: List() 21/01/17 17:43:04.889 dag-scheduler-event-loop INFO DAGScheduler: Missing parents: List() 21/01/17 17:43:04.914 Executor task launch worker for task 0.0 in stage 0.0 (TID 0) DEBUG BlockManager: Getting local block broadcast_0 21/01/17 17:43:04.915 Executor task launch worker for task 0.0 in stage 0.0 (TID 0) DEBUG BlockManager: Level for block broadcast_0 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:04.915 dag-scheduler-event-loop DEBUG DAGScheduler: submitStage(ResultStage 0 (name= at /usr/lib/python3.6/runpy.py:85;jobs=0)) 21/01/17 17:43:04.916 dag-scheduler-event-loop DEBUG DAGScheduler: missing: List() 21/01/17 17:43:04.917 dag-scheduler-event-loop INFO DAGScheduler: Submitting ResultStage 0 (MapPartitionsRDD[6] at at /usr/lib/python3.6/runpy.py:85), which has no missing parents 21/01/17 17:43:04.918 dag-scheduler-event-loop DEBUG DAGScheduler: submitMissingTasks(ResultStage 0) 21/01/17 17:43:04.953 Executor task launch worker for task 0.0 in stage 0.0 (TID 0) DEBUG BlockManager: Getting local block broadcast_0 21/01/17 17:43:04.955 Executor task launch worker for task 0.0 in stage 0.0 (TID 0) DEBUG BlockManager: Level for block broadcast_0 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:04.966 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_1 stored as values in memory (estimated size 31.4 KiB, free 366.3 MiB) 21/01/17 17:43:04.966 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_1 locally took 28 ms 21/01/17 17:43:04.966 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_1 without replication took 28 ms 21/01/17 17:43:04.970 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_1_piece0 stored as bytes in memory (estimated size 13.3 KiB, free 366.3 MiB) 21/01/17 17:43:04.970 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master broadcast_1_piece0 for BlockManagerId(driver, 192.168.1.225, 46295, None) 21/01/17 17:43:04.971 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added broadcast_1_piece0 in memory on 192.168.1.225:46295 (size: 13.3 KiB, free: 366.3 MiB) 21/01/17 17:43:04.971 dag-scheduler-event-loop DEBUG BlockManagerMaster: Updated info of block broadcast_1_piece0 21/01/17 17:43:04.971 dag-scheduler-event-loop DEBUG BlockManager: Told master about block broadcast_1_piece0 21/01/17 17:43:04.971 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_1_piece0 locally took 2 ms 21/01/17 17:43:04.971 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_1_piece0 without replication took 2 ms 21/01/17 17:43:04.972 dag-scheduler-event-loop INFO SparkContext: Created broadcast 1 from broadcast at DAGScheduler.scala:1383 21/01/17 17:43:04.991 dag-scheduler-event-loop INFO DAGScheduler: Submitting 4 missing tasks from ResultStage 0 (MapPartitionsRDD[6] at at /usr/lib/python3.6/runpy.py:85) (first 15 tasks are for partitions Vector(0, 1, 2, 3)) 21/01/17 17:43:04.992 dag-scheduler-event-loop INFO TaskSchedulerImpl: Adding task set 0.0 with 4 tasks resource profile 0 21/01/17 17:43:05.036 dag-scheduler-event-loop DEBUG TaskSetManager: Epoch for TaskSet 0.0: 0 21/01/17 17:43:05.042 dag-scheduler-event-loop DEBUG TaskSetManager: Adding pending tasks took 3 ms 21/01/17 17:43:05.044 dag-scheduler-event-loop DEBUG TaskSetManager: Valid locality levels for TaskSet 0.0: NO_PREF, ANY 21/01/17 17:43:05.059 dispatcher-event-loop-2 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_0.0, runningTasks: 0 21/01/17 17:43:05.060 dispatcher-event-loop-2 DEBUG TaskSetManager: Valid locality levels for TaskSet 0.0: NO_PREF, ANY 21/01/17 17:43:05.141 dispatcher-event-loop-2 INFO TaskSetManager: Starting task 0.0 in stage 0.0 (TID 0) (192.168.1.225, executor driver, partition 0, PROCESS_LOCAL, 7333 bytes) taskResourceAssignments Map() 21/01/17 17:43:05.150 dispatcher-event-loop-2 INFO TaskSetManager: Starting task 1.0 in stage 0.0 (TID 1) (192.168.1.225, executor driver, partition 1, PROCESS_LOCAL, 7421 bytes) taskResourceAssignments Map() 21/01/17 17:43:05.153 dispatcher-event-loop-2 INFO TaskSetManager: Starting task 2.0 in stage 0.0 (TID 2) (192.168.1.225, executor driver, partition 2, PROCESS_LOCAL, 7396 bytes) taskResourceAssignments Map() 21/01/17 17:43:05.154 dispatcher-event-loop-2 INFO TaskSetManager: Starting task 3.0 in stage 0.0 (TID 3) (192.168.1.225, executor driver, partition 3, PROCESS_LOCAL, 7385 bytes) taskResourceAssignments Map() 21/01/17 17:43:05.208 Executor task launch worker for task 1.0 in stage 0.0 (TID 1) INFO Executor: Running task 1.0 in stage 0.0 (TID 1) 21/01/17 17:43:05.208 Executor task launch worker for task 0.0 in stage 0.0 (TID 0) INFO Executor: Running task 0.0 in stage 0.0 (TID 0) 21/01/17 17:43:05.208 Executor task launch worker for task 3.0 in stage 0.0 (TID 3) INFO Executor: Running task 3.0 in stage 0.0 (TID 3) 21/01/17 17:43:05.208 Executor task launch worker for task 2.0 in stage 0.0 (TID 2) INFO Executor: Running task 2.0 in stage 0.0 (TID 2) 21/01/17 17:43:05.222 Executor task launch worker for task 1.0 in stage 0.0 (TID 1) DEBUG ExecutorMetricsPoller: stageTCMP: (0, 0) -> 1 21/01/17 17:43:05.224 Executor task launch worker for task 0.0 in stage 0.0 (TID 0) DEBUG ExecutorMetricsPoller: stageTCMP: (0, 0) -> 4 21/01/17 17:43:05.222 Executor task launch worker for task 2.0 in stage 0.0 (TID 2) DEBUG ExecutorMetricsPoller: stageTCMP: (0, 0) -> 3 21/01/17 17:43:05.223 Executor task launch worker for task 3.0 in stage 0.0 (TID 3) DEBUG ExecutorMetricsPoller: stageTCMP: (0, 0) -> 2 21/01/17 17:43:05.336 Executor task launch worker for task 0.0 in stage 0.0 (TID 0) DEBUG BlockManager: Getting local block broadcast_1 21/01/17 17:43:05.338 Executor task launch worker for task 0.0 in stage 0.0 (TID 0) DEBUG BlockManager: Level for block broadcast_1 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:05.465 Executor task launch worker for task 0.0 in stage 0.0 (TID 0) DEBUG GenerateMutableProjection: code for cast(input[0, bigint, false] as double): /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificMutableProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificMutableProjection extends org.apache.spark.sql.catalyst.expressions.codegen.BaseMutableProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private InternalRow mutableRow; /* 009 */ private double value_2; /* 010 */ /* 011 */ public SpecificMutableProjection(Object[] references) { /* 012 */ this.references = references; /* 013 */ mutableRow = new org.apache.spark.sql.catalyst.expressions.GenericInternalRow(1); /* 014 */ /* 015 */ /* 016 */ } /* 017 */ /* 018 */ public void initialize(int partitionIndex) { /* 019 */ /* 020 */ } /* 021 */ /* 022 */ public org.apache.spark.sql.catalyst.expressions.codegen.BaseMutableProjection target(InternalRow row) { /* 023 */ mutableRow = row; /* 024 */ return this; /* 025 */ } /* 026 */ /* 027 */ /* Provide immutable access to the last projected row. */ /* 028 */ public InternalRow currentValue() { /* 029 */ return (InternalRow) mutableRow; /* 030 */ } /* 031 */ /* 032 */ public java.lang.Object apply(java.lang.Object _i) { /* 033 */ InternalRow i = (InternalRow) _i; /* 034 */ /* 035 */ /* 036 */ long value_1 = i.getLong(0); /* 037 */ boolean isNull_0 = false; /* 038 */ double value_0 = -1.0; /* 039 */ if (!false) { /* 040 */ value_0 = (double) value_1; /* 041 */ } /* 042 */ value_2 = value_0; /* 043 */ /* 044 */ // copy all the results into MutableRow /* 045 */ mutableRow.setDouble(0, value_2); /* 046 */ return mutableRow; /* 047 */ } /* 048 */ /* 049 */ /* 050 */ } 21/01/17 17:43:05.479 Executor task launch worker for task 1.0 in stage 0.0 (TID 1) DEBUG GenerateMutableProjection: code for cast(input[0, bigint, false] as double): /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificMutableProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificMutableProjection extends org.apache.spark.sql.catalyst.expressions.codegen.BaseMutableProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private InternalRow mutableRow; /* 009 */ private double value_2; /* 010 */ /* 011 */ public SpecificMutableProjection(Object[] references) { /* 012 */ this.references = references; /* 013 */ mutableRow = new org.apache.spark.sql.catalyst.expressions.GenericInternalRow(1); /* 014 */ /* 015 */ /* 016 */ } /* 017 */ /* 018 */ public void initialize(int partitionIndex) { /* 019 */ /* 020 */ } /* 021 */ /* 022 */ public org.apache.spark.sql.catalyst.expressions.codegen.BaseMutableProjection target(InternalRow row) { /* 023 */ mutableRow = row; /* 024 */ return this; /* 025 */ } /* 026 */ /* 027 */ /* Provide immutable access to the last projected row. */ /* 028 */ public InternalRow currentValue() { /* 029 */ return (InternalRow) mutableRow; /* 030 */ } /* 031 */ /* 032 */ public java.lang.Object apply(java.lang.Object _i) { /* 033 */ InternalRow i = (InternalRow) _i; /* 034 */ /* 035 */ /* 036 */ long value_1 = i.getLong(0); /* 037 */ boolean isNull_0 = false; /* 038 */ double value_0 = -1.0; /* 039 */ if (!false) { /* 040 */ value_0 = (double) value_1; /* 041 */ } /* 042 */ value_2 = value_0; /* 043 */ /* 044 */ // copy all the results into MutableRow /* 045 */ mutableRow.setDouble(0, value_2); /* 046 */ return mutableRow; /* 047 */ } /* 048 */ /* 049 */ /* 050 */ } 21/01/17 17:43:05.488 Executor task launch worker for task 3.0 in stage 0.0 (TID 3) DEBUG GenerateMutableProjection: code for cast(input[0, bigint, false] as double): /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificMutableProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificMutableProjection extends org.apache.spark.sql.catalyst.expressions.codegen.BaseMutableProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private InternalRow mutableRow; /* 009 */ private double value_2; /* 010 */ /* 011 */ public SpecificMutableProjection(Object[] references) { /* 012 */ this.references = references; /* 013 */ mutableRow = new org.apache.spark.sql.catalyst.expressions.GenericInternalRow(1); /* 014 */ /* 015 */ /* 016 */ } /* 017 */ /* 018 */ public void initialize(int partitionIndex) { /* 019 */ /* 020 */ } /* 021 */ /* 022 */ public org.apache.spark.sql.catalyst.expressions.codegen.BaseMutableProjection target(InternalRow row) { /* 023 */ mutableRow = row; /* 024 */ return this; /* 025 */ } /* 026 */ /* 027 */ /* Provide immutable access to the last projected row. */ /* 028 */ public InternalRow currentValue() { /* 029 */ return (InternalRow) mutableRow; /* 030 */ } /* 031 */ /* 032 */ public java.lang.Object apply(java.lang.Object _i) { /* 033 */ InternalRow i = (InternalRow) _i; /* 034 */ /* 035 */ /* 036 */ long value_1 = i.getLong(0); /* 037 */ boolean isNull_0 = false; /* 038 */ double value_0 = -1.0; /* 039 */ if (!false) { /* 040 */ value_0 = (double) value_1; /* 041 */ } /* 042 */ value_2 = value_0; /* 043 */ /* 044 */ // copy all the results into MutableRow /* 045 */ mutableRow.setDouble(0, value_2); /* 046 */ return mutableRow; /* 047 */ } /* 048 */ /* 049 */ /* 050 */ } 21/01/17 17:43:05.468 Executor task launch worker for task 0.0 in stage 0.0 (TID 0) DEBUG CodeGenerator: /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificMutableProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificMutableProjection extends org.apache.spark.sql.catalyst.expressions.codegen.BaseMutableProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private InternalRow mutableRow; /* 009 */ private double value_2; /* 010 */ /* 011 */ public SpecificMutableProjection(Object[] references) { /* 012 */ this.references = references; /* 013 */ mutableRow = new org.apache.spark.sql.catalyst.expressions.GenericInternalRow(1); /* 014 */ /* 015 */ /* 016 */ } /* 017 */ /* 018 */ public void initialize(int partitionIndex) { /* 019 */ /* 020 */ } /* 021 */ /* 022 */ public org.apache.spark.sql.catalyst.expressions.codegen.BaseMutableProjection target(InternalRow row) { /* 023 */ mutableRow = row; /* 024 */ return this; /* 025 */ } /* 026 */ /* 027 */ /* Provide immutable access to the last projected row. */ /* 028 */ public InternalRow currentValue() { /* 029 */ return (InternalRow) mutableRow; /* 030 */ } /* 031 */ /* 032 */ public java.lang.Object apply(java.lang.Object _i) { /* 033 */ InternalRow i = (InternalRow) _i; /* 034 */ /* 035 */ /* 036 */ long value_1 = i.getLong(0); /* 037 */ boolean isNull_0 = false; /* 038 */ double value_0 = -1.0; /* 039 */ if (!false) { /* 040 */ value_0 = (double) value_1; /* 041 */ } /* 042 */ value_2 = value_0; /* 043 */ /* 044 */ // copy all the results into MutableRow /* 045 */ mutableRow.setDouble(0, value_2); /* 046 */ return mutableRow; /* 047 */ } /* 048 */ /* 049 */ /* 050 */ } 21/01/17 17:43:05.488 Executor task launch worker for task 2.0 in stage 0.0 (TID 2) DEBUG GenerateMutableProjection: code for cast(input[0, bigint, false] as double): /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificMutableProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificMutableProjection extends org.apache.spark.sql.catalyst.expressions.codegen.BaseMutableProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private InternalRow mutableRow; /* 009 */ private double value_2; /* 010 */ /* 011 */ public SpecificMutableProjection(Object[] references) { /* 012 */ this.references = references; /* 013 */ mutableRow = new org.apache.spark.sql.catalyst.expressions.GenericInternalRow(1); /* 014 */ /* 015 */ /* 016 */ } /* 017 */ /* 018 */ public void initialize(int partitionIndex) { /* 019 */ /* 020 */ } /* 021 */ /* 022 */ public org.apache.spark.sql.catalyst.expressions.codegen.BaseMutableProjection target(InternalRow row) { /* 023 */ mutableRow = row; /* 024 */ return this; /* 025 */ } /* 026 */ /* 027 */ /* Provide immutable access to the last projected row. */ /* 028 */ public InternalRow currentValue() { /* 029 */ return (InternalRow) mutableRow; /* 030 */ } /* 031 */ /* 032 */ public java.lang.Object apply(java.lang.Object _i) { /* 033 */ InternalRow i = (InternalRow) _i; /* 034 */ /* 035 */ /* 036 */ long value_1 = i.getLong(0); /* 037 */ boolean isNull_0 = false; /* 038 */ double value_0 = -1.0; /* 039 */ if (!false) { /* 040 */ value_0 = (double) value_1; /* 041 */ } /* 042 */ value_2 = value_0; /* 043 */ /* 044 */ // copy all the results into MutableRow /* 045 */ mutableRow.setDouble(0, value_2); /* 046 */ return mutableRow; /* 047 */ } /* 048 */ /* 049 */ /* 050 */ } 21/01/17 17:43:05.511 Executor task launch worker for task 0.0 in stage 0.0 (TID 0) INFO CodeGenerator: Code generated in 45.206096 ms 21/01/17 17:43:06.159 Executor task launch worker for task 3.0 in stage 0.0 (TID 3) DEBUG GenerateUnsafeProjection: code for input[0, bigint, false],input[1, double, true]: /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificUnsafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificUnsafeProjection extends org.apache.spark.sql.catalyst.expressions.UnsafeProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[] mutableStateArray_0 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[1]; /* 009 */ /* 010 */ public SpecificUnsafeProjection(Object[] references) { /* 011 */ this.references = references; /* 012 */ mutableStateArray_0[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(2, 0); /* 013 */ /* 014 */ } /* 015 */ /* 016 */ public void initialize(int partitionIndex) { /* 017 */ /* 018 */ } /* 019 */ /* 020 */ // Scala.Function1 need this /* 021 */ public java.lang.Object apply(java.lang.Object row) { /* 022 */ return apply((InternalRow) row); /* 023 */ } /* 024 */ /* 025 */ public UnsafeRow apply(InternalRow i) { /* 026 */ mutableStateArray_0[0].reset(); /* 027 */ /* 028 */ /* 029 */ mutableStateArray_0[0].zeroOutNullBytes(); /* 030 */ /* 031 */ long value_0 = i.getLong(0); /* 032 */ mutableStateArray_0[0].write(0, value_0); /* 033 */ /* 034 */ boolean isNull_1 = i.isNullAt(1); /* 035 */ double value_1 = isNull_1 ? /* 036 */ -1.0 : (i.getDouble(1)); /* 037 */ if (isNull_1) { /* 038 */ mutableStateArray_0[0].setNullAt(1); /* 039 */ } else { /* 040 */ mutableStateArray_0[0].write(1, value_1); /* 041 */ } /* 042 */ return (mutableStateArray_0[0].getRow()); /* 043 */ } /* 044 */ /* 045 */ /* 046 */ } 21/01/17 17:43:06.160 Executor task launch worker for task 1.0 in stage 0.0 (TID 1) DEBUG GenerateUnsafeProjection: code for input[0, bigint, false],input[1, double, true]: /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificUnsafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificUnsafeProjection extends org.apache.spark.sql.catalyst.expressions.UnsafeProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[] mutableStateArray_0 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[1]; /* 009 */ /* 010 */ public SpecificUnsafeProjection(Object[] references) { /* 011 */ this.references = references; /* 012 */ mutableStateArray_0[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(2, 0); /* 013 */ /* 014 */ } /* 015 */ /* 016 */ public void initialize(int partitionIndex) { /* 017 */ /* 018 */ } /* 019 */ /* 020 */ // Scala.Function1 need this /* 021 */ public java.lang.Object apply(java.lang.Object row) { /* 022 */ return apply((InternalRow) row); /* 023 */ } /* 024 */ /* 025 */ public UnsafeRow apply(InternalRow i) { /* 026 */ mutableStateArray_0[0].reset(); /* 027 */ /* 028 */ /* 029 */ mutableStateArray_0[0].zeroOutNullBytes(); /* 030 */ /* 031 */ long value_0 = i.getLong(0); /* 032 */ mutableStateArray_0[0].write(0, value_0); /* 033 */ /* 034 */ boolean isNull_1 = i.isNullAt(1); /* 035 */ double value_1 = isNull_1 ? /* 036 */ -1.0 : (i.getDouble(1)); /* 037 */ if (isNull_1) { /* 038 */ mutableStateArray_0[0].setNullAt(1); /* 039 */ } else { /* 040 */ mutableStateArray_0[0].write(1, value_1); /* 041 */ } /* 042 */ return (mutableStateArray_0[0].getRow()); /* 043 */ } /* 044 */ /* 045 */ /* 046 */ } 21/01/17 17:43:06.164 Executor task launch worker for task 3.0 in stage 0.0 (TID 3) DEBUG CodeGenerator: /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificUnsafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificUnsafeProjection extends org.apache.spark.sql.catalyst.expressions.UnsafeProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[] mutableStateArray_0 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[1]; /* 009 */ /* 010 */ public SpecificUnsafeProjection(Object[] references) { /* 011 */ this.references = references; /* 012 */ mutableStateArray_0[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(2, 0); /* 013 */ /* 014 */ } /* 015 */ /* 016 */ public void initialize(int partitionIndex) { /* 017 */ /* 018 */ } /* 019 */ /* 020 */ // Scala.Function1 need this /* 021 */ public java.lang.Object apply(java.lang.Object row) { /* 022 */ return apply((InternalRow) row); /* 023 */ } /* 024 */ /* 025 */ public UnsafeRow apply(InternalRow i) { /* 026 */ mutableStateArray_0[0].reset(); /* 027 */ /* 028 */ /* 029 */ mutableStateArray_0[0].zeroOutNullBytes(); /* 030 */ /* 031 */ long value_0 = i.getLong(0); /* 032 */ mutableStateArray_0[0].write(0, value_0); /* 033 */ /* 034 */ boolean isNull_1 = i.isNullAt(1); /* 035 */ double value_1 = isNull_1 ? /* 036 */ -1.0 : (i.getDouble(1)); /* 037 */ if (isNull_1) { /* 038 */ mutableStateArray_0[0].setNullAt(1); /* 039 */ } else { /* 040 */ mutableStateArray_0[0].write(1, value_1); /* 041 */ } /* 042 */ return (mutableStateArray_0[0].getRow()); /* 043 */ } /* 044 */ /* 045 */ /* 046 */ } 21/01/17 17:43:06.166 Executor task launch worker for task 0.0 in stage 0.0 (TID 0) DEBUG GenerateUnsafeProjection: code for input[0, bigint, false],input[1, double, true]: /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificUnsafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificUnsafeProjection extends org.apache.spark.sql.catalyst.expressions.UnsafeProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[] mutableStateArray_0 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[1]; /* 009 */ /* 010 */ public SpecificUnsafeProjection(Object[] references) { /* 011 */ this.references = references; /* 012 */ mutableStateArray_0[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(2, 0); /* 013 */ /* 014 */ } /* 015 */ /* 016 */ public void initialize(int partitionIndex) { /* 017 */ /* 018 */ } /* 019 */ /* 020 */ // Scala.Function1 need this /* 021 */ public java.lang.Object apply(java.lang.Object row) { /* 022 */ return apply((InternalRow) row); /* 023 */ } /* 024 */ /* 025 */ public UnsafeRow apply(InternalRow i) { /* 026 */ mutableStateArray_0[0].reset(); /* 027 */ /* 028 */ /* 029 */ mutableStateArray_0[0].zeroOutNullBytes(); /* 030 */ /* 031 */ long value_0 = i.getLong(0); /* 032 */ mutableStateArray_0[0].write(0, value_0); /* 033 */ /* 034 */ boolean isNull_1 = i.isNullAt(1); /* 035 */ double value_1 = isNull_1 ? /* 036 */ -1.0 : (i.getDouble(1)); /* 037 */ if (isNull_1) { /* 038 */ mutableStateArray_0[0].setNullAt(1); /* 039 */ } else { /* 040 */ mutableStateArray_0[0].write(1, value_1); /* 041 */ } /* 042 */ return (mutableStateArray_0[0].getRow()); /* 043 */ } /* 044 */ /* 045 */ /* 046 */ } 21/01/17 17:43:06.173 stdout writer for /home/jenkins/venv/py36/bin/python3.6 DEBUG TaskMemoryManager: Task 3 acquired 4.0 MiB for HybridRowQueue(org.apache.spark.memory.TaskMemoryManager@2d51bb0c,/home/jenkins/workspace/spark-master-test-python-arm/python/target/f344877f-8415-4e7f-bb44-a4c9c9d59bb8/spark-8f0ac86b-67bc-4e16-b3da-541fb377adc9,1,org.apache.spark.serializer.SerializerManager@1e88ea0c) 21/01/17 17:43:06.169 Executor task launch worker for task 2.0 in stage 0.0 (TID 2) DEBUG GenerateUnsafeProjection: code for input[0, bigint, false],input[1, double, true]: /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificUnsafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificUnsafeProjection extends org.apache.spark.sql.catalyst.expressions.UnsafeProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[] mutableStateArray_0 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[1]; /* 009 */ /* 010 */ public SpecificUnsafeProjection(Object[] references) { /* 011 */ this.references = references; /* 012 */ mutableStateArray_0[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(2, 0); /* 013 */ /* 014 */ } /* 015 */ /* 016 */ public void initialize(int partitionIndex) { /* 017 */ /* 018 */ } /* 019 */ /* 020 */ // Scala.Function1 need this /* 021 */ public java.lang.Object apply(java.lang.Object row) { /* 022 */ return apply((InternalRow) row); /* 023 */ } /* 024 */ /* 025 */ public UnsafeRow apply(InternalRow i) { /* 026 */ mutableStateArray_0[0].reset(); /* 027 */ /* 028 */ /* 029 */ mutableStateArray_0[0].zeroOutNullBytes(); /* 030 */ /* 031 */ long value_0 = i.getLong(0); /* 032 */ mutableStateArray_0[0].write(0, value_0); /* 033 */ /* 034 */ boolean isNull_1 = i.isNullAt(1); /* 035 */ double value_1 = isNull_1 ? /* 036 */ -1.0 : (i.getDouble(1)); /* 037 */ if (isNull_1) { /* 038 */ mutableStateArray_0[0].setNullAt(1); /* 039 */ } else { /* 040 */ mutableStateArray_0[0].write(1, value_1); /* 041 */ } /* 042 */ return (mutableStateArray_0[0].getRow()); /* 043 */ } /* 044 */ /* 045 */ /* 046 */ } 21/01/17 17:43:06.179 stdout writer for /home/jenkins/venv/py36/bin/python3.6 DEBUG TaskMemoryManager: Task 0 acquired 4.0 MiB for HybridRowQueue(org.apache.spark.memory.TaskMemoryManager@71af70bb,/home/jenkins/workspace/spark-master-test-python-arm/python/target/f344877f-8415-4e7f-bb44-a4c9c9d59bb8/spark-8f0ac86b-67bc-4e16-b3da-541fb377adc9,1,org.apache.spark.serializer.SerializerManager@1e88ea0c) 21/01/17 17:43:06.177 stdout writer for /home/jenkins/venv/py36/bin/python3.6 DEBUG TaskMemoryManager: Task 1 acquired 4.0 MiB for HybridRowQueue(org.apache.spark.memory.TaskMemoryManager@3959cb3,/home/jenkins/workspace/spark-master-test-python-arm/python/target/f344877f-8415-4e7f-bb44-a4c9c9d59bb8/spark-8f0ac86b-67bc-4e16-b3da-541fb377adc9,1,org.apache.spark.serializer.SerializerManager@1e88ea0c) 21/01/17 17:43:06.177 stdout writer for /home/jenkins/venv/py36/bin/python3.6 DEBUG TaskMemoryManager: Task 2 acquired 4.0 MiB for HybridRowQueue(org.apache.spark.memory.TaskMemoryManager@521da512,/home/jenkins/workspace/spark-master-test-python-arm/python/target/f344877f-8415-4e7f-bb44-a4c9c9d59bb8/spark-8f0ac86b-67bc-4e16-b3da-541fb377adc9,1,org.apache.spark.serializer.SerializerManager@1e88ea0c) 21/01/17 17:43:06.200 Executor task launch worker for task 3.0 in stage 0.0 (TID 3) INFO CodeGenerator: Code generated in 38.53566 ms 21/01/17 17:43:06.230 Executor task launch worker for task 2.0 in stage 0.0 (TID 2) INFO PythonUDFRunner: Times: total = 655, boot = 611, init = 43, finish = 1 21/01/17 17:43:06.230 Executor task launch worker for task 1.0 in stage 0.0 (TID 1) INFO PythonUDFRunner: Times: total = 655, boot = 604, init = 50, finish = 1 21/01/17 17:43:06.231 Executor task launch worker for task 3.0 in stage 0.0 (TID 3) INFO PythonUDFRunner: Times: total = 650, boot = 608, init = 42, finish = 0 21/01/17 17:43:06.230 Executor task launch worker for task 0.0 in stage 0.0 (TID 0) INFO PythonUDFRunner: Times: total = 659, boot = 605, init = 54, finish = 0 21/01/17 17:43:06.234 Executor task launch worker for task 0.0 in stage 0.0 (TID 0) DEBUG TaskMemoryManager: Task 0 release 4.0 MiB from HybridRowQueue(org.apache.spark.memory.TaskMemoryManager@71af70bb,/home/jenkins/workspace/spark-master-test-python-arm/python/target/f344877f-8415-4e7f-bb44-a4c9c9d59bb8/spark-8f0ac86b-67bc-4e16-b3da-541fb377adc9,1,org.apache.spark.serializer.SerializerManager@1e88ea0c) 21/01/17 17:43:06.234 Executor task launch worker for task 1.0 in stage 0.0 (TID 1) DEBUG TaskMemoryManager: Task 1 release 4.0 MiB from HybridRowQueue(org.apache.spark.memory.TaskMemoryManager@3959cb3,/home/jenkins/workspace/spark-master-test-python-arm/python/target/f344877f-8415-4e7f-bb44-a4c9c9d59bb8/spark-8f0ac86b-67bc-4e16-b3da-541fb377adc9,1,org.apache.spark.serializer.SerializerManager@1e88ea0c) 21/01/17 17:43:06.234 Executor task launch worker for task 2.0 in stage 0.0 (TID 2) DEBUG TaskMemoryManager: Task 2 release 4.0 MiB from HybridRowQueue(org.apache.spark.memory.TaskMemoryManager@521da512,/home/jenkins/workspace/spark-master-test-python-arm/python/target/f344877f-8415-4e7f-bb44-a4c9c9d59bb8/spark-8f0ac86b-67bc-4e16-b3da-541fb377adc9,1,org.apache.spark.serializer.SerializerManager@1e88ea0c) 21/01/17 17:43:06.234 Executor task launch worker for task 3.0 in stage 0.0 (TID 3) DEBUG TaskMemoryManager: Task 3 release 4.0 MiB from HybridRowQueue(org.apache.spark.memory.TaskMemoryManager@2d51bb0c,/home/jenkins/workspace/spark-master-test-python-arm/python/target/f344877f-8415-4e7f-bb44-a4c9c9d59bb8/spark-8f0ac86b-67bc-4e16-b3da-541fb377adc9,1,org.apache.spark.serializer.SerializerManager@1e88ea0c) 21/01/17 17:43:06.249 Executor task launch worker for task 0.0 in stage 0.0 (TID 0) DEBUG GenerateSafeProjection: code for createexternalrow(input[0, int, false], input[1, int, false], input[2, float, true], StructField(UDF(user),IntegerType,false), StructField(UDF(item),IntegerType,false), StructField(rating,FloatType,true)): /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificSafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificSafeProjection extends org.apache.spark.sql.catalyst.expressions.codegen.BaseProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private InternalRow mutableRow; /* 009 */ /* 010 */ /* 011 */ public SpecificSafeProjection(Object[] references) { /* 012 */ this.references = references; /* 013 */ mutableRow = (InternalRow) references[references.length - 1]; /* 014 */ /* 015 */ } /* 016 */ /* 017 */ public void initialize(int partitionIndex) { /* 018 */ /* 019 */ } /* 020 */ /* 021 */ public java.lang.Object apply(java.lang.Object _i) { /* 022 */ InternalRow i = (InternalRow) _i; /* 023 */ Object[] values_0 = new Object[3]; /* 024 */ /* 025 */ int value_1 = i.getInt(0); /* 026 */ if (false) { /* 027 */ values_0[0] = null; /* 028 */ } else { /* 029 */ values_0[0] = value_1; /* 030 */ } /* 031 */ /* 032 */ int value_2 = i.getInt(1); /* 033 */ if (false) { /* 034 */ values_0[1] = null; /* 035 */ } else { /* 036 */ values_0[1] = value_2; /* 037 */ } /* 038 */ /* 039 */ boolean isNull_3 = i.isNullAt(2); /* 040 */ float value_3 = isNull_3 ? /* 041 */ -1.0f : (i.getFloat(2)); /* 042 */ if (isNull_3) { /* 043 */ values_0[2] = null; /* 044 */ } else { /* 045 */ values_0[2] = value_3; /* 046 */ } /* 047 */ /* 048 */ final org.apache.spark.sql.Row value_0 = new org.apache.spark.sql.catalyst.expressions.GenericRowWithSchema(values_0, ((org.apache.spark.sql.types.StructType) references[0] /* schema */)); /* 049 */ if (false) { /* 050 */ mutableRow.setNullAt(0); /* 051 */ } else { /* 052 */ /* 053 */ mutableRow.update(0, value_0); /* 054 */ } /* 055 */ /* 056 */ return mutableRow; /* 057 */ } /* 058 */ /* 059 */ /* 060 */ } 21/01/17 17:43:06.253 Executor task launch worker for task 0.0 in stage 0.0 (TID 0) DEBUG CodeGenerator: /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificSafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificSafeProjection extends org.apache.spark.sql.catalyst.expressions.codegen.BaseProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private InternalRow mutableRow; /* 009 */ /* 010 */ /* 011 */ public SpecificSafeProjection(Object[] references) { /* 012 */ this.references = references; /* 013 */ mutableRow = (InternalRow) references[references.length - 1]; /* 014 */ /* 015 */ } /* 016 */ /* 017 */ public void initialize(int partitionIndex) { /* 018 */ /* 019 */ } /* 020 */ /* 021 */ public java.lang.Object apply(java.lang.Object _i) { /* 022 */ InternalRow i = (InternalRow) _i; /* 023 */ Object[] values_0 = new Object[3]; /* 024 */ /* 025 */ int value_1 = i.getInt(0); /* 026 */ if (false) { /* 027 */ values_0[0] = null; /* 028 */ } else { /* 029 */ values_0[0] = value_1; /* 030 */ } /* 031 */ /* 032 */ int value_2 = i.getInt(1); /* 033 */ if (false) { /* 034 */ values_0[1] = null; /* 035 */ } else { /* 036 */ values_0[1] = value_2; /* 037 */ } /* 038 */ /* 039 */ boolean isNull_3 = i.isNullAt(2); /* 040 */ float value_3 = isNull_3 ? /* 041 */ -1.0f : (i.getFloat(2)); /* 042 */ if (isNull_3) { /* 043 */ values_0[2] = null; /* 044 */ } else { /* 045 */ values_0[2] = value_3; /* 046 */ } /* 047 */ /* 048 */ final org.apache.spark.sql.Row value_0 = new org.apache.spark.sql.catalyst.expressions.GenericRowWithSchema(values_0, ((org.apache.spark.sql.types.StructType) references[0] /* schema */)); /* 049 */ if (false) { /* 050 */ mutableRow.setNullAt(0); /* 051 */ } else { /* 052 */ /* 053 */ mutableRow.update(0, value_0); /* 054 */ } /* 055 */ /* 056 */ return mutableRow; /* 057 */ } /* 058 */ /* 059 */ /* 060 */ } 21/01/17 17:43:06.255 Executor task launch worker for task 1.0 in stage 0.0 (TID 1) INFO Executor: Finished task 1.0 in stage 0.0 (TID 1). 2189 bytes result sent to driver 21/01/17 17:43:06.255 Executor task launch worker for task 0.0 in stage 0.0 (TID 0) INFO Executor: Finished task 0.0 in stage 0.0 (TID 0). 2169 bytes result sent to driver 21/01/17 17:43:06.256 Executor task launch worker for task 1.0 in stage 0.0 (TID 1) DEBUG ExecutorMetricsPoller: stageTCMP: (0, 0) -> 3 21/01/17 17:43:06.257 Executor task launch worker for task 0.0 in stage 0.0 (TID 0) DEBUG ExecutorMetricsPoller: stageTCMP: (0, 0) -> 2 21/01/17 17:43:06.257 Executor task launch worker for task 3.0 in stage 0.0 (TID 3) INFO Executor: Finished task 3.0 in stage 0.0 (TID 3). 2234 bytes result sent to driver 21/01/17 17:43:06.257 Executor task launch worker for task 3.0 in stage 0.0 (TID 3) DEBUG ExecutorMetricsPoller: stageTCMP: (0, 0) -> 1 21/01/17 17:43:06.258 Executor task launch worker for task 2.0 in stage 0.0 (TID 2) INFO Executor: Finished task 2.0 in stage 0.0 (TID 2). 2177 bytes result sent to driver 21/01/17 17:43:06.259 Executor task launch worker for task 0.0 in stage 0.0 (TID 0) INFO Executor: Finished task 0.0 in stage 0.0 (TID 0). 1928 bytes result sent to driver 21/01/17 17:43:06.259 Executor task launch worker for task 2.0 in stage 0.0 (TID 2) DEBUG ExecutorMetricsPoller: removing (0, 0) from stageTCMP 21/01/17 17:43:06.260 Executor task launch worker for task 0.0 in stage 0.0 (TID 0) DEBUG ExecutorMetricsPoller: removing (0, 0) from stageTCMP 21/01/17 17:43:06.260 dispatcher-event-loop-0 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_0.0, runningTasks: 3 21/01/17 17:43:06.261 dispatcher-event-loop-0 DEBUG TaskSetManager: No tasks for locality level NO_PREF, so moving to locality level ANY 21/01/17 17:43:06.267 task-result-getter-0 INFO TaskSetManager: Finished task 1.0 in stage 0.0 (TID 1) in 1925 ms on 192.168.1.225 (executor driver) (1/4) 21/01/17 17:43:06.273 dag-scheduler-event-loop INFO PythonAccumulatorV2: Connected to AccumulatorServer at host: 127.0.0.1 port: 53131 21/01/17 17:43:06.284 task-result-getter-0 INFO TaskSetManager: Finished task 0.0 in stage 0.0 (TID 0) in 1600 ms on 192.168.1.225 (executor driver) (1/1) 21/01/17 17:43:06.286 Executor task launch worker for task 0.0 in stage 0.0 (TID 0) INFO CodeGenerator: Code generated in 36.049473 ms 21/01/17 17:43:06.289 task-result-getter-0 INFO TaskSchedulerImpl: Removed TaskSet 0.0, whose tasks have all completed, from pool 21/01/17 17:43:06.291 dag-scheduler-event-loop INFO PythonAccumulatorV2: Connected to AccumulatorServer at host: 127.0.0.1 port: 39599 21/01/17 17:43:06.293 task-result-getter-1 INFO TaskSetManager: Finished task 0.0 in stage 0.0 (TID 0) in 2027 ms on 192.168.1.225 (executor driver) (2/4) 21/01/17 17:43:06.293 task-result-getter-2 INFO TaskSetManager: Finished task 3.0 in stage 0.0 (TID 3) in 1952 ms on 192.168.1.225 (executor driver) (3/4) 21/01/17 17:43:06.295 task-result-getter-3 INFO TaskSetManager: Finished task 2.0 in stage 0.0 (TID 2) in 1954 ms on 192.168.1.225 (executor driver) (4/4) 21/01/17 17:43:06.296 task-result-getter-3 INFO TaskSchedulerImpl: Removed TaskSet 0.0, whose tasks have all completed, from pool 21/01/17 17:43:06.298 dag-scheduler-event-loop INFO DAGScheduler: ResultStage 0 (head at MetadataUtils.scala:51) finished in 1.999 s 21/01/17 17:43:06.302 dag-scheduler-event-loop DEBUG DAGScheduler: After removal of stage 0, remaining stages = 0 21/01/17 17:43:06.303 dag-scheduler-event-loop INFO DAGScheduler: Job 0 is finished. Cancelling potential speculative or zombie tasks for this job 21/01/17 17:43:06.303 dag-scheduler-event-loop INFO DAGScheduler: ResultStage 0 ( at /usr/lib/python3.6/runpy.py:85) finished in 2.330 s 21/01/17 17:43:06.304 dag-scheduler-event-loop INFO TaskSchedulerImpl: Killing all running tasks in stage 0: Stage finished 21/01/17 17:43:06.306 Thread-3 INFO DAGScheduler: Job 0 finished: head at MetadataUtils.scala:51, took 2.118630 s 21/01/17 17:43:06.308 dag-scheduler-event-loop DEBUG DAGScheduler: After removal of stage 0, remaining stages = 0 21/01/17 17:43:06.308 dag-scheduler-event-loop INFO DAGScheduler: Job 0 is finished. Cancelling potential speculative or zombie tasks for this job 21/01/17 17:43:06.309 dag-scheduler-event-loop INFO TaskSchedulerImpl: Killing all running tasks in stage 0: Stage finished 21/01/17 17:43:06.316 Thread-3 INFO DAGScheduler: Job 0 finished: at /usr/lib/python3.6/runpy.py:85, took 2.444519 s 21/01/17 17:43:06.322 Executor task launch worker for task 0.0 in stage 0.0 (TID 0) INFO Executor: Finished task 0.0 in stage 0.0 (TID 0). 1997 bytes result sent to driver 21/01/17 17:43:06.343 Thread-3 DEBUG GenerateSafeProjection: code for createexternalrow(newInstance(class org.apache.spark.ml.linalg.VectorUDT).deserialize, StructField(features,org.apache.spark.ml.linalg.VectorUDT@3bfc3ba7,true)): /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificSafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificSafeProjection extends org.apache.spark.sql.catalyst.expressions.codegen.BaseProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private InternalRow mutableRow; /* 009 */ private boolean resultIsNull_0; /* 010 */ private InternalRow[] mutableStateArray_0 = new InternalRow[1]; /* 011 */ /* 012 */ public SpecificSafeProjection(Object[] references) { /* 013 */ this.references = references; /* 014 */ mutableRow = (InternalRow) references[references.length - 1]; /* 015 */ /* 016 */ /* 017 */ } /* 018 */ /* 019 */ public void initialize(int partitionIndex) { /* 020 */ /* 021 */ } /* 022 */ /* 023 */ public java.lang.Object apply(java.lang.Object _i) { /* 024 */ InternalRow i = (InternalRow) _i; /* 025 */ org.apache.spark.sql.Row value_4 = CreateExternalRow_0(i); /* 026 */ if (false) { /* 027 */ mutableRow.setNullAt(0); /* 028 */ } else { /* 029 */ /* 030 */ mutableRow.update(0, value_4); /* 031 */ } /* 032 */ /* 033 */ return mutableRow; /* 034 */ } /* 035 */ /* 036 */ /* 037 */ private org.apache.spark.sql.Row CreateExternalRow_0(InternalRow i) { /* 038 */ Object[] values_0 = new Object[1]; /* 039 */ /* 040 */ final org.apache.spark.ml.linalg.VectorUDT value_2 = false ? /* 041 */ null : new org.apache.spark.ml.linalg.VectorUDT(); /* 042 */ boolean isNull_1 = true; /* 043 */ org.apache.spark.ml.linalg.Vector value_1 = null; /* 044 */ if (!false) { /* 045 */ resultIsNull_0 = false; /* 046 */ if (!resultIsNull_0) { /* 047 */ boolean isNull_3 = i.isNullAt(0); /* 048 */ InternalRow value_3 = isNull_3 ? /* 049 */ null : (i.getStruct(0, 4)); /* 050 */ resultIsNull_0 = isNull_3; /* 051 */ mutableStateArray_0[0] = value_3; /* 052 */ } /* 053 */ /* 054 */ isNull_1 = resultIsNull_0; /* 055 */ if (!isNull_1) { /* 056 */ /* 057 */ Object funcResult_0 = null; /* 058 */ funcResult_0 = value_2.deserialize(mutableStateArray_0[0]); /* 059 */ /* 060 */ if (funcResult_0 != null) { /* 061 */ value_1 = (org.apache.spark.ml.linalg.Vector) funcResult_0; /* 062 */ } else { /* 063 */ isNull_1 = true; /* 064 */ } /* 065 */ /* 066 */ /* 067 */ } /* 068 */ } /* 069 */ if (isNull_1) { /* 070 */ values_0[0] = null; /* 071 */ } else { /* 072 */ values_0[0] = value_1; /* 073 */ } /* 074 */ /* 075 */ final org.apache.spark.sql.Row value_0 = new org.apache.spark.sql.catalyst.expressions.GenericRowWithSchema(values_0, ((org.apache.spark.sql.types.StructType) references[0] /* schema */)); /* 076 */ /* 077 */ return value_0; /* 078 */ } /* 079 */ /* 080 */ } 21/01/17 17:43:06.346 Thread-3 DEBUG CodeGenerator: /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificSafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificSafeProjection extends org.apache.spark.sql.catalyst.expressions.codegen.BaseProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private InternalRow mutableRow; /* 009 */ private boolean resultIsNull_0; /* 010 */ private InternalRow[] mutableStateArray_0 = new InternalRow[1]; /* 011 */ /* 012 */ public SpecificSafeProjection(Object[] references) { /* 013 */ this.references = references; /* 014 */ mutableRow = (InternalRow) references[references.length - 1]; /* 015 */ /* 016 */ /* 017 */ } /* 018 */ /* 019 */ public void initialize(int partitionIndex) { /* 020 */ /* 021 */ } /* 022 */ /* 023 */ public java.lang.Object apply(java.lang.Object _i) { /* 024 */ InternalRow i = (InternalRow) _i; /* 025 */ org.apache.spark.sql.Row value_4 = CreateExternalRow_0(i); /* 026 */ if (false) { /* 027 */ mutableRow.setNullAt(0); /* 028 */ } else { /* 029 */ /* 030 */ mutableRow.update(0, value_4); /* 031 */ } /* 032 */ /* 033 */ return mutableRow; /* 034 */ } /* 035 */ /* 036 */ /* 037 */ private org.apache.spark.sql.Row CreateExternalRow_0(InternalRow i) { /* 038 */ Object[] values_0 = new Object[1]; /* 039 */ /* 040 */ final org.apache.spark.ml.linalg.VectorUDT value_2 = false ? /* 041 */ null : new org.apache.spark.ml.linalg.VectorUDT(); /* 042 */ boolean isNull_1 = true; /* 043 */ org.apache.spark.ml.linalg.Vector value_1 = null; /* 044 */ if (!false) { /* 045 */ resultIsNull_0 = false; /* 046 */ if (!resultIsNull_0) { /* 047 */ boolean isNull_3 = i.isNullAt(0); /* 048 */ InternalRow value_3 = isNull_3 ? /* 049 */ null : (i.getStruct(0, 4)); /* 050 */ resultIsNull_0 = isNull_3; /* 051 */ mutableStateArray_0[0] = value_3; /* 052 */ } /* 053 */ /* 054 */ isNull_1 = resultIsNull_0; /* 055 */ if (!isNull_1) { /* 056 */ /* 057 */ Object funcResult_0 = null; /* 058 */ funcResult_0 = value_2.deserialize(mutableStateArray_0[0]); /* 059 */ /* 060 */ if (funcResult_0 != null) { /* 061 */ value_1 = (org.apache.spark.ml.linalg.Vector) funcResult_0; /* 062 */ } else { /* 063 */ isNull_1 = true; /* 064 */ } /* 065 */ /* 066 */ /* 067 */ } /* 068 */ } /* 069 */ if (isNull_1) { /* 070 */ values_0[0] = null; /* 071 */ } else { /* 072 */ values_0[0] = value_1; /* 073 */ } /* 074 */ /* 075 */ final org.apache.spark.sql.Row value_0 = new org.apache.spark.sql.catalyst.expressions.GenericRowWithSchema(values_0, ((org.apache.spark.sql.types.StructType) references[0] /* schema */)); /* 076 */ /* 077 */ return value_0; /* 078 */ } /* 079 */ /* 080 */ } 21/01/17 17:43:06.382 dispatcher-event-loop-3 INFO MapOutputTrackerMasterEndpoint: MapOutputTrackerMasterEndpoint stopped! 21/01/17 17:43:06.391 Thread-3 INFO CodeGenerator: Code generated in 47.614682 ms 21/01/17 17:43:06.406 Thread-3 INFO MemoryStore: MemoryStore cleared 21/01/17 17:43:06.407 Thread-3 INFO BlockManager: BlockManager stopped 21/01/17 17:43:06.415 Thread-3 INFO BlockManagerMaster: BlockManagerMaster stopped 21/01/17 17:43:06.415 Thread-3 DEBUG Analyzer$ResolveReferences: Resolving 'prediction to prediction#1 21/01/17 17:43:06.416 Thread-3 DEBUG Analyzer$ResolveReferences: Resolving 'normalizedFeatures to normalizedFeatures#11 21/01/17 17:43:06.417 Thread-3 DEBUG Analyzer$ResolveReferences: Resolving 'weightCol to weightCol#5 21/01/17 17:43:06.418 dispatcher-event-loop-3 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint: OutputCommitCoordinator stopped! 21/01/17 17:43:06.495 Thread-3 INFO SparkContext: Successfully stopped SparkContext 21/01/17 17:43:06.512 Executor task launch worker for task 0.0 in stage 0.0 (TID 0) DEBUG ExecutorMetricsPoller: removing (0, 0) from stageTCMP 21/01/17 17:43:06.520 task-result-getter-0 INFO TaskSetManager: Finished task 0.0 in stage 0.0 (TID 0) in 1836 ms on 192.168.1.225 (executor driver) (1/1) 21/01/17 17:43:06.522 task-result-getter-0 INFO TaskSchedulerImpl: Removed TaskSet 0.0, whose tasks have all completed, from pool 21/01/17 17:43:06.526 dag-scheduler-event-loop INFO PythonAccumulatorV2: Connected to AccumulatorServer at host: 127.0.0.1 port: 32877 21/01/17 17:43:06.531 dag-scheduler-event-loop INFO DAGScheduler: ResultStage 0 (isEmpty at ALS.scala:960) finished in 2.062 s 21/01/17 17:43:06.535 dag-scheduler-event-loop DEBUG DAGScheduler: After removal of stage 0, remaining stages = 0 21/01/17 17:43:06.536 dag-scheduler-event-loop INFO DAGScheduler: Job 0 is finished. Cancelling potential speculative or zombie tasks for this job 21/01/17 17:43:06.536 dag-scheduler-event-loop INFO TaskSchedulerImpl: Killing all running tasks in stage 0: Stage finished 21/01/17 17:43:06.539 Thread-3 INFO DAGScheduler: Job 0 finished: isEmpty at ALS.scala:960, took 2.236995 s 21/01/17 17:43:06.546 Thread-3 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$partitionRatings$1 21/01/17 17:43:06.548 Thread-3 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$partitionRatings$1) is now cleaned +++ 21/01/17 17:43:06.571 Thread-3 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$groupByKey$2 21/01/17 17:43:06.575 Thread-3 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$groupByKey$2) is now cleaned +++ 21/01/17 17:43:06.586 Thread-3 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$groupByKey$3 21/01/17 17:43:06.589 Thread-3 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$groupByKey$3) is now cleaned +++ 21/01/17 17:43:06.590 Thread-3 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$groupByKey$4 21/01/17 17:43:06.594 Thread-3 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$groupByKey$4) is now cleaned +++ 21/01/17 17:43:06.597 Thread-3 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$partitionRatings$7 21/01/17 17:43:06.599 Thread-3 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$partitionRatings$7) is now cleaned +++ 21/01/17 17:43:06.605 Thread-3 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$makeBlocks$1 21/01/17 17:43:06.607 Thread-3 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$makeBlocks$1) is now cleaned +++ 21/01/17 17:43:06.609 Thread-3 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$groupByKey$2 21/01/17 17:43:06.612 Thread-3 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$groupByKey$2) is now cleaned +++ 21/01/17 17:43:06.626 Thread-3 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$groupByKey$3 21/01/17 17:43:06.630 Thread-3 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$groupByKey$3) is now cleaned +++ 21/01/17 17:43:06.630 Thread-3 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$groupByKey$4 21/01/17 17:43:06.634 Thread-3 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$groupByKey$4) is now cleaned +++ 21/01/17 17:43:06.636 Thread-3 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$makeBlocks$5 21/01/17 17:43:06.638 Thread-3 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$makeBlocks$5) is now cleaned +++ 21/01/17 17:43:06.641 Thread-3 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$makeBlocks$7 21/01/17 17:43:06.644 Thread-3 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$makeBlocks$7) is now cleaned +++ 21/01/17 17:43:06.648 Thread-3 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$count$1$adapted 21/01/17 17:43:06.655 Thread-3 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$count$1$adapted) is now cleaned +++ 21/01/17 17:43:06.656 Thread-3 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$runJob$5 21/01/17 17:43:06.663 Thread-3 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$runJob$5) is now cleaned +++ 21/01/17 17:43:06.663 Thread-3 INFO SparkContext: Starting job: count at ALS.scala:973 21/01/17 17:43:06.669 dag-scheduler-event-loop DEBUG DAGScheduler: Merging stage rdd profiles: Set() 21/01/17 17:43:06.673 dag-scheduler-event-loop DEBUG DAGScheduler: Merging stage rdd profiles: Set() 21/01/17 17:43:06.675 dag-scheduler-event-loop INFO DAGScheduler: Registering RDD 14 (mapPartitions at ALS.scala:1369) as input to shuffle 1 21/01/17 17:43:06.677 dag-scheduler-event-loop DEBUG DAGScheduler: Merging stage rdd profiles: Set() 21/01/17 17:43:06.678 dag-scheduler-event-loop INFO DAGScheduler: Registering RDD 17 (map at ALS.scala:1612) as input to shuffle 0 21/01/17 17:43:06.678 dag-scheduler-event-loop INFO DAGScheduler: Got job 1 (count at ALS.scala:973) with 10 output partitions 21/01/17 17:43:06.678 dag-scheduler-event-loop INFO DAGScheduler: Final stage: ResultStage 3 (count at ALS.scala:973) 21/01/17 17:43:06.679 dag-scheduler-event-loop INFO DAGScheduler: Parents of final stage: List(ShuffleMapStage 2) 21/01/17 17:43:06.684 dag-scheduler-event-loop INFO DAGScheduler: Missing parents: List(ShuffleMapStage 2) 21/01/17 17:43:06.688 dag-scheduler-event-loop DEBUG DAGScheduler: submitStage(ResultStage 3 (name=count at ALS.scala:973;jobs=1)) 21/01/17 17:43:06.688 dag-scheduler-event-loop DEBUG DAGScheduler: missing: List(ShuffleMapStage 2) 21/01/17 17:43:06.689 dag-scheduler-event-loop DEBUG DAGScheduler: submitStage(ShuffleMapStage 2 (name=map at ALS.scala:1612;jobs=1)) 21/01/17 17:43:06.690 dag-scheduler-event-loop DEBUG DAGScheduler: missing: List(ShuffleMapStage 1) 21/01/17 17:43:06.690 dag-scheduler-event-loop DEBUG DAGScheduler: submitStage(ShuffleMapStage 1 (name=mapPartitions at ALS.scala:1369;jobs=1)) 21/01/17 17:43:06.690 dag-scheduler-event-loop DEBUG DAGScheduler: missing: List() 21/01/17 17:43:06.691 dag-scheduler-event-loop INFO DAGScheduler: Submitting ShuffleMapStage 1 (MapPartitionsRDD[14] at mapPartitions at ALS.scala:1369), which has no missing parents 21/01/17 17:43:06.691 dag-scheduler-event-loop DEBUG DAGScheduler: submitMissingTasks(ShuffleMapStage 1) 21/01/17 17:43:06.714 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_1 stored as values in memory (estimated size 31.8 KiB, free 366.2 MiB) 21/01/17 17:43:06.719 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_1 locally took 5 ms 21/01/17 17:43:06.719 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_1 without replication took 6 ms 21/01/17 17:43:06.721 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_1_piece0 stored as bytes in memory (estimated size 14.9 KiB, free 366.2 MiB) 21/01/17 17:43:06.723 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master broadcast_1_piece0 for BlockManagerId(driver, 192.168.1.225, 40985, None) 21/01/17 17:43:06.723 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added broadcast_1_piece0 in memory on 192.168.1.225:40985 (size: 14.9 KiB, free: 366.3 MiB) 21/01/17 17:43:06.724 dag-scheduler-event-loop DEBUG BlockManagerMaster: Updated info of block broadcast_1_piece0 21/01/17 17:43:06.724 dag-scheduler-event-loop DEBUG BlockManager: Told master about block broadcast_1_piece0 21/01/17 17:43:06.724 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_1_piece0 locally took 4 ms 21/01/17 17:43:06.724 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_1_piece0 without replication took 4 ms 21/01/17 17:43:06.725 dag-scheduler-event-loop INFO SparkContext: Created broadcast 1 from broadcast at DAGScheduler.scala:1383 21/01/17 17:43:06.729 dag-scheduler-event-loop INFO DAGScheduler: Submitting 4 missing tasks from ShuffleMapStage 1 (MapPartitionsRDD[14] at mapPartitions at ALS.scala:1369) (first 15 tasks are for partitions Vector(0, 1, 2, 3)) 21/01/17 17:43:06.729 dag-scheduler-event-loop INFO TaskSchedulerImpl: Adding task set 1.0 with 4 tasks resource profile 0 21/01/17 17:43:06.731 dag-scheduler-event-loop DEBUG TaskSetManager: Epoch for TaskSet 1.0: 0 21/01/17 17:43:06.731 dag-scheduler-event-loop DEBUG TaskSetManager: Adding pending tasks took 0 ms 21/01/17 17:43:06.731 dag-scheduler-event-loop DEBUG TaskSetManager: Valid locality levels for TaskSet 1.0: NO_PREF, ANY 21/01/17 17:43:06.732 dispatcher-event-loop-1 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_1.0, runningTasks: 0 21/01/17 17:43:06.734 dispatcher-event-loop-1 INFO TaskSetManager: Starting task 0.0 in stage 1.0 (TID 1) (192.168.1.225, executor driver, partition 0, PROCESS_LOCAL, 7362 bytes) taskResourceAssignments Map() 21/01/17 17:43:06.734 dispatcher-event-loop-1 INFO TaskSetManager: Starting task 1.0 in stage 1.0 (TID 2) (192.168.1.225, executor driver, partition 1, PROCESS_LOCAL, 7402 bytes) taskResourceAssignments Map() 21/01/17 17:43:06.735 dispatcher-event-loop-1 INFO TaskSetManager: Starting task 2.0 in stage 1.0 (TID 3) (192.168.1.225, executor driver, partition 2, PROCESS_LOCAL, 7362 bytes) taskResourceAssignments Map() 21/01/17 17:43:06.735 dispatcher-event-loop-1 INFO TaskSetManager: Starting task 3.0 in stage 1.0 (TID 4) (192.168.1.225, executor driver, partition 3, PROCESS_LOCAL, 7402 bytes) taskResourceAssignments Map() 21/01/17 17:43:06.736 Executor task launch worker for task 0.0 in stage 1.0 (TID 1) INFO Executor: Running task 0.0 in stage 1.0 (TID 1) 21/01/17 17:43:06.737 Executor task launch worker for task 0.0 in stage 1.0 (TID 1) DEBUG ExecutorMetricsPoller: stageTCMP: (1, 0) -> 1 21/01/17 17:43:06.738 Executor task launch worker for task 1.0 in stage 1.0 (TID 2) INFO Executor: Running task 1.0 in stage 1.0 (TID 2) 21/01/17 17:43:06.739 Executor task launch worker for task 0.0 in stage 1.0 (TID 1) DEBUG BlockManager: Getting local block broadcast_1 21/01/17 17:43:06.739 Executor task launch worker for task 0.0 in stage 1.0 (TID 1) DEBUG BlockManager: Level for block broadcast_1 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:06.747 Executor task launch worker for task 0.0 in stage 0.0 (TID 0) INFO PythonRunner: Times: total = 670, boot = 661, init = 9, finish = 0 21/01/17 17:43:06.739 Executor task launch worker for task 1.0 in stage 1.0 (TID 2) DEBUG ExecutorMetricsPoller: stageTCMP: (1, 0) -> 2 21/01/17 17:43:06.761 Executor task launch worker for task 1.0 in stage 0.0 (TID 1) DEBUG GenerateSafeProjection: code for mapobjects(lambdavariable(MapObject, StringType, true, -1), lambdavariable(MapObject, StringType, true, -1).toString, input[0, array, true], Some(interface scala.collection.Seq)): /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificSafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificSafeProjection extends org.apache.spark.sql.catalyst.expressions.codegen.BaseProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private InternalRow mutableRow; /* 009 */ private UTF8String value_MapObject_lambda_variable_1; /* 010 */ private boolean isNull_MapObject_lambda_variable_1; /* 011 */ private boolean globalIsNull_0; /* 012 */ /* 013 */ public SpecificSafeProjection(Object[] references) { /* 014 */ this.references = references; /* 015 */ mutableRow = (InternalRow) references[references.length - 1]; /* 016 */ /* 017 */ /* 018 */ } /* 019 */ /* 020 */ public void initialize(int partitionIndex) { /* 021 */ /* 022 */ } /* 023 */ /* 024 */ public java.lang.Object apply(java.lang.Object _i) { /* 025 */ InternalRow i = (InternalRow) _i; /* 026 */ scala.collection.Seq value_3 = MapObjects_0(i); /* 027 */ if (globalIsNull_0) { /* 028 */ mutableRow.setNullAt(0); /* 029 */ } else { /* 030 */ /* 031 */ mutableRow.update(0, value_3); /* 032 */ } /* 033 */ /* 034 */ return mutableRow; /* 035 */ } /* 036 */ /* 037 */ /* 038 */ private scala.collection.Seq MapObjects_0(InternalRow i) { /* 039 */ boolean isNull_1 = i.isNullAt(0); /* 040 */ ArrayData value_1 = isNull_1 ? /* 041 */ null : (i.getArray(0)); /* 042 */ scala.collection.Seq value_0 = null; /* 043 */ /* 044 */ if (!isNull_1) { /* 045 */ /* 046 */ int dataLength_0 = value_1.numElements(); /* 047 */ /* 048 */ scala.collection.mutable.Builder collectionBuilder_0 = scala.collection.Seq$.MODULE$.newBuilder(); /* 049 */ collectionBuilder_0.sizeHint(dataLength_0); /* 050 */ /* 051 */ /* 052 */ int loopIndex_0 = 0; /* 053 */ /* 054 */ while (loopIndex_0 < dataLength_0) { /* 055 */ value_MapObject_lambda_variable_1 = (UTF8String) (value_1.getUTF8String(loopIndex_0)); /* 056 */ isNull_MapObject_lambda_variable_1 = value_1.isNullAt(loopIndex_0); /* 057 */ /* 058 */ boolean isNull_2 = true; /* 059 */ java.lang.String value_2 = null; /* 060 */ if (!isNull_MapObject_lambda_variable_1) { /* 061 */ /* 062 */ isNull_2 = false; /* 063 */ if (!isNull_2) { /* 064 */ /* 065 */ Object funcResult_0 = null; /* 066 */ funcResult_0 = value_MapObject_lambda_variable_1.toString(); /* 067 */ value_2 = (java.lang.String) funcResult_0; /* 068 */ /* 069 */ } /* 070 */ } /* 071 */ if (isNull_2) { /* 072 */ collectionBuilder_0.$plus$eq(null); /* 073 */ } else { /* 074 */ collectionBuilder_0.$plus$eq(value_2); /* 075 */ } /* 076 */ /* 077 */ loopIndex_0 += 1; /* 078 */ } /* 079 */ /* 080 */ value_0 = (scala.collection.Seq) collectionBuilder_0.result(); /* 081 */ } /* 082 */ globalIsNull_0 = isNull_1; /* 083 */ return value_0; /* 084 */ } /* 085 */ /* 086 */ } 21/01/17 17:43:06.764 Executor task launch worker for task 1.0 in stage 0.0 (TID 1) DEBUG CodeGenerator: /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificSafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificSafeProjection extends org.apache.spark.sql.catalyst.expressions.codegen.BaseProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private InternalRow mutableRow; /* 009 */ private UTF8String value_MapObject_lambda_variable_1; /* 010 */ private boolean isNull_MapObject_lambda_variable_1; /* 011 */ private boolean globalIsNull_0; /* 012 */ /* 013 */ public SpecificSafeProjection(Object[] references) { /* 014 */ this.references = references; /* 015 */ mutableRow = (InternalRow) references[references.length - 1]; /* 016 */ /* 017 */ /* 018 */ } /* 019 */ /* 020 */ public void initialize(int partitionIndex) { /* 021 */ /* 022 */ } /* 023 */ /* 024 */ public java.lang.Object apply(java.lang.Object _i) { /* 025 */ InternalRow i = (InternalRow) _i; /* 026 */ scala.collection.Seq value_3 = MapObjects_0(i); /* 027 */ if (globalIsNull_0) { /* 028 */ mutableRow.setNullAt(0); /* 029 */ } else { /* 030 */ /* 031 */ mutableRow.update(0, value_3); /* 032 */ } /* 033 */ /* 034 */ return mutableRow; /* 035 */ } /* 036 */ /* 037 */ /* 038 */ private scala.collection.Seq MapObjects_0(InternalRow i) { /* 039 */ boolean isNull_1 = i.isNullAt(0); /* 040 */ ArrayData value_1 = isNull_1 ? /* 041 */ null : (i.getArray(0)); /* 042 */ scala.collection.Seq value_0 = null; /* 043 */ /* 044 */ if (!isNull_1) { /* 045 */ /* 046 */ int dataLength_0 = value_1.numElements(); /* 047 */ /* 048 */ scala.collection.mutable.Builder collectionBuilder_0 = scala.collection.Seq$.MODULE$.newBuilder(); /* 049 */ collectionBuilder_0.sizeHint(dataLength_0); /* 050 */ /* 051 */ /* 052 */ int loopIndex_0 = 0; /* 053 */ /* 054 */ while (loopIndex_0 < dataLength_0) { /* 055 */ value_MapObject_lambda_variable_1 = (UTF8String) (value_1.getUTF8String(loopIndex_0)); /* 056 */ isNull_MapObject_lambda_variable_1 = value_1.isNullAt(loopIndex_0); /* 057 */ /* 058 */ boolean isNull_2 = true; /* 059 */ java.lang.String value_2 = null; /* 060 */ if (!isNull_MapObject_lambda_variable_1) { /* 061 */ /* 062 */ isNull_2 = false; /* 063 */ if (!isNull_2) { /* 064 */ /* 065 */ Object funcResult_0 = null; /* 066 */ funcResult_0 = value_MapObject_lambda_variable_1.toString(); /* 067 */ value_2 = (java.lang.String) funcResult_0; /* 068 */ /* 069 */ } /* 070 */ } /* 071 */ if (isNull_2) { /* 072 */ collectionBuilder_0.$plus$eq(null); /* 073 */ } else { /* 074 */ collectionBuilder_0.$plus$eq(value_2); /* 075 */ } /* 076 */ /* 077 */ loopIndex_0 += 1; /* 078 */ } /* 079 */ /* 080 */ value_0 = (scala.collection.Seq) collectionBuilder_0.result(); /* 081 */ } /* 082 */ globalIsNull_0 = isNull_1; /* 083 */ return value_0; /* 084 */ } /* 085 */ /* 086 */ } 21/01/17 17:43:06.780 Executor task launch worker for task 3.0 in stage 1.0 (TID 4) INFO Executor: Running task 3.0 in stage 1.0 (TID 4) 21/01/17 17:43:06.781 Executor task launch worker for task 3.0 in stage 1.0 (TID 4) DEBUG ExecutorMetricsPoller: stageTCMP: (1, 0) -> 3 21/01/17 17:43:06.767 Executor task launch worker for task 3.0 in stage 0.0 (TID 3) DEBUG GenerateSafeProjection: code for mapobjects(lambdavariable(MapObject, StringType, true, -1), lambdavariable(MapObject, StringType, true, -1).toString, input[0, array, true], Some(interface scala.collection.Seq)): /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificSafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificSafeProjection extends org.apache.spark.sql.catalyst.expressions.codegen.BaseProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private InternalRow mutableRow; /* 009 */ private UTF8String value_MapObject_lambda_variable_1; /* 010 */ private boolean isNull_MapObject_lambda_variable_1; /* 011 */ private boolean globalIsNull_0; /* 012 */ /* 013 */ public SpecificSafeProjection(Object[] references) { /* 014 */ this.references = references; /* 015 */ mutableRow = (InternalRow) references[references.length - 1]; /* 016 */ /* 017 */ /* 018 */ } /* 019 */ /* 020 */ public void initialize(int partitionIndex) { /* 021 */ /* 022 */ } /* 023 */ /* 024 */ public java.lang.Object apply(java.lang.Object _i) { /* 025 */ InternalRow i = (InternalRow) _i; /* 026 */ scala.collection.Seq value_3 = MapObjects_0(i); /* 027 */ if (globalIsNull_0) { /* 028 */ mutableRow.setNullAt(0); /* 029 */ } else { /* 030 */ /* 031 */ mutableRow.update(0, value_3); /* 032 */ } /* 033 */ /* 034 */ return mutableRow; /* 035 */ } /* 036 */ /* 037 */ /* 038 */ private scala.collection.Seq MapObjects_0(InternalRow i) { /* 039 */ boolean isNull_1 = i.isNullAt(0); /* 040 */ ArrayData value_1 = isNull_1 ? /* 041 */ null : (i.getArray(0)); /* 042 */ scala.collection.Seq value_0 = null; /* 043 */ /* 044 */ if (!isNull_1) { /* 045 */ /* 046 */ int dataLength_0 = value_1.numElements(); /* 047 */ /* 048 */ scala.collection.mutable.Builder collectionBuilder_0 = scala.collection.Seq$.MODULE$.newBuilder(); /* 049 */ collectionBuilder_0.sizeHint(dataLength_0); /* 050 */ /* 051 */ /* 052 */ int loopIndex_0 = 0; /* 053 */ /* 054 */ while (loopIndex_0 < dataLength_0) { /* 055 */ value_MapObject_lambda_variable_1 = (UTF8String) (value_1.getUTF8String(loopIndex_0)); /* 056 */ isNull_MapObject_lambda_variable_1 = value_1.isNullAt(loopIndex_0); /* 057 */ /* 058 */ boolean isNull_2 = true; /* 059 */ java.lang.String value_2 = null; /* 060 */ if (!isNull_MapObject_lambda_variable_1) { /* 061 */ /* 062 */ isNull_2 = false; /* 063 */ if (!isNull_2) { /* 064 */ /* 065 */ Object funcResult_0 = null; /* 066 */ funcResult_0 = value_MapObject_lambda_variable_1.toString(); /* 067 */ value_2 = (java.lang.String) funcResult_0; /* 068 */ /* 069 */ } /* 070 */ } /* 071 */ if (isNull_2) { /* 072 */ collectionBuilder_0.$plus$eq(null); /* 073 */ } else { /* 074 */ collectionBuilder_0.$plus$eq(value_2); /* 075 */ } /* 076 */ /* 077 */ loopIndex_0 += 1; /* 078 */ } /* 079 */ /* 080 */ value_0 = (scala.collection.Seq) collectionBuilder_0.result(); /* 081 */ } /* 082 */ globalIsNull_0 = isNull_1; /* 083 */ return value_0; /* 084 */ } /* 085 */ /* 086 */ } 21/01/17 17:43:06.791 Executor task launch worker for task 2.0 in stage 1.0 (TID 3) INFO Executor: Running task 2.0 in stage 1.0 (TID 3) 21/01/17 17:43:06.792 Executor task launch worker for task 2.0 in stage 1.0 (TID 3) DEBUG ExecutorMetricsPoller: stageTCMP: (1, 0) -> 4 21/01/17 17:43:06.793 Executor task launch worker for task 2.0 in stage 0.0 (TID 2) DEBUG GenerateSafeProjection: code for mapobjects(lambdavariable(MapObject, StringType, true, -1), lambdavariable(MapObject, StringType, true, -1).toString, input[0, array, true], Some(interface scala.collection.Seq)): /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificSafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificSafeProjection extends org.apache.spark.sql.catalyst.expressions.codegen.BaseProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private InternalRow mutableRow; /* 009 */ private UTF8String value_MapObject_lambda_variable_1; /* 010 */ private boolean isNull_MapObject_lambda_variable_1; /* 011 */ private boolean globalIsNull_0; /* 012 */ /* 013 */ public SpecificSafeProjection(Object[] references) { /* 014 */ this.references = references; /* 015 */ mutableRow = (InternalRow) references[references.length - 1]; /* 016 */ /* 017 */ /* 018 */ } /* 019 */ /* 020 */ public void initialize(int partitionIndex) { /* 021 */ /* 022 */ } /* 023 */ /* 024 */ public java.lang.Object apply(java.lang.Object _i) { /* 025 */ InternalRow i = (InternalRow) _i; /* 026 */ scala.collection.Seq value_3 = MapObjects_0(i); /* 027 */ if (globalIsNull_0) { /* 028 */ mutableRow.setNullAt(0); /* 029 */ } else { /* 030 */ /* 031 */ mutableRow.update(0, value_3); /* 032 */ } /* 033 */ /* 034 */ return mutableRow; /* 035 */ } /* 036 */ /* 037 */ /* 038 */ private scala.collection.Seq MapObjects_0(InternalRow i) { /* 039 */ boolean isNull_1 = i.isNullAt(0); /* 040 */ ArrayData value_1 = isNull_1 ? /* 041 */ null : (i.getArray(0)); /* 042 */ scala.collection.Seq value_0 = null; /* 043 */ /* 044 */ if (!isNull_1) { /* 045 */ /* 046 */ int dataLength_0 = value_1.numElements(); /* 047 */ /* 048 */ scala.collection.mutable.Builder collectionBuilder_0 = scala.collection.Seq$.MODULE$.newBuilder(); /* 049 */ collectionBuilder_0.sizeHint(dataLength_0); /* 050 */ /* 051 */ /* 052 */ int loopIndex_0 = 0; /* 053 */ /* 054 */ while (loopIndex_0 < dataLength_0) { /* 055 */ value_MapObject_lambda_variable_1 = (UTF8String) (value_1.getUTF8String(loopIndex_0)); /* 056 */ isNull_MapObject_lambda_variable_1 = value_1.isNullAt(loopIndex_0); /* 057 */ /* 058 */ boolean isNull_2 = true; /* 059 */ java.lang.String value_2 = null; /* 060 */ if (!isNull_MapObject_lambda_variable_1) { /* 061 */ /* 062 */ isNull_2 = false; /* 063 */ if (!isNull_2) { /* 064 */ /* 065 */ Object funcResult_0 = null; /* 066 */ funcResult_0 = value_MapObject_lambda_variable_1.toString(); /* 067 */ value_2 = (java.lang.String) funcResult_0; /* 068 */ /* 069 */ } /* 070 */ } /* 071 */ if (isNull_2) { /* 072 */ collectionBuilder_0.$plus$eq(null); /* 073 */ } else { /* 074 */ collectionBuilder_0.$plus$eq(value_2); /* 075 */ } /* 076 */ /* 077 */ loopIndex_0 += 1; /* 078 */ } /* 079 */ /* 080 */ value_0 = (scala.collection.Seq) collectionBuilder_0.result(); /* 081 */ } /* 082 */ globalIsNull_0 = isNull_1; /* 083 */ return value_0; /* 084 */ } /* 085 */ /* 086 */ } 21/01/17 17:43:06.798 Executor task launch worker for task 0.0 in stage 0.0 (TID 0) INFO Executor: Finished task 0.0 in stage 0.0 (TID 0). 2085 bytes result sent to driver 21/01/17 17:43:06.805 Executor task launch worker for task 0.0 in stage 0.0 (TID 0) DEBUG ExecutorMetricsPoller: stageTCMP: (0, 0) -> 3 21/01/17 17:43:06.809 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanBroadcast(0) 21/01/17 17:43:06.810 Spark Context Cleaner DEBUG ContextCleaner: Cleaning broadcast 0 21/01/17 17:43:06.811 Spark Context Cleaner DEBUG TorrentBroadcast: Unpersisting TorrentBroadcast 0 21/01/17 17:43:06.807 dispatcher-event-loop-3 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_0.0, runningTasks: 3 21/01/17 17:43:06.817 dispatcher-event-loop-3 DEBUG TaskSetManager: No tasks for locality level NO_PREF, so moving to locality level ANY 21/01/17 17:43:06.824 task-result-getter-0 INFO TaskSetManager: Finished task 0.0 in stage 0.0 (TID 0) in 1744 ms on 192.168.1.225 (executor driver) (1/4) 21/01/17 17:43:06.833 dag-scheduler-event-loop INFO PythonAccumulatorV2: Connected to AccumulatorServer at host: 127.0.0.1 port: 40323 21/01/17 17:43:06.843 block-manager-storage-async-thread-pool-0 DEBUG BlockManagerStorageEndpoint: removing broadcast 0 21/01/17 17:43:06.843 block-manager-storage-async-thread-pool-0 DEBUG BlockManager: Removing broadcast 0 21/01/17 17:43:06.846 block-manager-storage-async-thread-pool-0 DEBUG BlockManager: Removing block broadcast_0_piece0 21/01/17 17:43:06.856 Executor task launch worker for task 1.0 in stage 0.0 (TID 1) INFO CodeGenerator: Code generated in 94.169046 ms 21/01/17 17:43:06.859 block-manager-storage-async-thread-pool-0 DEBUG MemoryStore: Block broadcast_0_piece0 of size 7188 dropped from memory (free 384077932) 21/01/17 17:43:06.860 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master broadcast_0_piece0 for BlockManagerId(driver, 192.168.1.225, 33787, None) 21/01/17 17:43:06.862 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Removed broadcast_0_piece0 on 192.168.1.225:33787 in memory (size: 7.0 KiB, free: 366.3 MiB) 21/01/17 17:43:06.862 block-manager-storage-async-thread-pool-0 DEBUG BlockManagerMaster: Updated info of block broadcast_0_piece0 21/01/17 17:43:06.862 block-manager-storage-async-thread-pool-0 DEBUG BlockManager: Told master about block broadcast_0_piece0 21/01/17 17:43:06.863 block-manager-storage-async-thread-pool-0 DEBUG BlockManager: Removing block broadcast_0 21/01/17 17:43:06.863 block-manager-storage-async-thread-pool-0 DEBUG MemoryStore: Block broadcast_0 of size 15456 dropped from memory (free 384093388) 21/01/17 17:43:06.868 block-manager-storage-async-thread-pool-2 DEBUG BlockManagerStorageEndpoint: Done removing broadcast 0, response is 0 21/01/17 17:43:06.870 Executor task launch worker for task 3.0 in stage 0.0 (TID 3) DEBUG BlockManager: Getting local block broadcast_0 21/01/17 17:43:06.870 Executor task launch worker for task 3.0 in stage 0.0 (TID 3) DEBUG BlockManager: Level for block broadcast_0 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:06.873 block-manager-storage-async-thread-pool-2 DEBUG BlockManagerStorageEndpoint: Sent response: 0 to 192.168.1.225:37029 21/01/17 17:43:06.875 Spark Context Cleaner DEBUG ContextCleaner: Cleaned broadcast 0 21/01/17 17:43:06.875 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(36) 21/01/17 17:43:06.876 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 36 21/01/17 17:43:06.877 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 36 21/01/17 17:43:06.877 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(35) 21/01/17 17:43:06.877 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 35 21/01/17 17:43:06.877 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 35 21/01/17 17:43:06.877 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(23) 21/01/17 17:43:06.877 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 23 21/01/17 17:43:06.877 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 23 21/01/17 17:43:06.877 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(31) 21/01/17 17:43:06.877 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 31 21/01/17 17:43:06.877 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 31 21/01/17 17:43:06.877 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(22) 21/01/17 17:43:06.877 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 22 21/01/17 17:43:06.877 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 22 21/01/17 17:43:06.877 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(32) 21/01/17 17:43:06.877 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 32 21/01/17 17:43:06.877 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 32 21/01/17 17:43:06.878 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(25) 21/01/17 17:43:06.878 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 25 21/01/17 17:43:06.878 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 25 21/01/17 17:43:06.878 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(34) 21/01/17 17:43:06.878 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 34 21/01/17 17:43:06.878 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 34 21/01/17 17:43:06.878 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(13) 21/01/17 17:43:06.878 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 13 21/01/17 17:43:06.878 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 13 21/01/17 17:43:06.878 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(17) 21/01/17 17:43:06.878 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 17 21/01/17 17:43:06.878 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 17 21/01/17 17:43:06.878 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(24) 21/01/17 17:43:06.878 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 24 21/01/17 17:43:06.878 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 24 21/01/17 17:43:06.878 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(30) 21/01/17 17:43:06.878 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 30 21/01/17 17:43:06.878 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 30 21/01/17 17:43:06.878 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(37) 21/01/17 17:43:06.878 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 37 21/01/17 17:43:06.878 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 37 21/01/17 17:43:06.879 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(16) 21/01/17 17:43:06.879 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 16 21/01/17 17:43:06.879 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 16 21/01/17 17:43:06.879 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(27) 21/01/17 17:43:06.879 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 27 21/01/17 17:43:06.879 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 27 21/01/17 17:43:06.879 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(26) 21/01/17 17:43:06.879 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 26 21/01/17 17:43:06.879 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 26 21/01/17 17:43:06.879 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(21) 21/01/17 17:43:06.879 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 21 21/01/17 17:43:06.879 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 21 21/01/17 17:43:06.879 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(20) 21/01/17 17:43:06.879 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 20 21/01/17 17:43:06.879 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 20 21/01/17 17:43:06.879 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(33) 21/01/17 17:43:06.879 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 33 21/01/17 17:43:06.879 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 33 21/01/17 17:43:06.879 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(18) 21/01/17 17:43:06.879 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 18 21/01/17 17:43:06.879 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 18 21/01/17 17:43:06.879 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(29) 21/01/17 17:43:06.879 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 29 21/01/17 17:43:06.880 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 29 21/01/17 17:43:06.880 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(14) 21/01/17 17:43:06.880 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 14 21/01/17 17:43:06.880 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 14 21/01/17 17:43:06.880 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(28) 21/01/17 17:43:06.880 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 28 21/01/17 17:43:06.880 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 28 21/01/17 17:43:06.880 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(15) 21/01/17 17:43:06.880 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 15 21/01/17 17:43:06.880 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 15 21/01/17 17:43:06.880 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(19) 21/01/17 17:43:06.880 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 19 21/01/17 17:43:06.880 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 19 21/01/17 17:43:06.898 Executor task launch worker for task 3.0 in stage 0.0 (TID 3) DEBUG GenerateUnsafeProjection: code for newInstance(class org.apache.spark.ml.linalg.VectorUDT).serialize: /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificUnsafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificUnsafeProjection extends org.apache.spark.sql.catalyst.expressions.UnsafeProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private boolean resultIsNull_0; /* 009 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[] mutableStateArray_1 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[2]; /* 010 */ private org.apache.spark.ml.linalg.Vector[] mutableStateArray_0 = new org.apache.spark.ml.linalg.Vector[1]; /* 011 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[] mutableStateArray_2 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[2]; /* 012 */ /* 013 */ public SpecificUnsafeProjection(Object[] references) { /* 014 */ this.references = references; /* 015 */ /* 016 */ mutableStateArray_1[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(1, 32); /* 017 */ mutableStateArray_1[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(mutableStateArray_1[0], 4); /* 018 */ mutableStateArray_2[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(mutableStateArray_1[1], 4); /* 019 */ mutableStateArray_2[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(mutableStateArray_1[1], 8); /* 020 */ /* 021 */ } /* 022 */ /* 023 */ public void initialize(int partitionIndex) { /* 024 */ /* 025 */ } /* 026 */ /* 027 */ // Scala.Function1 need this /* 028 */ public java.lang.Object apply(java.lang.Object row) { /* 029 */ return apply((InternalRow) row); /* 030 */ } /* 031 */ /* 032 */ public UnsafeRow apply(InternalRow i) { /* 033 */ mutableStateArray_1[0].reset(); /* 034 */ /* 035 */ /* 036 */ mutableStateArray_1[0].zeroOutNullBytes(); /* 037 */ /* 038 */ final org.apache.spark.ml.linalg.VectorUDT value_1 = false ? /* 039 */ null : new org.apache.spark.ml.linalg.VectorUDT(); /* 040 */ boolean isNull_0 = true; /* 041 */ InternalRow value_0 = null; /* 042 */ if (!false) { /* 043 */ resultIsNull_0 = false; /* 044 */ if (!resultIsNull_0) { /* 045 */ boolean isNull_2 = i.isNullAt(0); /* 046 */ org.apache.spark.ml.linalg.Vector value_2 = isNull_2 ? /* 047 */ null : ((org.apache.spark.ml.linalg.Vector)i.get(0, null)); /* 048 */ resultIsNull_0 = isNull_2; /* 049 */ mutableStateArray_0[0] = value_2; /* 050 */ } /* 051 */ /* 052 */ isNull_0 = resultIsNull_0; /* 053 */ if (!isNull_0) { /* 054 */ /* 055 */ Object funcResult_0 = null; /* 056 */ funcResult_0 = value_1.serialize(mutableStateArray_0[0]); /* 057 */ /* 058 */ if (funcResult_0 != null) { /* 059 */ value_0 = (InternalRow) funcResult_0; /* 060 */ } else { /* 061 */ isNull_0 = true; /* 062 */ } /* 063 */ /* 064 */ /* 065 */ } /* 066 */ } /* 067 */ if (isNull_0) { /* 068 */ mutableStateArray_1[0].setNullAt(0); /* 069 */ } else { /* 070 */ final InternalRow tmpInput_0 = value_0; /* 071 */ if (tmpInput_0 instanceof UnsafeRow) { /* 072 */ mutableStateArray_1[0].write(0, (UnsafeRow) tmpInput_0); /* 073 */ } else { /* 074 */ // Remember the current cursor so that we can calculate how many bytes are /* 075 */ // written later. /* 076 */ final int previousCursor_0 = mutableStateArray_1[0].cursor(); /* 077 */ /* 078 */ mutableStateArray_1[1].resetRowWriter(); /* 079 */ /* 080 */ /* 081 */ mutableStateArray_1[1].write(0, (tmpInput_0.getByte(0))); /* 082 */ /* 083 */ /* 084 */ if ((tmpInput_0.isNullAt(1))) { /* 085 */ mutableStateArray_1[1].setNullAt(1); /* 086 */ } else { /* 087 */ mutableStateArray_1[1].write(1, (tmpInput_0.getInt(1))); /* 088 */ } /* 089 */ /* 090 */ /* 091 */ if ((tmpInput_0.isNullAt(2))) { /* 092 */ mutableStateArray_1[1].setNullAt(2); /* 093 */ } else { /* 094 */ // Remember the current cursor so that we can calculate how many bytes are /* 095 */ // written later. /* 096 */ final int previousCursor_1 = mutableStateArray_1[1].cursor(); /* 097 */ /* 098 */ final ArrayData tmpInput_1 = (tmpInput_0.getArray(2)); /* 099 */ if (tmpInput_1 instanceof UnsafeArrayData) { /* 100 */ mutableStateArray_1[1].write((UnsafeArrayData) tmpInput_1); /* 101 */ } else { /* 102 */ final int numElements_0 = tmpInput_1.numElements(); /* 103 */ mutableStateArray_2[0].initialize(numElements_0); /* 104 */ /* 105 */ for (int index_0 = 0; index_0 < numElements_0; index_0++) { /* 106 */ mutableStateArray_2[0].write(index_0, tmpInput_1.getInt(index_0)); /* 107 */ } /* 108 */ } /* 109 */ /* 110 */ mutableStateArray_1[1].setOffsetAndSizeFromPreviousCursor(2, previousCursor_1); /* 111 */ } /* 112 */ /* 113 */ /* 114 */ if ((tmpInput_0.isNullAt(3))) { /* 115 */ mutableStateArray_1[1].setNullAt(3); /* 116 */ } else { /* 117 */ // Remember the current cursor so that we can calculate how many bytes are /* 118 */ // written later. /* 119 */ final int previousCursor_2 = mutableStateArray_1[1].cursor(); /* 120 */ /* 121 */ final ArrayData tmpInput_2 = (tmpInput_0.getArray(3)); /* 122 */ if (tmpInput_2 instanceof UnsafeArrayData) { /* 123 */ mutableStateArray_1[1].write((UnsafeArrayData) tmpInput_2); /* 124 */ } else { /* 125 */ final int numElements_1 = tmpInput_2.numElements(); /* 126 */ mutableStateArray_2[1].initialize(numElements_1); /* 127 */ /* 128 */ for (int index_1 = 0; index_1 < numElements_1; index_1++) { /* 129 */ mutableStateArray_2[1].write(index_1, tmpInput_2.getDouble(index_1)); /* 130 */ } /* 131 */ } /* 132 */ /* 133 */ mutableStateArray_1[1].setOffsetAndSizeFromPreviousCursor(3, previousCursor_2); /* 134 */ } /* 135 */ /* 136 */ /* 137 */ mutableStateArray_1[0].setOffsetAndSizeFromPreviousCursor(0, previousCursor_0); /* 138 */ } /* 139 */ } /* 140 */ return (mutableStateArray_1[0].getRow()); /* 141 */ } /* 142 */ /* 143 */ /* 144 */ } 21/01/17 17:43:06.901 Executor task launch worker for task 3.0 in stage 0.0 (TID 3) DEBUG CodeGenerator: /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificUnsafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificUnsafeProjection extends org.apache.spark.sql.catalyst.expressions.UnsafeProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private boolean resultIsNull_0; /* 009 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[] mutableStateArray_1 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[2]; /* 010 */ private org.apache.spark.ml.linalg.Vector[] mutableStateArray_0 = new org.apache.spark.ml.linalg.Vector[1]; /* 011 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[] mutableStateArray_2 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[2]; /* 012 */ /* 013 */ public SpecificUnsafeProjection(Object[] references) { /* 014 */ this.references = references; /* 015 */ /* 016 */ mutableStateArray_1[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(1, 32); /* 017 */ mutableStateArray_1[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(mutableStateArray_1[0], 4); /* 018 */ mutableStateArray_2[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(mutableStateArray_1[1], 4); /* 019 */ mutableStateArray_2[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(mutableStateArray_1[1], 8); /* 020 */ /* 021 */ } /* 022 */ /* 023 */ public void initialize(int partitionIndex) { /* 024 */ /* 025 */ } /* 026 */ /* 027 */ // Scala.Function1 need this /* 028 */ public java.lang.Object apply(java.lang.Object row) { /* 029 */ return apply((InternalRow) row); /* 030 */ } /* 031 */ /* 032 */ public UnsafeRow apply(InternalRow i) { /* 033 */ mutableStateArray_1[0].reset(); /* 034 */ /* 035 */ /* 036 */ mutableStateArray_1[0].zeroOutNullBytes(); /* 037 */ /* 038 */ final org.apache.spark.ml.linalg.VectorUDT value_1 = false ? /* 039 */ null : new org.apache.spark.ml.linalg.VectorUDT(); /* 040 */ boolean isNull_0 = true; /* 041 */ InternalRow value_0 = null; /* 042 */ if (!false) { /* 043 */ resultIsNull_0 = false; /* 044 */ if (!resultIsNull_0) { /* 045 */ boolean isNull_2 = i.isNullAt(0); /* 046 */ org.apache.spark.ml.linalg.Vector value_2 = isNull_2 ? /* 047 */ null : ((org.apache.spark.ml.linalg.Vector)i.get(0, null)); /* 048 */ resultIsNull_0 = isNull_2; /* 049 */ mutableStateArray_0[0] = value_2; /* 050 */ } /* 051 */ /* 052 */ isNull_0 = resultIsNull_0; /* 053 */ if (!isNull_0) { /* 054 */ /* 055 */ Object funcResult_0 = null; /* 056 */ funcResult_0 = value_1.serialize(mutableStateArray_0[0]); /* 057 */ /* 058 */ if (funcResult_0 != null) { /* 059 */ value_0 = (InternalRow) funcResult_0; /* 060 */ } else { /* 061 */ isNull_0 = true; /* 062 */ } /* 063 */ /* 064 */ /* 065 */ } /* 066 */ } /* 067 */ if (isNull_0) { /* 068 */ mutableStateArray_1[0].setNullAt(0); /* 069 */ } else { /* 070 */ final InternalRow tmpInput_0 = value_0; /* 071 */ if (tmpInput_0 instanceof UnsafeRow) { /* 072 */ mutableStateArray_1[0].write(0, (UnsafeRow) tmpInput_0); /* 073 */ } else { /* 074 */ // Remember the current cursor so that we can calculate how many bytes are /* 075 */ // written later. /* 076 */ final int previousCursor_0 = mutableStateArray_1[0].cursor(); /* 077 */ /* 078 */ mutableStateArray_1[1].resetRowWriter(); /* 079 */ /* 080 */ /* 081 */ mutableStateArray_1[1].write(0, (tmpInput_0.getByte(0))); /* 082 */ /* 083 */ /* 084 */ if ((tmpInput_0.isNullAt(1))) { /* 085 */ mutableStateArray_1[1].setNullAt(1); /* 086 */ } else { /* 087 */ mutableStateArray_1[1].write(1, (tmpInput_0.getInt(1))); /* 088 */ } /* 089 */ /* 090 */ /* 091 */ if ((tmpInput_0.isNullAt(2))) { /* 092 */ mutableStateArray_1[1].setNullAt(2); /* 093 */ } else { /* 094 */ // Remember the current cursor so that we can calculate how many bytes are /* 095 */ // written later. /* 096 */ final int previousCursor_1 = mutableStateArray_1[1].cursor(); /* 097 */ /* 098 */ final ArrayData tmpInput_1 = (tmpInput_0.getArray(2)); /* 099 */ if (tmpInput_1 instanceof UnsafeArrayData) { /* 100 */ mutableStateArray_1[1].write((UnsafeArrayData) tmpInput_1); /* 101 */ } else { /* 102 */ final int numElements_0 = tmpInput_1.numElements(); /* 103 */ mutableStateArray_2[0].initialize(numElements_0); /* 104 */ /* 105 */ for (int index_0 = 0; index_0 < numElements_0; index_0++) { /* 106 */ mutableStateArray_2[0].write(index_0, tmpInput_1.getInt(index_0)); /* 107 */ } /* 108 */ } /* 109 */ /* 110 */ mutableStateArray_1[1].setOffsetAndSizeFromPreviousCursor(2, previousCursor_1); /* 111 */ } /* 112 */ /* 113 */ /* 114 */ if ((tmpInput_0.isNullAt(3))) { /* 115 */ mutableStateArray_1[1].setNullAt(3); /* 116 */ } else { /* 117 */ // Remember the current cursor so that we can calculate how many bytes are /* 118 */ // written later. /* 119 */ final int previousCursor_2 = mutableStateArray_1[1].cursor(); /* 120 */ /* 121 */ final ArrayData tmpInput_2 = (tmpInput_0.getArray(3)); /* 122 */ if (tmpInput_2 instanceof UnsafeArrayData) { /* 123 */ mutableStateArray_1[1].write((UnsafeArrayData) tmpInput_2); /* 124 */ } else { /* 125 */ final int numElements_1 = tmpInput_2.numElements(); /* 126 */ mutableStateArray_2[1].initialize(numElements_1); /* 127 */ /* 128 */ for (int index_1 = 0; index_1 < numElements_1; index_1++) { /* 129 */ mutableStateArray_2[1].write(index_1, tmpInput_2.getDouble(index_1)); /* 130 */ } /* 131 */ } /* 132 */ /* 133 */ mutableStateArray_1[1].setOffsetAndSizeFromPreviousCursor(3, previousCursor_2); /* 134 */ } /* 135 */ /* 136 */ /* 137 */ mutableStateArray_1[0].setOffsetAndSizeFromPreviousCursor(0, previousCursor_0); /* 138 */ } /* 139 */ } /* 140 */ return (mutableStateArray_1[0].getRow()); /* 141 */ } /* 142 */ /* 143 */ /* 144 */ } 21/01/17 17:43:06.906 Thread-3 DEBUG WholeStageCodegenExec: /* 001 */ public Object generate(Object[] references) { /* 002 */ return new GeneratedIteratorForCodegenStage1(references); /* 003 */ } /* 004 */ /* 005 */ // codegenStageId=1 /* 006 */ final class GeneratedIteratorForCodegenStage1 extends org.apache.spark.sql.execution.BufferedRowIterator { /* 007 */ private Object[] references; /* 008 */ private scala.collection.Iterator[] inputs; /* 009 */ private scala.collection.Iterator rdd_input_0; /* 010 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[] rdd_mutableStateArray_0 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[4]; /* 011 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[] rdd_mutableStateArray_1 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[4]; /* 012 */ /* 013 */ public GeneratedIteratorForCodegenStage1(Object[] references) { /* 014 */ this.references = references; /* 015 */ } /* 016 */ /* 017 */ public void init(int index, scala.collection.Iterator[] inputs) { /* 018 */ partitionIndex = index; /* 019 */ this.inputs = inputs; /* 020 */ rdd_input_0 = inputs[0]; /* 021 */ rdd_mutableStateArray_0[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(2, 32); /* 022 */ rdd_mutableStateArray_0[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(rdd_mutableStateArray_0[0], 4); /* 023 */ rdd_mutableStateArray_1[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[1], 4); /* 024 */ rdd_mutableStateArray_1[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[1], 8); /* 025 */ rdd_mutableStateArray_0[2] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(3, 32); /* 026 */ rdd_mutableStateArray_0[3] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(rdd_mutableStateArray_0[2], 4); /* 027 */ rdd_mutableStateArray_1[2] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[3], 4); /* 028 */ rdd_mutableStateArray_1[3] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[3], 8); /* 029 */ /* 030 */ } /* 031 */ /* 032 */ private void project_writeFields_0_1(InternalRow project_tmpInput_0) { /* 033 */ if ((project_tmpInput_0.isNullAt(3))) { /* 034 */ rdd_mutableStateArray_0[3].setNullAt(3); /* 035 */ } else { /* 036 */ // Remember the current cursor so that we can calculate how many bytes are /* 037 */ // written later. /* 038 */ final int project_previousCursor_2 = rdd_mutableStateArray_0[3].cursor(); /* 039 */ /* 040 */ final ArrayData project_tmpInput_2 = (project_tmpInput_0.getArray(3)); /* 041 */ if (project_tmpInput_2 instanceof UnsafeArrayData) { /* 042 */ rdd_mutableStateArray_0[3].write((UnsafeArrayData) project_tmpInput_2); /* 043 */ } else { /* 044 */ final int project_numElements_1 = project_tmpInput_2.numElements(); /* 045 */ rdd_mutableStateArray_1[3].initialize(project_numElements_1); /* 046 */ /* 047 */ for (int project_index_1 = 0; project_index_1 < project_numElements_1; project_index_1++) { /* 048 */ rdd_mutableStateArray_1[3].write(project_index_1, project_tmpInput_2.getDouble(project_index_1)); /* 049 */ } /* 050 */ } /* 051 */ /* 052 */ rdd_mutableStateArray_0[3].setOffsetAndSizeFromPreviousCursor(3, project_previousCursor_2); /* 053 */ } /* 054 */ /* 055 */ } /* 056 */ /* 057 */ private void project_writeFields_0_0(InternalRow project_tmpInput_0) { /* 058 */ rdd_mutableStateArray_0[3].write(0, (project_tmpInput_0.getByte(0))); /* 059 */ /* 060 */ if ((project_tmpInput_0.isNullAt(1))) { /* 061 */ rdd_mutableStateArray_0[3].setNullAt(1); /* 062 */ } else { /* 063 */ rdd_mutableStateArray_0[3].write(1, (project_tmpInput_0.getInt(1))); /* 064 */ } /* 065 */ /* 066 */ if ((project_tmpInput_0.isNullAt(2))) { /* 067 */ rdd_mutableStateArray_0[3].setNullAt(2); /* 068 */ } else { /* 069 */ // Remember the current cursor so that we can calculate how many bytes are /* 070 */ // written later. /* 071 */ final int project_previousCursor_1 = rdd_mutableStateArray_0[3].cursor(); /* 072 */ /* 073 */ final ArrayData project_tmpInput_1 = (project_tmpInput_0.getArray(2)); /* 074 */ if (project_tmpInput_1 instanceof UnsafeArrayData) { /* 075 */ rdd_mutableStateArray_0[3].write((UnsafeArrayData) project_tmpInput_1); /* 076 */ } else { /* 077 */ final int project_numElements_0 = project_tmpInput_1.numElements(); /* 078 */ rdd_mutableStateArray_1[2].initialize(project_numElements_0); /* 079 */ /* 080 */ for (int project_index_0 = 0; project_index_0 < project_numElements_0; project_index_0++) { /* 081 */ rdd_mutableStateArray_1[2].write(project_index_0, project_tmpInput_1.getInt(project_index_0)); /* 082 */ } /* 083 */ } /* 084 */ /* 085 */ rdd_mutableStateArray_0[3].setOffsetAndSizeFromPreviousCursor(2, project_previousCursor_1); /* 086 */ } /* 087 */ /* 088 */ } /* 089 */ /* 090 */ protected void processNext() throws java.io.IOException { /* 091 */ while ( rdd_input_0.hasNext()) { /* 092 */ InternalRow rdd_row_0 = (InternalRow) rdd_input_0.next(); /* 093 */ ((org.apache.spark.sql.execution.metric.SQLMetric) references[0] /* numOutputRows */).add(1); /* 094 */ // common sub-expressions /* 095 */ /* 096 */ boolean rdd_isNull_1 = rdd_row_0.isNullAt(1); /* 097 */ double rdd_value_1 = rdd_isNull_1 ? /* 098 */ -1.0 : (rdd_row_0.getDouble(1)); /* 099 */ boolean rdd_isNull_0 = rdd_row_0.isNullAt(0); /* 100 */ InternalRow rdd_value_0 = rdd_isNull_0 ? /* 101 */ null : (rdd_row_0.getStruct(0, 4)); /* 102 */ /* 103 */ Object project_arg_0 = null; /* 104 */ if (rdd_isNull_0) { /* 105 */ project_arg_0 = ((scala.Function1[]) references[1] /* converters */)[0].apply(null); /* 106 */ } else { /* 107 */ project_arg_0 = ((scala.Function1[]) references[1] /* converters */)[0].apply(rdd_value_0); /* 108 */ } /* 109 */ /* 110 */ InternalRow project_result_0 = null; /* 111 */ try { /* 112 */ project_result_0 = (InternalRow)((scala.Function1[]) references[1] /* converters */)[1].apply(((scala.Function1) references[3] /* udf */).apply(project_arg_0)); /* 113 */ } catch (Exception e) { /* 114 */ throw new org.apache.spark.SparkException(((java.lang.String) references[2] /* errMsg */), e); /* 115 */ } /* 116 */ /* 117 */ boolean project_isNull_1 = project_result_0 == null; /* 118 */ InternalRow project_value_1 = null; /* 119 */ if (!project_isNull_1) { /* 120 */ project_value_1 = project_result_0; /* 121 */ } /* 122 */ rdd_mutableStateArray_0[2].reset(); /* 123 */ /* 124 */ rdd_mutableStateArray_0[2].zeroOutNullBytes(); /* 125 */ /* 126 */ if (rdd_isNull_1) { /* 127 */ rdd_mutableStateArray_0[2].setNullAt(0); /* 128 */ } else { /* 129 */ rdd_mutableStateArray_0[2].write(0, rdd_value_1); /* 130 */ } /* 131 */ /* 132 */ if (project_isNull_1) { /* 133 */ rdd_mutableStateArray_0[2].setNullAt(1); /* 134 */ } else { /* 135 */ final InternalRow project_tmpInput_0 = project_value_1; /* 136 */ if (project_tmpInput_0 instanceof UnsafeRow) { /* 137 */ rdd_mutableStateArray_0[2].write(1, (UnsafeRow) project_tmpInput_0); /* 138 */ } else { /* 139 */ // Remember the current cursor so that we can calculate how many bytes are /* 140 */ // written later. /* 141 */ final int project_previousCursor_0 = rdd_mutableStateArray_0[2].cursor(); /* 142 */ /* 143 */ rdd_mutableStateArray_0[3].resetRowWriter(); /* 144 */ project_writeFields_0_0(project_tmpInput_0); /* 145 */ project_writeFields_0_1(project_tmpInput_0); /* 146 */ /* 147 */ rdd_mutableStateArray_0[2].setOffsetAndSizeFromPreviousCursor(1, project_previousCursor_0); /* 148 */ } /* 149 */ } /* 150 */ /* 151 */ rdd_mutableStateArray_0[2].write(2, 1.0D); /* 152 */ append((rdd_mutableStateArray_0[2].getRow())); /* 153 */ if (shouldStop()) return; /* 154 */ } /* 155 */ } /* 156 */ /* 157 */ } 21/01/17 17:43:06.911 Thread-3 DEBUG CodeGenerator: /* 001 */ public Object generate(Object[] references) { /* 002 */ return new GeneratedIteratorForCodegenStage1(references); /* 003 */ } /* 004 */ /* 005 */ // codegenStageId=1 /* 006 */ final class GeneratedIteratorForCodegenStage1 extends org.apache.spark.sql.execution.BufferedRowIterator { /* 007 */ private Object[] references; /* 008 */ private scala.collection.Iterator[] inputs; /* 009 */ private scala.collection.Iterator rdd_input_0; /* 010 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[] rdd_mutableStateArray_0 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[4]; /* 011 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[] rdd_mutableStateArray_1 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[4]; /* 012 */ /* 013 */ public GeneratedIteratorForCodegenStage1(Object[] references) { /* 014 */ this.references = references; /* 015 */ } /* 016 */ /* 017 */ public void init(int index, scala.collection.Iterator[] inputs) { /* 018 */ partitionIndex = index; /* 019 */ this.inputs = inputs; /* 020 */ rdd_input_0 = inputs[0]; /* 021 */ rdd_mutableStateArray_0[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(2, 32); /* 022 */ rdd_mutableStateArray_0[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(rdd_mutableStateArray_0[0], 4); /* 023 */ rdd_mutableStateArray_1[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[1], 4); /* 024 */ rdd_mutableStateArray_1[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[1], 8); /* 025 */ rdd_mutableStateArray_0[2] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(3, 32); /* 026 */ rdd_mutableStateArray_0[3] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(rdd_mutableStateArray_0[2], 4); /* 027 */ rdd_mutableStateArray_1[2] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[3], 4); /* 028 */ rdd_mutableStateArray_1[3] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[3], 8); /* 029 */ /* 030 */ } /* 031 */ /* 032 */ private void project_writeFields_0_1(InternalRow project_tmpInput_0) { /* 033 */ if ((project_tmpInput_0.isNullAt(3))) { /* 034 */ rdd_mutableStateArray_0[3].setNullAt(3); /* 035 */ } else { /* 036 */ // Remember the current cursor so that we can calculate how many bytes are /* 037 */ // written later. /* 038 */ final int project_previousCursor_2 = rdd_mutableStateArray_0[3].cursor(); /* 039 */ /* 040 */ final ArrayData project_tmpInput_2 = (project_tmpInput_0.getArray(3)); /* 041 */ if (project_tmpInput_2 instanceof UnsafeArrayData) { /* 042 */ rdd_mutableStateArray_0[3].write((UnsafeArrayData) project_tmpInput_2); /* 043 */ } else { /* 044 */ final int project_numElements_1 = project_tmpInput_2.numElements(); /* 045 */ rdd_mutableStateArray_1[3].initialize(project_numElements_1); /* 046 */ /* 047 */ for (int project_index_1 = 0; project_index_1 < project_numElements_1; project_index_1++) { /* 048 */ rdd_mutableStateArray_1[3].write(project_index_1, project_tmpInput_2.getDouble(project_index_1)); /* 049 */ } /* 050 */ } /* 051 */ /* 052 */ rdd_mutableStateArray_0[3].setOffsetAndSizeFromPreviousCursor(3, project_previousCursor_2); /* 053 */ } /* 054 */ /* 055 */ } /* 056 */ /* 057 */ private void project_writeFields_0_0(InternalRow project_tmpInput_0) { /* 058 */ rdd_mutableStateArray_0[3].write(0, (project_tmpInput_0.getByte(0))); /* 059 */ /* 060 */ if ((project_tmpInput_0.isNullAt(1))) { /* 061 */ rdd_mutableStateArray_0[3].setNullAt(1); /* 062 */ } else { /* 063 */ rdd_mutableStateArray_0[3].write(1, (project_tmpInput_0.getInt(1))); /* 064 */ } /* 065 */ /* 066 */ if ((project_tmpInput_0.isNullAt(2))) { /* 067 */ rdd_mutableStateArray_0[3].setNullAt(2); /* 068 */ } else { /* 069 */ // Remember the current cursor so that we can calculate how many bytes are /* 070 */ // written later. /* 071 */ final int project_previousCursor_1 = rdd_mutableStateArray_0[3].cursor(); /* 072 */ /* 073 */ final ArrayData project_tmpInput_1 = (project_tmpInput_0.getArray(2)); /* 074 */ if (project_tmpInput_1 instanceof UnsafeArrayData) { /* 075 */ rdd_mutableStateArray_0[3].write((UnsafeArrayData) project_tmpInput_1); /* 076 */ } else { /* 077 */ final int project_numElements_0 = project_tmpInput_1.numElements(); /* 078 */ rdd_mutableStateArray_1[2].initialize(project_numElements_0); /* 079 */ /* 080 */ for (int project_index_0 = 0; project_index_0 < project_numElements_0; project_index_0++) { /* 081 */ rdd_mutableStateArray_1[2].write(project_index_0, project_tmpInput_1.getInt(project_index_0)); /* 082 */ } /* 083 */ } /* 084 */ /* 085 */ rdd_mutableStateArray_0[3].setOffsetAndSizeFromPreviousCursor(2, project_previousCursor_1); /* 086 */ } /* 087 */ /* 088 */ } /* 089 */ /* 090 */ protected void processNext() throws java.io.IOException { /* 091 */ while ( rdd_input_0.hasNext()) { /* 092 */ InternalRow rdd_row_0 = (InternalRow) rdd_input_0.next(); /* 093 */ ((org.apache.spark.sql.execution.metric.SQLMetric) references[0] /* numOutputRows */).add(1); /* 094 */ // common sub-expressions /* 095 */ /* 096 */ boolean rdd_isNull_1 = rdd_row_0.isNullAt(1); /* 097 */ double rdd_value_1 = rdd_isNull_1 ? /* 098 */ -1.0 : (rdd_row_0.getDouble(1)); /* 099 */ boolean rdd_isNull_0 = rdd_row_0.isNullAt(0); /* 100 */ InternalRow rdd_value_0 = rdd_isNull_0 ? /* 101 */ null : (rdd_row_0.getStruct(0, 4)); /* 102 */ /* 103 */ Object project_arg_0 = null; /* 104 */ if (rdd_isNull_0) { /* 105 */ project_arg_0 = ((scala.Function1[]) references[1] /* converters */)[0].apply(null); /* 106 */ } else { /* 107 */ project_arg_0 = ((scala.Function1[]) references[1] /* converters */)[0].apply(rdd_value_0); /* 108 */ } /* 109 */ /* 110 */ InternalRow project_result_0 = null; /* 111 */ try { /* 112 */ project_result_0 = (InternalRow)((scala.Function1[]) references[1] /* converters */)[1].apply(((scala.Function1) references[3] /* udf */).apply(project_arg_0)); /* 113 */ } catch (Exception e) { /* 114 */ throw new org.apache.spark.SparkException(((java.lang.String) references[2] /* errMsg */), e); /* 115 */ } /* 116 */ /* 117 */ boolean project_isNull_1 = project_result_0 == null; /* 118 */ InternalRow project_value_1 = null; /* 119 */ if (!project_isNull_1) { /* 120 */ project_value_1 = project_result_0; /* 121 */ } /* 122 */ rdd_mutableStateArray_0[2].reset(); /* 123 */ /* 124 */ rdd_mutableStateArray_0[2].zeroOutNullBytes(); /* 125 */ /* 126 */ if (rdd_isNull_1) { /* 127 */ rdd_mutableStateArray_0[2].setNullAt(0); /* 128 */ } else { /* 129 */ rdd_mutableStateArray_0[2].write(0, rdd_value_1); /* 130 */ } /* 131 */ /* 132 */ if (project_isNull_1) { /* 133 */ rdd_mutableStateArray_0[2].setNullAt(1); /* 134 */ } else { /* 135 */ final InternalRow project_tmpInput_0 = project_value_1; /* 136 */ if (project_tmpInput_0 instanceof UnsafeRow) { /* 137 */ rdd_mutableStateArray_0[2].write(1, (UnsafeRow) project_tmpInput_0); /* 138 */ } else { /* 139 */ // Remember the current cursor so that we can calculate how many bytes are /* 140 */ // written later. /* 141 */ final int project_previousCursor_0 = rdd_mutableStateArray_0[2].cursor(); /* 142 */ /* 143 */ rdd_mutableStateArray_0[3].resetRowWriter(); /* 144 */ project_writeFields_0_0(project_tmpInput_0); /* 145 */ project_writeFields_0_1(project_tmpInput_0); /* 146 */ /* 147 */ rdd_mutableStateArray_0[2].setOffsetAndSizeFromPreviousCursor(1, project_previousCursor_0); /* 148 */ } /* 149 */ } /* 150 */ /* 151 */ rdd_mutableStateArray_0[2].write(2, 1.0D); /* 152 */ append((rdd_mutableStateArray_0[2].getRow())); /* 153 */ if (shouldStop()) return; /* 154 */ } /* 155 */ } /* 156 */ /* 157 */ } 21/01/17 17:43:06.918 Executor task launch worker for task 2.0 in stage 0.0 (TID 2) DEBUG GenerateUnsafeProjection: code for newInstance(class org.apache.spark.ml.linalg.VectorUDT).serialize: /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificUnsafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificUnsafeProjection extends org.apache.spark.sql.catalyst.expressions.UnsafeProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private boolean resultIsNull_0; /* 009 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[] mutableStateArray_1 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[2]; /* 010 */ private org.apache.spark.ml.linalg.Vector[] mutableStateArray_0 = new org.apache.spark.ml.linalg.Vector[1]; /* 011 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[] mutableStateArray_2 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[2]; /* 012 */ /* 013 */ public SpecificUnsafeProjection(Object[] references) { /* 014 */ this.references = references; /* 015 */ /* 016 */ mutableStateArray_1[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(1, 32); /* 017 */ mutableStateArray_1[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(mutableStateArray_1[0], 4); /* 018 */ mutableStateArray_2[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(mutableStateArray_1[1], 4); /* 019 */ mutableStateArray_2[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(mutableStateArray_1[1], 8); /* 020 */ /* 021 */ } /* 022 */ /* 023 */ public void initialize(int partitionIndex) { /* 024 */ /* 025 */ } /* 026 */ /* 027 */ // Scala.Function1 need this /* 028 */ public java.lang.Object apply(java.lang.Object row) { /* 029 */ return apply((InternalRow) row); /* 030 */ } /* 031 */ /* 032 */ public UnsafeRow apply(InternalRow i) { /* 033 */ mutableStateArray_1[0].reset(); /* 034 */ /* 035 */ /* 036 */ mutableStateArray_1[0].zeroOutNullBytes(); /* 037 */ /* 038 */ final org.apache.spark.ml.linalg.VectorUDT value_1 = false ? /* 039 */ null : new org.apache.spark.ml.linalg.VectorUDT(); /* 040 */ boolean isNull_0 = true; /* 041 */ InternalRow value_0 = null; /* 042 */ if (!false) { /* 043 */ resultIsNull_0 = false; /* 044 */ if (!resultIsNull_0) { /* 045 */ boolean isNull_2 = i.isNullAt(0); /* 046 */ org.apache.spark.ml.linalg.Vector value_2 = isNull_2 ? /* 047 */ null : ((org.apache.spark.ml.linalg.Vector)i.get(0, null)); /* 048 */ resultIsNull_0 = isNull_2; /* 049 */ mutableStateArray_0[0] = value_2; /* 050 */ } /* 051 */ /* 052 */ isNull_0 = resultIsNull_0; /* 053 */ if (!isNull_0) { /* 054 */ /* 055 */ Object funcResult_0 = null; /* 056 */ funcResult_0 = value_1.serialize(mutableStateArray_0[0]); /* 057 */ /* 058 */ if (funcResult_0 != null) { /* 059 */ value_0 = (InternalRow) funcResult_0; /* 060 */ } else { /* 061 */ isNull_0 = true; /* 062 */ } /* 063 */ /* 064 */ /* 065 */ } /* 066 */ } /* 067 */ if (isNull_0) { /* 068 */ mutableStateArray_1[0].setNullAt(0); /* 069 */ } else { /* 070 */ final InternalRow tmpInput_0 = value_0; /* 071 */ if (tmpInput_0 instanceof UnsafeRow) { /* 072 */ mutableStateArray_1[0].write(0, (UnsafeRow) tmpInput_0); /* 073 */ } else { /* 074 */ // Remember the current cursor so that we can calculate how many bytes are /* 075 */ // written later. /* 076 */ final int previousCursor_0 = mutableStateArray_1[0].cursor(); /* 077 */ /* 078 */ mutableStateArray_1[1].resetRowWriter(); /* 079 */ /* 080 */ /* 081 */ mutableStateArray_1[1].write(0, (tmpInput_0.getByte(0))); /* 082 */ /* 083 */ /* 084 */ if ((tmpInput_0.isNullAt(1))) { /* 085 */ mutableStateArray_1[1].setNullAt(1); /* 086 */ } else { /* 087 */ mutableStateArray_1[1].write(1, (tmpInput_0.getInt(1))); /* 088 */ } /* 089 */ /* 090 */ /* 091 */ if ((tmpInput_0.isNullAt(2))) { /* 092 */ mutableStateArray_1[1].setNullAt(2); /* 093 */ } else { /* 094 */ // Remember the current cursor so that we can calculate how many bytes are /* 095 */ // written later. /* 096 */ final int previousCursor_1 = mutableStateArray_1[1].cursor(); /* 097 */ /* 098 */ final ArrayData tmpInput_1 = (tmpInput_0.getArray(2)); /* 099 */ if (tmpInput_1 instanceof UnsafeArrayData) { /* 100 */ mutableStateArray_1[1].write((UnsafeArrayData) tmpInput_1); /* 101 */ } else { /* 102 */ final int numElements_0 = tmpInput_1.numElements(); /* 103 */ mutableStateArray_2[0].initialize(numElements_0); /* 104 */ /* 105 */ for (int index_0 = 0; index_0 < numElements_0; index_0++) { /* 106 */ mutableStateArray_2[0].write(index_0, tmpInput_1.getInt(index_0)); /* 107 */ } /* 108 */ } /* 109 */ /* 110 */ mutableStateArray_1[1].setOffsetAndSizeFromPreviousCursor(2, previousCursor_1); /* 111 */ } /* 112 */ /* 113 */ /* 114 */ if ((tmpInput_0.isNullAt(3))) { /* 115 */ mutableStateArray_1[1].setNullAt(3); /* 116 */ } else { /* 117 */ // Remember the current cursor so that we can calculate how many bytes are /* 118 */ // written later. /* 119 */ final int previousCursor_2 = mutableStateArray_1[1].cursor(); /* 120 */ /* 121 */ final ArrayData tmpInput_2 = (tmpInput_0.getArray(3)); /* 122 */ if (tmpInput_2 instanceof UnsafeArrayData) { /* 123 */ mutableStateArray_1[1].write((UnsafeArrayData) tmpInput_2); /* 124 */ } else { /* 125 */ final int numElements_1 = tmpInput_2.numElements(); /* 126 */ mutableStateArray_2[1].initialize(numElements_1); /* 127 */ /* 128 */ for (int index_1 = 0; index_1 < numElements_1; index_1++) { /* 129 */ mutableStateArray_2[1].write(index_1, tmpInput_2.getDouble(index_1)); /* 130 */ } /* 131 */ } /* 132 */ /* 133 */ mutableStateArray_1[1].setOffsetAndSizeFromPreviousCursor(3, previousCursor_2); /* 134 */ } /* 135 */ /* 136 */ /* 137 */ mutableStateArray_1[0].setOffsetAndSizeFromPreviousCursor(0, previousCursor_0); /* 138 */ } /* 139 */ } /* 140 */ return (mutableStateArray_1[0].getRow()); /* 141 */ } /* 142 */ /* 143 */ /* 144 */ } 21/01/17 17:43:06.903 Executor task launch worker for task 1.0 in stage 0.0 (TID 1) DEBUG GenerateUnsafeProjection: code for newInstance(class org.apache.spark.ml.linalg.VectorUDT).serialize: /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificUnsafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificUnsafeProjection extends org.apache.spark.sql.catalyst.expressions.UnsafeProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private boolean resultIsNull_0; /* 009 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[] mutableStateArray_1 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[2]; /* 010 */ private org.apache.spark.ml.linalg.Vector[] mutableStateArray_0 = new org.apache.spark.ml.linalg.Vector[1]; /* 011 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[] mutableStateArray_2 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[2]; /* 012 */ /* 013 */ public SpecificUnsafeProjection(Object[] references) { /* 014 */ this.references = references; /* 015 */ /* 016 */ mutableStateArray_1[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(1, 32); /* 017 */ mutableStateArray_1[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(mutableStateArray_1[0], 4); /* 018 */ mutableStateArray_2[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(mutableStateArray_1[1], 4); /* 019 */ mutableStateArray_2[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(mutableStateArray_1[1], 8); /* 020 */ /* 021 */ } /* 022 */ /* 023 */ public void initialize(int partitionIndex) { /* 024 */ /* 025 */ } /* 026 */ /* 027 */ // Scala.Function1 need this /* 028 */ public java.lang.Object apply(java.lang.Object row) { /* 029 */ return apply((InternalRow) row); /* 030 */ } /* 031 */ /* 032 */ public UnsafeRow apply(InternalRow i) { /* 033 */ mutableStateArray_1[0].reset(); /* 034 */ /* 035 */ /* 036 */ mutableStateArray_1[0].zeroOutNullBytes(); /* 037 */ /* 038 */ final org.apache.spark.ml.linalg.VectorUDT value_1 = false ? /* 039 */ null : new org.apache.spark.ml.linalg.VectorUDT(); /* 040 */ boolean isNull_0 = true; /* 041 */ InternalRow value_0 = null; /* 042 */ if (!false) { /* 043 */ resultIsNull_0 = false; /* 044 */ if (!resultIsNull_0) { /* 045 */ boolean isNull_2 = i.isNullAt(0); /* 046 */ org.apache.spark.ml.linalg.Vector value_2 = isNull_2 ? /* 047 */ null : ((org.apache.spark.ml.linalg.Vector)i.get(0, null)); /* 048 */ resultIsNull_0 = isNull_2; /* 049 */ mutableStateArray_0[0] = value_2; /* 050 */ } /* 051 */ /* 052 */ isNull_0 = resultIsNull_0; /* 053 */ if (!isNull_0) { /* 054 */ /* 055 */ Object funcResult_0 = null; /* 056 */ funcResult_0 = value_1.serialize(mutableStateArray_0[0]); /* 057 */ /* 058 */ if (funcResult_0 != null) { /* 059 */ value_0 = (InternalRow) funcResult_0; /* 060 */ } else { /* 061 */ isNull_0 = true; /* 062 */ } /* 063 */ /* 064 */ /* 065 */ } /* 066 */ } /* 067 */ if (isNull_0) { /* 068 */ mutableStateArray_1[0].setNullAt(0); /* 069 */ } else { /* 070 */ final InternalRow tmpInput_0 = value_0; /* 071 */ if (tmpInput_0 instanceof UnsafeRow) { /* 072 */ mutableStateArray_1[0].write(0, (UnsafeRow) tmpInput_0); /* 073 */ } else { /* 074 */ // Remember the current cursor so that we can calculate how many bytes are /* 075 */ // written later. /* 076 */ final int previousCursor_0 = mutableStateArray_1[0].cursor(); /* 077 */ /* 078 */ mutableStateArray_1[1].resetRowWriter(); /* 079 */ /* 080 */ /* 081 */ mutableStateArray_1[1].write(0, (tmpInput_0.getByte(0))); /* 082 */ /* 083 */ /* 084 */ if ((tmpInput_0.isNullAt(1))) { /* 085 */ mutableStateArray_1[1].setNullAt(1); /* 086 */ } else { /* 087 */ mutableStateArray_1[1].write(1, (tmpInput_0.getInt(1))); /* 088 */ } /* 089 */ /* 090 */ /* 091 */ if ((tmpInput_0.isNullAt(2))) { /* 092 */ mutableStateArray_1[1].setNullAt(2); /* 093 */ } else { /* 094 */ // Remember the current cursor so that we can calculate how many bytes are /* 095 */ // written later. /* 096 */ final int previousCursor_1 = mutableStateArray_1[1].cursor(); /* 097 */ /* 098 */ final ArrayData tmpInput_1 = (tmpInput_0.getArray(2)); /* 099 */ if (tmpInput_1 instanceof UnsafeArrayData) { /* 100 */ mutableStateArray_1[1].write((UnsafeArrayData) tmpInput_1); /* 101 */ } else { /* 102 */ final int numElements_0 = tmpInput_1.numElements(); /* 103 */ mutableStateArray_2[0].initialize(numElements_0); /* 104 */ /* 105 */ for (int index_0 = 0; index_0 < numElements_0; index_0++) { /* 106 */ mutableStateArray_2[0].write(index_0, tmpInput_1.getInt(index_0)); /* 107 */ } /* 108 */ } /* 109 */ /* 110 */ mutableStateArray_1[1].setOffsetAndSizeFromPreviousCursor(2, previousCursor_1); /* 111 */ } /* 112 */ /* 113 */ /* 114 */ if ((tmpInput_0.isNullAt(3))) { /* 115 */ mutableStateArray_1[1].setNullAt(3); /* 116 */ } else { /* 117 */ // Remember the current cursor so that we can calculate how many bytes are /* 118 */ // written later. /* 119 */ final int previousCursor_2 = mutableStateArray_1[1].cursor(); /* 120 */ /* 121 */ final ArrayData tmpInput_2 = (tmpInput_0.getArray(3)); /* 122 */ if (tmpInput_2 instanceof UnsafeArrayData) { /* 123 */ mutableStateArray_1[1].write((UnsafeArrayData) tmpInput_2); /* 124 */ } else { /* 125 */ final int numElements_1 = tmpInput_2.numElements(); /* 126 */ mutableStateArray_2[1].initialize(numElements_1); /* 127 */ /* 128 */ for (int index_1 = 0; index_1 < numElements_1; index_1++) { /* 129 */ mutableStateArray_2[1].write(index_1, tmpInput_2.getDouble(index_1)); /* 130 */ } /* 131 */ } /* 132 */ /* 133 */ mutableStateArray_1[1].setOffsetAndSizeFromPreviousCursor(3, previousCursor_2); /* 134 */ } /* 135 */ /* 136 */ /* 137 */ mutableStateArray_1[0].setOffsetAndSizeFromPreviousCursor(0, previousCursor_0); /* 138 */ } /* 139 */ } /* 140 */ return (mutableStateArray_1[0].getRow()); /* 141 */ } /* 142 */ /* 143 */ /* 144 */ } 21/01/17 17:43:06.952 Thread-3 INFO CodeGenerator: Code generated in 45.184905 ms 21/01/17 17:43:06.952 Thread-3 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$doExecute$4$adapted 21/01/17 17:43:06.954 Thread-3 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$doExecute$4$adapted) is now cleaned +++ 21/01/17 17:43:06.957 Executor task launch worker for task 3.0 in stage 0.0 (TID 3) INFO CodeGenerator: Code generated in 59.21731 ms 21/01/17 17:43:06.963 Executor task launch worker for task 1.0 in stage 0.0 (TID 1) INFO PythonRunner: Times: total = 629, boot = 618, init = 10, finish = 1 21/01/17 17:43:06.963 Executor task launch worker for task 2.0 in stage 0.0 (TID 2) INFO PythonRunner: Times: total = 642, boot = 621, init = 21, finish = 0 21/01/17 17:43:06.968 Executor task launch worker for task 1.0 in stage 0.0 (TID 1) INFO Executor: Finished task 1.0 in stage 0.0 (TID 1). 2117 bytes result sent to driver 21/01/17 17:43:06.968 Executor task launch worker for task 1.0 in stage 0.0 (TID 1) DEBUG ExecutorMetricsPoller: stageTCMP: (0, 0) -> 2 21/01/17 17:43:06.968 Executor task launch worker for task 3.0 in stage 0.0 (TID 3) INFO PythonRunner: Times: total = 648, boot = 603, init = 45, finish = 0 21/01/17 17:43:06.969 Executor task launch worker for task 2.0 in stage 0.0 (TID 2) INFO Executor: Finished task 2.0 in stage 0.0 (TID 2). 2103 bytes result sent to driver 21/01/17 17:43:06.969 Executor task launch worker for task 2.0 in stage 0.0 (TID 2) DEBUG ExecutorMetricsPoller: stageTCMP: (0, 0) -> 1 21/01/17 17:43:06.979 Executor task launch worker for task 3.0 in stage 0.0 (TID 3) INFO Executor: Finished task 3.0 in stage 0.0 (TID 3). 2091 bytes result sent to driver 21/01/17 17:43:06.980 Executor task launch worker for task 3.0 in stage 0.0 (TID 3) DEBUG ExecutorMetricsPoller: removing (0, 0) from stageTCMP 21/01/17 17:43:06.987 Executor task launch worker for task 0.0 in stage 1.0 (TID 1) DEBUG GenerateSafeProjection: code for createexternalrow(input[0, int, false], input[1, int, false], input[2, float, true], StructField(UDF(user),IntegerType,false), StructField(UDF(item),IntegerType,false), StructField(rating,FloatType,true)): /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificSafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificSafeProjection extends org.apache.spark.sql.catalyst.expressions.codegen.BaseProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private InternalRow mutableRow; /* 009 */ /* 010 */ /* 011 */ public SpecificSafeProjection(Object[] references) { /* 012 */ this.references = references; /* 013 */ mutableRow = (InternalRow) references[references.length - 1]; /* 014 */ /* 015 */ } /* 016 */ /* 017 */ public void initialize(int partitionIndex) { /* 018 */ /* 019 */ } /* 020 */ /* 021 */ public java.lang.Object apply(java.lang.Object _i) { /* 022 */ InternalRow i = (InternalRow) _i; /* 023 */ Object[] values_0 = new Object[3]; /* 024 */ /* 025 */ int value_1 = i.getInt(0); /* 026 */ if (false) { /* 027 */ values_0[0] = null; /* 028 */ } else { /* 029 */ values_0[0] = value_1; /* 030 */ } /* 031 */ /* 032 */ int value_2 = i.getInt(1); /* 033 */ if (false) { /* 034 */ values_0[1] = null; /* 035 */ } else { /* 036 */ values_0[1] = value_2; /* 037 */ } /* 038 */ /* 039 */ boolean isNull_3 = i.isNullAt(2); /* 040 */ float value_3 = isNull_3 ? /* 041 */ -1.0f : (i.getFloat(2)); /* 042 */ if (isNull_3) { /* 043 */ values_0[2] = null; /* 044 */ } else { /* 045 */ values_0[2] = value_3; /* 046 */ } /* 047 */ /* 048 */ final org.apache.spark.sql.Row value_0 = new org.apache.spark.sql.catalyst.expressions.GenericRowWithSchema(values_0, ((org.apache.spark.sql.types.StructType) references[0] /* schema */)); /* 049 */ if (false) { /* 050 */ mutableRow.setNullAt(0); /* 051 */ } else { /* 052 */ /* 053 */ mutableRow.update(0, value_0); /* 054 */ } /* 055 */ /* 056 */ return mutableRow; /* 057 */ } /* 058 */ /* 059 */ /* 060 */ } 21/01/17 17:43:06.998 task-result-getter-2 INFO TaskSetManager: Finished task 2.0 in stage 0.0 (TID 2) in 1848 ms on 192.168.1.225 (executor driver) (2/4) 21/01/17 17:43:06.998 task-result-getter-1 INFO TaskSetManager: Finished task 1.0 in stage 0.0 (TID 1) in 1851 ms on 192.168.1.225 (executor driver) (3/4) 21/01/17 17:43:06.999 task-result-getter-3 INFO TaskSetManager: Finished task 3.0 in stage 0.0 (TID 3) in 1846 ms on 192.168.1.225 (executor driver) (4/4) 21/01/17 17:43:07.004 task-result-getter-3 INFO TaskSchedulerImpl: Removed TaskSet 0.0, whose tasks have all completed, from pool 21/01/17 17:43:07.005 dag-scheduler-event-loop INFO DAGScheduler: ResultStage 0 ( at /usr/lib/python3.6/runpy.py:85) finished in 2.072 s 21/01/17 17:43:06.991 Executor task launch worker for task 3.0 in stage 1.0 (TID 4) DEBUG GenerateSafeProjection: code for createexternalrow(input[0, int, false], input[1, int, false], input[2, float, true], StructField(UDF(user),IntegerType,false), StructField(UDF(item),IntegerType,false), StructField(rating,FloatType,true)): /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificSafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificSafeProjection extends org.apache.spark.sql.catalyst.expressions.codegen.BaseProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private InternalRow mutableRow; /* 009 */ /* 010 */ /* 011 */ public SpecificSafeProjection(Object[] references) { /* 012 */ this.references = references; /* 013 */ mutableRow = (InternalRow) references[references.length - 1]; /* 014 */ /* 015 */ } /* 016 */ /* 017 */ public void initialize(int partitionIndex) { /* 018 */ /* 019 */ } /* 020 */ /* 021 */ public java.lang.Object apply(java.lang.Object _i) { /* 022 */ InternalRow i = (InternalRow) _i; /* 023 */ Object[] values_0 = new Object[3]; /* 024 */ /* 025 */ int value_1 = i.getInt(0); /* 026 */ if (false) { /* 027 */ values_0[0] = null; /* 028 */ } else { /* 029 */ values_0[0] = value_1; /* 030 */ } /* 031 */ /* 032 */ int value_2 = i.getInt(1); /* 033 */ if (false) { /* 034 */ values_0[1] = null; /* 035 */ } else { /* 036 */ values_0[1] = value_2; /* 037 */ } /* 038 */ /* 039 */ boolean isNull_3 = i.isNullAt(2); /* 040 */ float value_3 = isNull_3 ? /* 041 */ -1.0f : (i.getFloat(2)); /* 042 */ if (isNull_3) { /* 043 */ values_0[2] = null; /* 044 */ } else { /* 045 */ values_0[2] = value_3; /* 046 */ } /* 047 */ /* 048 */ final org.apache.spark.sql.Row value_0 = new org.apache.spark.sql.catalyst.expressions.GenericRowWithSchema(values_0, ((org.apache.spark.sql.types.StructType) references[0] /* schema */)); /* 049 */ if (false) { /* 050 */ mutableRow.setNullAt(0); /* 051 */ } else { /* 052 */ /* 053 */ mutableRow.update(0, value_0); /* 054 */ } /* 055 */ /* 056 */ return mutableRow; /* 057 */ } /* 058 */ /* 059 */ /* 060 */ } 21/01/17 17:43:07.015 dag-scheduler-event-loop DEBUG DAGScheduler: After removal of stage 0, remaining stages = 0 21/01/17 17:43:07.015 dag-scheduler-event-loop INFO DAGScheduler: Job 0 is finished. Cancelling potential speculative or zombie tasks for this job 21/01/17 17:43:07.016 dag-scheduler-event-loop INFO TaskSchedulerImpl: Killing all running tasks in stage 0: Stage finished 21/01/17 17:43:07.010 Executor task launch worker for task 1.0 in stage 1.0 (TID 2) DEBUG GenerateSafeProjection: code for createexternalrow(input[0, int, false], input[1, int, false], input[2, float, true], StructField(UDF(user),IntegerType,false), StructField(UDF(item),IntegerType,false), StructField(rating,FloatType,true)): /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificSafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificSafeProjection extends org.apache.spark.sql.catalyst.expressions.codegen.BaseProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private InternalRow mutableRow; /* 009 */ /* 010 */ /* 011 */ public SpecificSafeProjection(Object[] references) { /* 012 */ this.references = references; /* 013 */ mutableRow = (InternalRow) references[references.length - 1]; /* 014 */ /* 015 */ } /* 016 */ /* 017 */ public void initialize(int partitionIndex) { /* 018 */ /* 019 */ } /* 020 */ /* 021 */ public java.lang.Object apply(java.lang.Object _i) { /* 022 */ InternalRow i = (InternalRow) _i; /* 023 */ Object[] values_0 = new Object[3]; /* 024 */ /* 025 */ int value_1 = i.getInt(0); /* 026 */ if (false) { /* 027 */ values_0[0] = null; /* 028 */ } else { /* 029 */ values_0[0] = value_1; /* 030 */ } /* 031 */ /* 032 */ int value_2 = i.getInt(1); /* 033 */ if (false) { /* 034 */ values_0[1] = null; /* 035 */ } else { /* 036 */ values_0[1] = value_2; /* 037 */ } /* 038 */ /* 039 */ boolean isNull_3 = i.isNullAt(2); /* 040 */ float value_3 = isNull_3 ? /* 041 */ -1.0f : (i.getFloat(2)); /* 042 */ if (isNull_3) { /* 043 */ values_0[2] = null; /* 044 */ } else { /* 045 */ values_0[2] = value_3; /* 046 */ } /* 047 */ /* 048 */ final org.apache.spark.sql.Row value_0 = new org.apache.spark.sql.catalyst.expressions.GenericRowWithSchema(values_0, ((org.apache.spark.sql.types.StructType) references[0] /* schema */)); /* 049 */ if (false) { /* 050 */ mutableRow.setNullAt(0); /* 051 */ } else { /* 052 */ /* 053 */ mutableRow.update(0, value_0); /* 054 */ } /* 055 */ /* 056 */ return mutableRow; /* 057 */ } /* 058 */ /* 059 */ /* 060 */ } 21/01/17 17:43:07.018 Thread-3 INFO DAGScheduler: Job 0 finished: at /usr/lib/python3.6/runpy.py:85, took 2.153469 s 21/01/17 17:43:07.024 Thread-3 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$rdd$1 21/01/17 17:43:07.023 Executor task launch worker for task 2.0 in stage 1.0 (TID 3) DEBUG GenerateSafeProjection: code for createexternalrow(input[0, int, false], input[1, int, false], input[2, float, true], StructField(UDF(user),IntegerType,false), StructField(UDF(item),IntegerType,false), StructField(rating,FloatType,true)): /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificSafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificSafeProjection extends org.apache.spark.sql.catalyst.expressions.codegen.BaseProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private InternalRow mutableRow; /* 009 */ /* 010 */ /* 011 */ public SpecificSafeProjection(Object[] references) { /* 012 */ this.references = references; /* 013 */ mutableRow = (InternalRow) references[references.length - 1]; /* 014 */ /* 015 */ } /* 016 */ /* 017 */ public void initialize(int partitionIndex) { /* 018 */ /* 019 */ } /* 020 */ /* 021 */ public java.lang.Object apply(java.lang.Object _i) { /* 022 */ InternalRow i = (InternalRow) _i; /* 023 */ Object[] values_0 = new Object[3]; /* 024 */ /* 025 */ int value_1 = i.getInt(0); /* 026 */ if (false) { /* 027 */ values_0[0] = null; /* 028 */ } else { /* 029 */ values_0[0] = value_1; /* 030 */ } /* 031 */ /* 032 */ int value_2 = i.getInt(1); /* 033 */ if (false) { /* 034 */ values_0[1] = null; /* 035 */ } else { /* 036 */ values_0[1] = value_2; /* 037 */ } /* 038 */ /* 039 */ boolean isNull_3 = i.isNullAt(2); /* 040 */ float value_3 = isNull_3 ? /* 041 */ -1.0f : (i.getFloat(2)); /* 042 */ if (isNull_3) { /* 043 */ values_0[2] = null; /* 044 */ } else { /* 045 */ values_0[2] = value_3; /* 046 */ } /* 047 */ /* 048 */ final org.apache.spark.sql.Row value_0 = new org.apache.spark.sql.catalyst.expressions.GenericRowWithSchema(values_0, ((org.apache.spark.sql.types.StructType) references[0] /* schema */)); /* 049 */ if (false) { /* 050 */ mutableRow.setNullAt(0); /* 051 */ } else { /* 052 */ /* 053 */ mutableRow.update(0, value_0); /* 054 */ } /* 055 */ /* 056 */ return mutableRow; /* 057 */ } /* 058 */ /* 059 */ /* 060 */ } 21/01/17 17:43:07.027 Thread-3 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$rdd$1) is now cleaned +++ 21/01/17 17:43:07.046 Executor task launch worker for task 0.0 in stage 1.0 (TID 1) INFO PythonRunner: Times: total = 107, boot = 33, init = 74, finish = 0 21/01/17 17:43:07.047 Thread-3 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$computeClusterStats$5 21/01/17 17:43:07.047 Thread-3 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$computeClusterStats$5) is now cleaned +++ 21/01/17 17:43:07.055 Executor task launch worker for task 3.0 in stage 1.0 (TID 4) INFO PythonRunner: Times: total = 116, boot = 36, init = 80, finish = 0 21/01/17 17:43:07.054 Executor task launch worker for task 1.0 in stage 1.0 (TID 2) INFO PythonRunner: Times: total = 116, boot = 55, init = 61, finish = 0 21/01/17 17:43:07.062 Executor task launch worker for task 2.0 in stage 1.0 (TID 3) INFO PythonRunner: Times: total = 125, boot = 61, init = 64, finish = 0 21/01/17 17:43:07.065 Thread-3 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$computeClusterStats$6 21/01/17 17:43:07.065 Thread-3 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$computeClusterStats$6) is now cleaned +++ 21/01/17 17:43:07.069 Thread-3 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$aggregateByKey$3 21/01/17 17:43:07.071 Thread-3 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$aggregateByKey$3) is now cleaned +++ 21/01/17 17:43:07.079 Executor task launch worker for task 2.0 in stage 1.0 (TID 3) DEBUG LocalDiskShuffleMapOutputWriter: Writing shuffle index file for mapId 3 with length 4 21/01/17 17:43:07.080 Executor task launch worker for task 3.0 in stage 1.0 (TID 4) DEBUG LocalDiskShuffleMapOutputWriter: Writing shuffle index file for mapId 4 with length 4 21/01/17 17:43:07.080 Executor task launch worker for task 1.0 in stage 1.0 (TID 2) DEBUG LocalDiskShuffleMapOutputWriter: Writing shuffle index file for mapId 2 with length 4 21/01/17 17:43:07.079 Executor task launch worker for task 0.0 in stage 1.0 (TID 1) DEBUG LocalDiskShuffleMapOutputWriter: Writing shuffle index file for mapId 1 with length 4 21/01/17 17:43:07.080 Thread-3 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$computeClusterStats$6 21/01/17 17:43:07.081 Thread-3 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$computeClusterStats$6) is now cleaned +++ 21/01/17 17:43:07.081 Thread-3 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$computeClusterStats$7 21/01/17 17:43:07.082 Thread-3 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$computeClusterStats$7) is now cleaned +++ 21/01/17 17:43:07.086 Thread-3 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$collect$2 21/01/17 17:43:07.086 Executor task launch worker for task 2.0 in stage 1.0 (TID 3) DEBUG IndexShuffleBlockResolver: Shuffle index for mapId 3: [0,0,325,0] 21/01/17 17:43:07.087 Executor task launch worker for task 3.0 in stage 1.0 (TID 4) DEBUG IndexShuffleBlockResolver: Shuffle index for mapId 4: [329,0,0,328] 21/01/17 17:43:07.086 Executor task launch worker for task 1.0 in stage 1.0 (TID 2) DEBUG IndexShuffleBlockResolver: Shuffle index for mapId 2: [0,0,327,324] 21/01/17 17:43:07.086 Executor task launch worker for task 0.0 in stage 1.0 (TID 1) DEBUG IndexShuffleBlockResolver: Shuffle index for mapId 1: [0,325,0,0] 21/01/17 17:43:07.090 Thread-3 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$collect$2) is now cleaned +++ 21/01/17 17:43:07.097 Executor task launch worker for task 1.0 in stage 1.0 (TID 2) INFO Executor: Finished task 1.0 in stage 1.0 (TID 2). 2030 bytes result sent to driver 21/01/17 17:43:07.098 Executor task launch worker for task 1.0 in stage 1.0 (TID 2) DEBUG ExecutorMetricsPoller: stageTCMP: (1, 0) -> 3 21/01/17 17:43:07.099 Thread-3 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$runJob$5 21/01/17 17:43:07.100 dispatcher-event-loop-1 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_1.0, runningTasks: 3 21/01/17 17:43:07.101 dispatcher-event-loop-1 DEBUG TaskSetManager: No tasks for locality level NO_PREF, so moving to locality level ANY 21/01/17 17:43:07.101 Executor task launch worker for task 3.0 in stage 1.0 (TID 4) INFO Executor: Finished task 3.0 in stage 1.0 (TID 4). 2030 bytes result sent to driver 21/01/17 17:43:07.101 Executor task launch worker for task 2.0 in stage 1.0 (TID 3) INFO Executor: Finished task 2.0 in stage 1.0 (TID 3). 2030 bytes result sent to driver 21/01/17 17:43:07.101 Executor task launch worker for task 0.0 in stage 1.0 (TID 1) INFO Executor: Finished task 0.0 in stage 1.0 (TID 1). 2030 bytes result sent to driver 21/01/17 17:43:07.101 Executor task launch worker for task 3.0 in stage 1.0 (TID 4) DEBUG ExecutorMetricsPoller: stageTCMP: (1, 0) -> 2 21/01/17 17:43:07.101 Executor task launch worker for task 0.0 in stage 1.0 (TID 1) DEBUG ExecutorMetricsPoller: stageTCMP: (1, 0) -> 1 21/01/17 17:43:07.101 Executor task launch worker for task 2.0 in stage 1.0 (TID 3) DEBUG ExecutorMetricsPoller: removing (1, 0) from stageTCMP 21/01/17 17:43:07.102 Thread-3 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$runJob$5) is now cleaned +++ 21/01/17 17:43:07.103 Thread-3 INFO SparkContext: Starting job: collectAsMap at ClusteringMetrics.scala:515 21/01/17 17:43:07.105 task-result-getter-1 INFO TaskSetManager: Finished task 1.0 in stage 1.0 (TID 2) in 371 ms on 192.168.1.225 (executor driver) (1/4) 21/01/17 17:43:07.107 dag-scheduler-event-loop DEBUG DAGScheduler: ShuffleMapTask finished on driver 21/01/17 17:43:07.107 task-result-getter-2 INFO TaskSetManager: Finished task 3.0 in stage 1.0 (TID 4) in 372 ms on 192.168.1.225 (executor driver) (2/4) 21/01/17 17:43:07.108 task-result-getter-3 INFO TaskSetManager: Finished task 2.0 in stage 1.0 (TID 3) in 373 ms on 192.168.1.225 (executor driver) (3/4) 21/01/17 17:43:07.109 task-result-getter-0 INFO TaskSetManager: Finished task 0.0 in stage 1.0 (TID 1) in 377 ms on 192.168.1.225 (executor driver) (4/4) 21/01/17 17:43:07.109 task-result-getter-0 INFO TaskSchedulerImpl: Removed TaskSet 1.0, whose tasks have all completed, from pool 21/01/17 17:43:07.108 dag-scheduler-event-loop DEBUG SortShuffleManager: Can't use serialized shuffle for shuffle 0 because the serializer, org.apache.spark.serializer.JavaSerializer, does not support object relocation 21/01/17 17:43:07.109 dag-scheduler-event-loop DEBUG DAGScheduler: ShuffleMapTask finished on driver 21/01/17 17:43:07.110 dag-scheduler-event-loop DEBUG DAGScheduler: Merging stage rdd profiles: Set() 21/01/17 17:43:07.111 dag-scheduler-event-loop DEBUG DAGScheduler: ShuffleMapTask finished on driver 21/01/17 17:43:07.112 dag-scheduler-event-loop DEBUG DAGScheduler: Merging stage rdd profiles: Set() 21/01/17 17:43:07.112 dag-scheduler-event-loop DEBUG DAGScheduler: ShuffleMapTask finished on driver 21/01/17 17:43:07.112 dag-scheduler-event-loop INFO DAGScheduler: ShuffleMapStage 1 (mapPartitions at ALS.scala:1369) finished in 0.410 s 21/01/17 17:43:07.113 dag-scheduler-event-loop INFO DAGScheduler: looking for newly runnable stages 21/01/17 17:43:07.113 dag-scheduler-event-loop INFO DAGScheduler: running: Set() 21/01/17 17:43:07.114 dag-scheduler-event-loop INFO DAGScheduler: Registering RDD 11 (map at ClusteringMetrics.scala:500) as input to shuffle 0 21/01/17 17:43:07.114 dag-scheduler-event-loop INFO DAGScheduler: waiting: Set(ShuffleMapStage 2, ResultStage 3) 21/01/17 17:43:07.114 dag-scheduler-event-loop INFO DAGScheduler: failed: Set() 21/01/17 17:43:07.115 dag-scheduler-event-loop DEBUG MapOutputTrackerMaster: Increasing epoch to 1 21/01/17 17:43:07.116 dag-scheduler-event-loop INFO DAGScheduler: Got job 1 (collectAsMap at ClusteringMetrics.scala:515) with 4 output partitions 21/01/17 17:43:07.116 dag-scheduler-event-loop INFO DAGScheduler: Final stage: ResultStage 2 (collectAsMap at ClusteringMetrics.scala:515) 21/01/17 17:43:07.116 dag-scheduler-event-loop INFO DAGScheduler: Parents of final stage: List(ShuffleMapStage 1) 21/01/17 17:43:07.117 dag-scheduler-event-loop DEBUG DAGScheduler: submitStage(ShuffleMapStage 2 (name=map at ALS.scala:1612;jobs=1)) 21/01/17 17:43:07.118 dag-scheduler-event-loop INFO DAGScheduler: Missing parents: List(ShuffleMapStage 1) 21/01/17 17:43:07.118 dag-scheduler-event-loop DEBUG DAGScheduler: submitStage(ResultStage 2 (name=collectAsMap at ClusteringMetrics.scala:515;jobs=1)) 21/01/17 17:43:07.119 dag-scheduler-event-loop DEBUG DAGScheduler: missing: List(ShuffleMapStage 1) 21/01/17 17:43:07.119 dag-scheduler-event-loop DEBUG DAGScheduler: missing: List() 21/01/17 17:43:07.119 dag-scheduler-event-loop INFO DAGScheduler: Submitting ShuffleMapStage 2 (MapPartitionsRDD[17] at map at ALS.scala:1612), which has no missing parents 21/01/17 17:43:07.119 dag-scheduler-event-loop DEBUG DAGScheduler: submitMissingTasks(ShuffleMapStage 2) 21/01/17 17:43:07.119 dag-scheduler-event-loop DEBUG DAGScheduler: submitStage(ShuffleMapStage 1 (name=map at ClusteringMetrics.scala:500;jobs=1)) 21/01/17 17:43:07.120 dag-scheduler-event-loop DEBUG DAGScheduler: missing: List() 21/01/17 17:43:07.120 dag-scheduler-event-loop INFO DAGScheduler: Submitting ShuffleMapStage 1 (MapPartitionsRDD[11] at map at ClusteringMetrics.scala:500), which has no missing parents 21/01/17 17:43:07.120 dag-scheduler-event-loop DEBUG DAGScheduler: submitMissingTasks(ShuffleMapStage 1) 21/01/17 17:43:07.131 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_2 stored as values in memory (estimated size 33.3 KiB, free 366.2 MiB) 21/01/17 17:43:07.132 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_2 locally took 1 ms 21/01/17 17:43:07.132 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_2 without replication took 1 ms 21/01/17 17:43:07.133 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_2_piece0 stored as bytes in memory (estimated size 15.8 KiB, free 366.2 MiB) 21/01/17 17:43:07.134 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master broadcast_2_piece0 for BlockManagerId(driver, 192.168.1.225, 40985, None) 21/01/17 17:43:07.134 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added broadcast_2_piece0 in memory on 192.168.1.225:40985 (size: 15.8 KiB, free: 366.3 MiB) 21/01/17 17:43:07.134 dag-scheduler-event-loop DEBUG BlockManagerMaster: Updated info of block broadcast_2_piece0 21/01/17 17:43:07.134 dag-scheduler-event-loop DEBUG BlockManager: Told master about block broadcast_2_piece0 21/01/17 17:43:07.135 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_2_piece0 locally took 1 ms 21/01/17 17:43:07.135 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_2_piece0 without replication took 1 ms 21/01/17 17:43:07.135 dag-scheduler-event-loop INFO SparkContext: Created broadcast 2 from broadcast at DAGScheduler.scala:1383 21/01/17 17:43:07.136 dag-scheduler-event-loop INFO DAGScheduler: Submitting 4 missing tasks from ShuffleMapStage 2 (MapPartitionsRDD[17] at map at ALS.scala:1612) (first 15 tasks are for partitions Vector(0, 1, 2, 3)) 21/01/17 17:43:07.136 dag-scheduler-event-loop INFO TaskSchedulerImpl: Adding task set 2.0 with 4 tasks resource profile 0 21/01/17 17:43:07.136 dag-scheduler-event-loop DEBUG TaskSetManager: Epoch for TaskSet 2.0: 1 21/01/17 17:43:07.138 dag-scheduler-event-loop DEBUG TaskSetManager: Adding pending tasks took 1 ms 21/01/17 17:43:07.139 dag-scheduler-event-loop DEBUG TaskSetManager: Valid locality levels for TaskSet 2.0: NODE_LOCAL, ANY 21/01/17 17:43:07.140 dispatcher-event-loop-3 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_2.0, runningTasks: 0 21/01/17 17:43:07.140 Thread-3 DEBUG CountVectorizerModel: Input schema: {"type":"struct","fields":[{"name":"id","type":"long","nullable":true,"metadata":{}},{"name":"words","type":{"type":"array","elementType":"string","containsNull":true},"nullable":true,"metadata":{}},{"name":"expected","type":{"type":"udt","class":"org.apache.spark.ml.linalg.VectorUDT","pyClass":"pyspark.ml.linalg.VectorUDT","sqlType":{"type":"struct","fields":[{"name":"type","type":"byte","nullable":false,"metadata":{}},{"name":"size","type":"integer","nullable":true,"metadata":{}},{"name":"indices","type":{"type":"array","elementType":"integer","containsNull":false},"nullable":true,"metadata":{}},{"name":"values","type":{"type":"array","elementType":"double","containsNull":false},"nullable":true,"metadata":{}}]}},"nullable":true,"metadata":{}}]} 21/01/17 17:43:07.142 Thread-3 DEBUG CountVectorizerModel: Expected output schema: {"type":"struct","fields":[{"name":"id","type":"long","nullable":true,"metadata":{}},{"name":"words","type":{"type":"array","elementType":"string","containsNull":true},"nullable":true,"metadata":{}},{"name":"expected","type":{"type":"udt","class":"org.apache.spark.ml.linalg.VectorUDT","pyClass":"pyspark.ml.linalg.VectorUDT","sqlType":{"type":"struct","fields":[{"name":"type","type":"byte","nullable":false,"metadata":{}},{"name":"size","type":"integer","nullable":true,"metadata":{}},{"name":"indices","type":{"type":"array","elementType":"integer","containsNull":false},"nullable":true,"metadata":{}},{"name":"values","type":{"type":"array","elementType":"double","containsNull":false},"nullable":true,"metadata":{}}]}},"nullable":true,"metadata":{}},{"name":"CountVectorizerModel_ee3a53d9e8b8__output","type":{"type":"udt","class":"org.apache.spark.ml.linalg.VectorUDT","pyClass":"pyspark.ml.linalg.VectorUDT","sqlType":{"type":"struct","fields":[{"name":"type","type":"byte","nullable":false,"metadata":{}},{"name":"size","type":"integer","nullable":true,"metadata":{}},{"name":"indices","type":{"type":"array","elementType":"integer","containsNull":false},"nullable":true,"metadata":{}},{"name":"values","type":{"type":"array","elementType":"double","containsNull":false},"nullable":true,"metadata":{}}]}},"nullable":false,"metadata":{"ml_attr":{"attrs":{},"num_attrs":3}}}]} 21/01/17 17:43:07.142 dispatcher-event-loop-3 INFO TaskSetManager: Starting task 0.0 in stage 2.0 (TID 5) (192.168.1.225, executor driver, partition 0, NODE_LOCAL, 7132 bytes) taskResourceAssignments Map() 21/01/17 17:43:07.143 dispatcher-event-loop-3 INFO TaskSetManager: Starting task 1.0 in stage 2.0 (TID 6) (192.168.1.225, executor driver, partition 1, NODE_LOCAL, 7132 bytes) taskResourceAssignments Map() 21/01/17 17:43:07.143 dispatcher-event-loop-3 INFO TaskSetManager: Starting task 2.0 in stage 2.0 (TID 7) (192.168.1.225, executor driver, partition 2, NODE_LOCAL, 7132 bytes) taskResourceAssignments Map() 21/01/17 17:43:07.144 Thread-3 INFO MemoryStore: Block broadcast_2 stored as values in memory (estimated size 256.0 B, free 366.3 MiB) 21/01/17 17:43:07.144 dispatcher-event-loop-3 INFO TaskSetManager: Starting task 3.0 in stage 2.0 (TID 8) (192.168.1.225, executor driver, partition 3, NODE_LOCAL, 7132 bytes) taskResourceAssignments Map() 21/01/17 17:43:07.144 Thread-3 DEBUG BlockManager: Put block broadcast_2 locally took 1 ms 21/01/17 17:43:07.144 Thread-3 DEBUG BlockManager: Putting block broadcast_2 without replication took 1 ms 21/01/17 17:43:07.144 Executor task launch worker for task 0.0 in stage 2.0 (TID 5) INFO Executor: Running task 0.0 in stage 2.0 (TID 5) 21/01/17 17:43:07.145 Executor task launch worker for task 2.0 in stage 2.0 (TID 7) INFO Executor: Running task 2.0 in stage 2.0 (TID 7) 21/01/17 17:43:07.145 Executor task launch worker for task 0.0 in stage 2.0 (TID 5) DEBUG ExecutorMetricsPoller: stageTCMP: (2, 0) -> 1 21/01/17 17:43:07.145 Executor task launch worker for task 3.0 in stage 2.0 (TID 8) INFO Executor: Running task 3.0 in stage 2.0 (TID 8) 21/01/17 17:43:07.145 Executor task launch worker for task 2.0 in stage 2.0 (TID 7) DEBUG ExecutorMetricsPoller: stageTCMP: (2, 0) -> 2 21/01/17 17:43:07.146 Executor task launch worker for task 3.0 in stage 2.0 (TID 8) DEBUG ExecutorMetricsPoller: stageTCMP: (2, 0) -> 3 21/01/17 17:43:07.146 Executor task launch worker for task 0.0 in stage 2.0 (TID 5) DEBUG BlockManager: Getting local block broadcast_2 21/01/17 17:43:07.146 Executor task launch worker for task 0.0 in stage 2.0 (TID 5) DEBUG BlockManager: Level for block broadcast_2 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:07.147 Executor task launch worker for task 1.0 in stage 2.0 (TID 6) INFO Executor: Running task 1.0 in stage 2.0 (TID 6) 21/01/17 17:43:07.148 Executor task launch worker for task 1.0 in stage 2.0 (TID 6) DEBUG ExecutorMetricsPoller: stageTCMP: (2, 0) -> 4 21/01/17 17:43:07.153 Thread-3 INFO MemoryStore: Block broadcast_2_piece0 stored as bytes in memory (estimated size 264.0 B, free 366.3 MiB) 21/01/17 17:43:07.154 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master broadcast_2_piece0 for BlockManagerId(driver, 192.168.1.225, 46295, None) 21/01/17 17:43:07.155 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added broadcast_2_piece0 in memory on 192.168.1.225:46295 (size: 264.0 B, free: 366.3 MiB) 21/01/17 17:43:07.155 Thread-3 DEBUG BlockManagerMaster: Updated info of block broadcast_2_piece0 21/01/17 17:43:07.155 Thread-3 DEBUG BlockManager: Told master about block broadcast_2_piece0 21/01/17 17:43:07.155 Thread-3 DEBUG BlockManager: Put block broadcast_2_piece0 locally took 2 ms 21/01/17 17:43:07.156 Thread-3 DEBUG BlockManager: Putting block broadcast_2_piece0 without replication took 2 ms 21/01/17 17:43:07.156 Thread-3 INFO SparkContext: Created broadcast 2 from broadcast at CountVectorizer.scala:309 21/01/17 17:43:07.157 Executor task launch worker for task 0.0 in stage 2.0 (TID 5) DEBUG BlockManager: Getting local block rdd_16_0 21/01/17 17:43:07.157 Executor task launch worker for task 3.0 in stage 2.0 (TID 8) DEBUG BlockManager: Getting local block rdd_16_3 21/01/17 17:43:07.158 Executor task launch worker for task 0.0 in stage 2.0 (TID 5) DEBUG BlockManager: Block rdd_16_0 was not found 21/01/17 17:43:07.158 Executor task launch worker for task 3.0 in stage 2.0 (TID 8) DEBUG BlockManager: Block rdd_16_3 was not found 21/01/17 17:43:07.159 Executor task launch worker for task 1.0 in stage 2.0 (TID 6) DEBUG BlockManager: Getting local block rdd_16_1 21/01/17 17:43:07.159 Executor task launch worker for task 3.0 in stage 2.0 (TID 8) DEBUG BlockManager: Getting remote block rdd_16_3 21/01/17 17:43:07.159 Executor task launch worker for task 1.0 in stage 2.0 (TID 6) DEBUG BlockManager: Block rdd_16_1 was not found 21/01/17 17:43:07.160 Executor task launch worker for task 1.0 in stage 2.0 (TID 6) DEBUG BlockManager: Getting remote block rdd_16_1 21/01/17 17:43:07.161 Executor task launch worker for task 0.0 in stage 2.0 (TID 5) DEBUG BlockManager: Getting remote block rdd_16_0 21/01/17 17:43:07.162 Executor task launch worker for task 3.0 in stage 2.0 (TID 8) DEBUG BlockManager: Block rdd_16_3 is unknown by block manager master 21/01/17 17:43:07.163 Executor task launch worker for task 0.0 in stage 2.0 (TID 5) DEBUG BlockManager: Block rdd_16_0 is unknown by block manager master 21/01/17 17:43:07.163 Executor task launch worker for task 0.0 in stage 2.0 (TID 5) DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 1 21/01/17 17:43:07.163 Executor task launch worker for task 1.0 in stage 2.0 (TID 6) DEBUG BlockManager: Block rdd_16_1 is unknown by block manager master 21/01/17 17:43:07.164 Executor task launch worker for task 1.0 in stage 2.0 (TID 6) DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 1 21/01/17 17:43:07.164 Executor task launch worker for task 2.0 in stage 2.0 (TID 7) DEBUG BlockManager: Getting local block rdd_16_2 21/01/17 17:43:07.164 Executor task launch worker for task 2.0 in stage 2.0 (TID 7) DEBUG BlockManager: Block rdd_16_2 was not found 21/01/17 17:43:07.164 Executor task launch worker for task 2.0 in stage 2.0 (TID 7) DEBUG BlockManager: Getting remote block rdd_16_2 21/01/17 17:43:07.163 Executor task launch worker for task 3.0 in stage 2.0 (TID 8) DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 1 21/01/17 17:43:07.165 Executor task launch worker for task 2.0 in stage 2.0 (TID 7) DEBUG BlockManager: Block rdd_16_2 is unknown by block manager master 21/01/17 17:43:07.165 Executor task launch worker for task 2.0 in stage 2.0 (TID 7) DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 1 21/01/17 17:43:07.169 Executor task launch worker for task 0.0 in stage 2.0 (TID 5) DEBUG MapOutputTrackerMaster: Convert map statuses for shuffle 1, mappers 0-4, partitions 0-1 21/01/17 17:43:07.169 Executor task launch worker for task 2.0 in stage 2.0 (TID 7) DEBUG MapOutputTrackerMaster: Convert map statuses for shuffle 1, mappers 0-4, partitions 2-3 21/01/17 17:43:07.169 Executor task launch worker for task 1.0 in stage 2.0 (TID 6) DEBUG MapOutputTrackerMaster: Convert map statuses for shuffle 1, mappers 0-4, partitions 1-2 21/01/17 17:43:07.174 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_1 stored as values in memory (estimated size 29.8 KiB, free 366.3 MiB) 21/01/17 17:43:07.175 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_1 locally took 1 ms 21/01/17 17:43:07.175 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_1 without replication took 1 ms 21/01/17 17:43:07.169 Executor task launch worker for task 3.0 in stage 2.0 (TID 8) DEBUG MapOutputTrackerMaster: Convert map statuses for shuffle 1, mappers 0-4, partitions 3-4 21/01/17 17:43:07.177 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_1_piece0 stored as bytes in memory (estimated size 13.3 KiB, free 366.3 MiB) 21/01/17 17:43:07.178 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master broadcast_1_piece0 for BlockManagerId(driver, 192.168.1.225, 33787, None) 21/01/17 17:43:07.178 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added broadcast_1_piece0 in memory on 192.168.1.225:33787 (size: 13.3 KiB, free: 366.3 MiB) 21/01/17 17:43:07.178 dag-scheduler-event-loop DEBUG BlockManagerMaster: Updated info of block broadcast_1_piece0 21/01/17 17:43:07.179 dag-scheduler-event-loop DEBUG BlockManager: Told master about block broadcast_1_piece0 21/01/17 17:43:07.179 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_1_piece0 locally took 2 ms 21/01/17 17:43:07.179 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_1_piece0 without replication took 2 ms 21/01/17 17:43:07.179 dag-scheduler-event-loop INFO SparkContext: Created broadcast 1 from broadcast at DAGScheduler.scala:1383 21/01/17 17:43:07.181 dag-scheduler-event-loop INFO DAGScheduler: Submitting 4 missing tasks from ShuffleMapStage 1 (MapPartitionsRDD[11] at map at ClusteringMetrics.scala:500) (first 15 tasks are for partitions Vector(0, 1, 2, 3)) 21/01/17 17:43:07.182 dag-scheduler-event-loop INFO TaskSchedulerImpl: Adding task set 1.0 with 4 tasks resource profile 0 21/01/17 17:43:07.182 dag-scheduler-event-loop DEBUG TaskSetManager: Epoch for TaskSet 1.0: 0 21/01/17 17:43:07.182 dag-scheduler-event-loop DEBUG TaskSetManager: Adding pending tasks took 0 ms 21/01/17 17:43:07.183 dag-scheduler-event-loop DEBUG TaskSetManager: Valid locality levels for TaskSet 1.0: NO_PREF, ANY 21/01/17 17:43:07.183 dispatcher-event-loop-1 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_1.0, runningTasks: 0 21/01/17 17:43:07.185 dispatcher-event-loop-1 INFO TaskSetManager: Starting task 0.0 in stage 1.0 (TID 1) (192.168.1.225, executor driver, partition 0, PROCESS_LOCAL, 7387 bytes) taskResourceAssignments Map() 21/01/17 17:43:07.185 dispatcher-event-loop-1 INFO TaskSetManager: Starting task 1.0 in stage 1.0 (TID 2) (192.168.1.225, executor driver, partition 1, PROCESS_LOCAL, 7452 bytes) taskResourceAssignments Map() 21/01/17 17:43:07.186 Executor task launch worker for task 0.0 in stage 2.0 (TID 5) DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 21/01/17 17:43:07.186 dispatcher-event-loop-1 INFO TaskSetManager: Starting task 2.0 in stage 1.0 (TID 3) (192.168.1.225, executor driver, partition 2, PROCESS_LOCAL, 7387 bytes) taskResourceAssignments Map() 21/01/17 17:43:07.186 dispatcher-event-loop-1 INFO TaskSetManager: Starting task 3.0 in stage 1.0 (TID 4) (192.168.1.225, executor driver, partition 3, PROCESS_LOCAL, 7452 bytes) taskResourceAssignments Map() 21/01/17 17:43:07.187 Executor task launch worker for task 0.0 in stage 1.0 (TID 1) INFO Executor: Running task 0.0 in stage 1.0 (TID 1) 21/01/17 17:43:07.188 Executor task launch worker for task 1.0 in stage 1.0 (TID 2) INFO Executor: Running task 1.0 in stage 1.0 (TID 2) 21/01/17 17:43:07.189 Executor task launch worker for task 2.0 in stage 1.0 (TID 3) INFO Executor: Running task 2.0 in stage 1.0 (TID 3) 21/01/17 17:43:07.189 Executor task launch worker for task 0.0 in stage 1.0 (TID 1) DEBUG ExecutorMetricsPoller: stageTCMP: (1, 0) -> 1 21/01/17 17:43:07.189 Thread-3 DEBUG Analyzer$ResolveReferences: Resolving 'words to words#1 21/01/17 17:43:07.191 Executor task launch worker for task 0.0 in stage 1.0 (TID 1) DEBUG BlockManager: Getting local block broadcast_1 21/01/17 17:43:07.191 Executor task launch worker for task 0.0 in stage 1.0 (TID 1) DEBUG BlockManager: Level for block broadcast_1 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:07.188 Executor task launch worker for task 3.0 in stage 2.0 (TID 8) DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 21/01/17 17:43:07.196 Executor task launch worker for task 2.0 in stage 2.0 (TID 7) DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 21/01/17 17:43:07.189 Executor task launch worker for task 1.0 in stage 1.0 (TID 2) DEBUG ExecutorMetricsPoller: stageTCMP: (1, 0) -> 2 21/01/17 17:43:07.190 Executor task launch worker for task 2.0 in stage 1.0 (TID 3) DEBUG ExecutorMetricsPoller: stageTCMP: (1, 0) -> 3 21/01/17 17:43:07.203 Executor task launch worker for task 3.0 in stage 1.0 (TID 4) INFO Executor: Running task 3.0 in stage 1.0 (TID 4) 21/01/17 17:43:07.203 Executor task launch worker for task 3.0 in stage 2.0 (TID 8) INFO ShuffleBlockFetcherIterator: Getting 2 (668.0 B) non-empty blocks including 2 (668.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 21/01/17 17:43:07.204 Executor task launch worker for task 3.0 in stage 1.0 (TID 4) DEBUG ExecutorMetricsPoller: stageTCMP: (1, 0) -> 4 21/01/17 17:43:07.205 Executor task launch worker for task 3.0 in stage 2.0 (TID 8) INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 17 ms 21/01/17 17:43:07.206 Executor task launch worker for task 3.0 in stage 2.0 (TID 8) DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_1_2_3,1), (shuffle_1_4_3,3) 21/01/17 17:43:07.206 Executor task launch worker for task 3.0 in stage 2.0 (TID 8) DEBUG BlockManager: Getting local shuffle block shuffle_1_2_3 21/01/17 17:43:07.209 Executor task launch worker for task 2.0 in stage 2.0 (TID 7) INFO ShuffleBlockFetcherIterator: Getting 2 (668.0 B) non-empty blocks including 2 (668.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 21/01/17 17:43:07.209 Executor task launch worker for task 2.0 in stage 2.0 (TID 7) INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 13 ms 21/01/17 17:43:07.209 Executor task launch worker for task 2.0 in stage 2.0 (TID 7) DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_1_2_2,1), (shuffle_1_3_2,2) 21/01/17 17:43:07.209 Executor task launch worker for task 2.0 in stage 2.0 (TID 7) DEBUG BlockManager: Getting local shuffle block shuffle_1_2_2 21/01/17 17:43:07.210 Executor task launch worker for task 2.0 in stage 2.0 (TID 7) DEBUG BlockManager: Getting local shuffle block shuffle_1_3_2 21/01/17 17:43:07.211 Executor task launch worker for task 2.0 in stage 2.0 (TID 7) DEBUG ShuffleBlockFetcherIterator: Got local blocks in 15 ms 21/01/17 17:43:07.217 Executor task launch worker for task 3.0 in stage 2.0 (TID 8) DEBUG BlockManager: Getting local shuffle block shuffle_1_4_3 21/01/17 17:43:07.217 Executor task launch worker for task 3.0 in stage 2.0 (TID 8) DEBUG ShuffleBlockFetcherIterator: Got local blocks in 29 ms 21/01/17 17:43:07.218 Executor task launch worker for task 1.0 in stage 2.0 (TID 6) DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 21/01/17 17:43:07.218 Executor task launch worker for task 0.0 in stage 2.0 (TID 5) INFO ShuffleBlockFetcherIterator: Getting 1 (334.0 B) non-empty blocks including 1 (334.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 21/01/17 17:43:07.219 Executor task launch worker for task 1.0 in stage 2.0 (TID 6) INFO ShuffleBlockFetcherIterator: Getting 1 (334.0 B) non-empty blocks including 1 (334.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 21/01/17 17:43:07.219 Executor task launch worker for task 0.0 in stage 2.0 (TID 5) INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 33 ms 21/01/17 17:43:07.219 Executor task launch worker for task 1.0 in stage 2.0 (TID 6) INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 1 ms 21/01/17 17:43:07.219 Executor task launch worker for task 0.0 in stage 2.0 (TID 5) DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_1_4_0,3) 21/01/17 17:43:07.219 Executor task launch worker for task 1.0 in stage 2.0 (TID 6) DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_1_1_1,0) 21/01/17 17:43:07.219 Executor task launch worker for task 0.0 in stage 2.0 (TID 5) DEBUG BlockManager: Getting local shuffle block shuffle_1_4_0 21/01/17 17:43:07.219 Executor task launch worker for task 1.0 in stage 2.0 (TID 6) DEBUG BlockManager: Getting local shuffle block shuffle_1_1_1 21/01/17 17:43:07.219 Executor task launch worker for task 1.0 in stage 2.0 (TID 6) DEBUG ShuffleBlockFetcherIterator: Got local blocks in 1 ms 21/01/17 17:43:07.223 Executor task launch worker for task 0.0 in stage 2.0 (TID 5) DEBUG ShuffleBlockFetcherIterator: Got local blocks in 38 ms 21/01/17 17:43:07.240 Thread-3 DEBUG Analyzer$ResolveReferences: Resolving 'CountVectorizerModel_ee3a53d9e8b8__output to CountVectorizerModel_ee3a53d9e8b8__output#25 21/01/17 17:43:07.267 Executor task launch worker for task 1.0 in stage 2.0 (TID 6) DEBUG TaskMemoryManager: Task 6 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@3e6c1f 21/01/17 17:43:07.268 Executor task launch worker for task 1.0 in stage 2.0 (TID 6) INFO MemoryStore: Block rdd_16_1 stored as values in memory (estimated size 176.0 B, free 366.2 MiB) 21/01/17 17:43:07.268 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_16_1 for BlockManagerId(driver, 192.168.1.225, 40985, None) 21/01/17 17:43:07.269 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_16_1 in memory on 192.168.1.225:40985 (size: 176.0 B, free: 366.3 MiB) 21/01/17 17:43:07.269 Executor task launch worker for task 1.0 in stage 2.0 (TID 6) DEBUG BlockManagerMaster: Updated info of block rdd_16_1 21/01/17 17:43:07.269 Executor task launch worker for task 1.0 in stage 2.0 (TID 6) DEBUG BlockManager: Told master about block rdd_16_1 21/01/17 17:43:07.269 Executor task launch worker for task 1.0 in stage 2.0 (TID 6) DEBUG BlockManager: Put block rdd_16_1 locally took 105 ms 21/01/17 17:43:07.270 Executor task launch worker for task 1.0 in stage 2.0 (TID 6) DEBUG BlockManager: Putting block rdd_16_1 without replication took 106 ms 21/01/17 17:43:07.270 Executor task launch worker for task 1.0 in stage 2.0 (TID 6) DEBUG BlockManager: Getting local block rdd_16_1 21/01/17 17:43:07.273 Executor task launch worker for task 1.0 in stage 2.0 (TID 6) DEBUG BlockManager: Level for block rdd_16_1 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:07.269 Executor task launch worker for task 0.0 in stage 2.0 (TID 5) DEBUG TaskMemoryManager: Task 5 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@20332e27 21/01/17 17:43:07.268 Executor task launch worker for task 2.0 in stage 2.0 (TID 7) DEBUG TaskMemoryManager: Task 7 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@37504aca 21/01/17 17:43:07.270 Executor task launch worker for task 3.0 in stage 2.0 (TID 8) DEBUG TaskMemoryManager: Task 8 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@19c85d11 21/01/17 17:43:07.274 Executor task launch worker for task 2.0 in stage 2.0 (TID 7) INFO MemoryStore: Block rdd_16_2 stored as values in memory (estimated size 328.0 B, free 366.2 MiB) 21/01/17 17:43:07.274 Executor task launch worker for task 3.0 in stage 2.0 (TID 8) INFO MemoryStore: Block rdd_16_3 stored as values in memory (estimated size 328.0 B, free 366.2 MiB) 21/01/17 17:43:07.274 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_16_2 for BlockManagerId(driver, 192.168.1.225, 40985, None) 21/01/17 17:43:07.276 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_16_2 in memory on 192.168.1.225:40985 (size: 328.0 B, free: 366.3 MiB) 21/01/17 17:43:07.276 Executor task launch worker for task 0.0 in stage 2.0 (TID 5) INFO MemoryStore: Block rdd_16_0 stored as values in memory (estimated size 176.0 B, free 366.2 MiB) 21/01/17 17:43:07.276 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_16_3 for BlockManagerId(driver, 192.168.1.225, 40985, None) 21/01/17 17:43:07.276 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_16_3 in memory on 192.168.1.225:40985 (size: 328.0 B, free: 366.3 MiB) 21/01/17 17:43:07.277 Executor task launch worker for task 3.0 in stage 2.0 (TID 8) DEBUG BlockManagerMaster: Updated info of block rdd_16_3 21/01/17 17:43:07.277 Executor task launch worker for task 2.0 in stage 2.0 (TID 7) DEBUG BlockManagerMaster: Updated info of block rdd_16_2 21/01/17 17:43:07.277 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_16_0 for BlockManagerId(driver, 192.168.1.225, 40985, None) 21/01/17 17:43:07.277 Executor task launch worker for task 2.0 in stage 2.0 (TID 7) DEBUG BlockManager: Told master about block rdd_16_2 21/01/17 17:43:07.277 Executor task launch worker for task 2.0 in stage 2.0 (TID 7) DEBUG BlockManager: Put block rdd_16_2 locally took 111 ms 21/01/17 17:43:07.277 Executor task launch worker for task 2.0 in stage 2.0 (TID 7) DEBUG BlockManager: Putting block rdd_16_2 without replication took 112 ms 21/01/17 17:43:07.277 Executor task launch worker for task 2.0 in stage 2.0 (TID 7) DEBUG BlockManager: Getting local block rdd_16_2 21/01/17 17:43:07.277 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_16_0 in memory on 192.168.1.225:40985 (size: 176.0 B, free: 366.3 MiB) 21/01/17 17:43:07.277 Executor task launch worker for task 2.0 in stage 2.0 (TID 7) DEBUG BlockManager: Level for block rdd_16_2 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:07.277 Executor task launch worker for task 3.0 in stage 2.0 (TID 8) DEBUG BlockManager: Told master about block rdd_16_3 21/01/17 17:43:07.278 Executor task launch worker for task 3.0 in stage 2.0 (TID 8) DEBUG BlockManager: Put block rdd_16_3 locally took 116 ms 21/01/17 17:43:07.278 Executor task launch worker for task 3.0 in stage 2.0 (TID 8) DEBUG BlockManager: Putting block rdd_16_3 without replication took 116 ms 21/01/17 17:43:07.279 Executor task launch worker for task 3.0 in stage 2.0 (TID 8) DEBUG BlockManager: Getting local block rdd_16_3 21/01/17 17:43:07.279 Executor task launch worker for task 3.0 in stage 2.0 (TID 8) DEBUG BlockManager: Level for block rdd_16_3 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:07.282 Executor task launch worker for task 0.0 in stage 2.0 (TID 5) DEBUG BlockManagerMaster: Updated info of block rdd_16_0 21/01/17 17:43:07.282 Executor task launch worker for task 0.0 in stage 2.0 (TID 5) DEBUG BlockManager: Told master about block rdd_16_0 21/01/17 17:43:07.282 Executor task launch worker for task 0.0 in stage 2.0 (TID 5) DEBUG BlockManager: Put block rdd_16_0 locally took 118 ms 21/01/17 17:43:07.282 Executor task launch worker for task 0.0 in stage 2.0 (TID 5) DEBUG BlockManager: Putting block rdd_16_0 without replication took 119 ms 21/01/17 17:43:07.282 Executor task launch worker for task 0.0 in stage 2.0 (TID 5) DEBUG BlockManager: Getting local block rdd_16_0 21/01/17 17:43:07.282 Executor task launch worker for task 0.0 in stage 2.0 (TID 5) DEBUG BlockManager: Level for block rdd_16_0 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:07.290 Executor task launch worker for task 2.0 in stage 2.0 (TID 7) DEBUG ALS: Converting to local indices took 0.011504228 seconds. 21/01/17 17:43:07.292 Executor task launch worker for task 3.0 in stage 2.0 (TID 8) DEBUG ALS: Converting to local indices took 0.010979315 seconds. 21/01/17 17:43:07.290 Executor task launch worker for task 1.0 in stage 2.0 (TID 6) DEBUG ALS: Converting to local indices took 0.014569299 seconds. 21/01/17 17:43:07.292 Executor task launch worker for task 0.0 in stage 2.0 (TID 5) DEBUG ALS: Converting to local indices took 0.008217735 seconds. 21/01/17 17:43:07.293 Executor task launch worker for task 0.0 in stage 2.0 (TID 5) DEBUG LocalDiskShuffleMapOutputWriter: Writing shuffle index file for mapId 5 with length 10 21/01/17 17:43:07.293 Executor task launch worker for task 1.0 in stage 2.0 (TID 6) DEBUG LocalDiskShuffleMapOutputWriter: Writing shuffle index file for mapId 6 with length 10 21/01/17 17:43:07.294 Executor task launch worker for task 0.0 in stage 2.0 (TID 5) DEBUG IndexShuffleBlockResolver: Shuffle index for mapId 5: [0,0,252,0,0,0,0,0,0,0] 21/01/17 17:43:07.294 Executor task launch worker for task 1.0 in stage 2.0 (TID 6) DEBUG IndexShuffleBlockResolver: Shuffle index for mapId 6: [244,0,0,0,0,0,0,0,0,0] 21/01/17 17:43:07.294 Executor task launch worker for task 2.0 in stage 2.0 (TID 7) DEBUG ALS: Converting to local indices took 0.001694861 seconds. 21/01/17 17:43:07.299 Thread-3 DEBUG WholeStageCodegenExec: /* 001 */ public Object generate(Object[] references) { /* 002 */ return new GeneratedIteratorForCodegenStage1(references); /* 003 */ } /* 004 */ /* 005 */ // codegenStageId=1 /* 006 */ final class GeneratedIteratorForCodegenStage1 extends org.apache.spark.sql.execution.BufferedRowIterator { /* 007 */ private Object[] references; /* 008 */ private scala.collection.Iterator[] inputs; /* 009 */ private scala.collection.Iterator rdd_input_0; /* 010 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[] rdd_mutableStateArray_0 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[4]; /* 011 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[] rdd_mutableStateArray_1 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[5]; /* 012 */ /* 013 */ public GeneratedIteratorForCodegenStage1(Object[] references) { /* 014 */ this.references = references; /* 015 */ } /* 016 */ /* 017 */ public void init(int index, scala.collection.Iterator[] inputs) { /* 018 */ partitionIndex = index; /* 019 */ this.inputs = inputs; /* 020 */ wholestagecodegen_init_0_0(); /* 021 */ wholestagecodegen_init_0_1(); /* 022 */ /* 023 */ } /* 024 */ /* 025 */ private void project_writeFields_0_1(InternalRow project_tmpInput_0) { /* 026 */ if ((project_tmpInput_0.isNullAt(3))) { /* 027 */ rdd_mutableStateArray_0[3].setNullAt(3); /* 028 */ } else { /* 029 */ // Remember the current cursor so that we can calculate how many bytes are /* 030 */ // written later. /* 031 */ final int project_previousCursor_2 = rdd_mutableStateArray_0[3].cursor(); /* 032 */ /* 033 */ final ArrayData project_tmpInput_2 = (project_tmpInput_0.getArray(3)); /* 034 */ if (project_tmpInput_2 instanceof UnsafeArrayData) { /* 035 */ rdd_mutableStateArray_0[3].write((UnsafeArrayData) project_tmpInput_2); /* 036 */ } else { /* 037 */ final int project_numElements_1 = project_tmpInput_2.numElements(); /* 038 */ rdd_mutableStateArray_1[4].initialize(project_numElements_1); /* 039 */ /* 040 */ for (int project_index_1 = 0; project_index_1 < project_numElements_1; project_index_1++) { /* 041 */ rdd_mutableStateArray_1[4].write(project_index_1, project_tmpInput_2.getDouble(project_index_1)); /* 042 */ } /* 043 */ } /* 044 */ /* 045 */ rdd_mutableStateArray_0[3].setOffsetAndSizeFromPreviousCursor(3, project_previousCursor_2); /* 046 */ } /* 047 */ /* 048 */ } /* 049 */ /* 050 */ private void wholestagecodegen_init_0_1() { /* 051 */ rdd_mutableStateArray_1[4] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[3], 8); /* 052 */ /* 053 */ } /* 054 */ /* 055 */ private void project_writeFields_0_0(InternalRow project_tmpInput_0) { /* 056 */ rdd_mutableStateArray_0[3].write(0, (project_tmpInput_0.getByte(0))); /* 057 */ /* 058 */ if ((project_tmpInput_0.isNullAt(1))) { /* 059 */ rdd_mutableStateArray_0[3].setNullAt(1); /* 060 */ } else { /* 061 */ rdd_mutableStateArray_0[3].write(1, (project_tmpInput_0.getInt(1))); /* 062 */ } /* 063 */ /* 064 */ if ((project_tmpInput_0.isNullAt(2))) { /* 065 */ rdd_mutableStateArray_0[3].setNullAt(2); /* 066 */ } else { /* 067 */ // Remember the current cursor so that we can calculate how many bytes are /* 068 */ // written later. /* 069 */ final int project_previousCursor_1 = rdd_mutableStateArray_0[3].cursor(); /* 070 */ /* 071 */ final ArrayData project_tmpInput_1 = (project_tmpInput_0.getArray(2)); /* 072 */ if (project_tmpInput_1 instanceof UnsafeArrayData) { /* 073 */ rdd_mutableStateArray_0[3].write((UnsafeArrayData) project_tmpInput_1); /* 074 */ } else { /* 075 */ final int project_numElements_0 = project_tmpInput_1.numElements(); /* 076 */ rdd_mutableStateArray_1[3].initialize(project_numElements_0); /* 077 */ /* 078 */ for (int project_index_0 = 0; project_index_0 < project_numElements_0; project_index_0++) { /* 079 */ rdd_mutableStateArray_1[3].write(project_index_0, project_tmpInput_1.getInt(project_index_0)); /* 080 */ } /* 081 */ } /* 082 */ /* 083 */ rdd_mutableStateArray_0[3].setOffsetAndSizeFromPreviousCursor(2, project_previousCursor_1); /* 084 */ } /* 085 */ /* 086 */ } /* 087 */ /* 088 */ protected void processNext() throws java.io.IOException { /* 089 */ while ( rdd_input_0.hasNext()) { /* 090 */ InternalRow rdd_row_0 = (InternalRow) rdd_input_0.next(); /* 091 */ ((org.apache.spark.sql.execution.metric.SQLMetric) references[0] /* numOutputRows */).add(1); /* 092 */ // common sub-expressions /* 093 */ /* 094 */ boolean rdd_isNull_1 = rdd_row_0.isNullAt(1); /* 095 */ ArrayData rdd_value_1 = rdd_isNull_1 ? /* 096 */ null : (rdd_row_0.getArray(1)); /* 097 */ /* 098 */ Object project_arg_0 = null; /* 099 */ if (rdd_isNull_1) { /* 100 */ project_arg_0 = ((scala.Function1[]) references[1] /* converters */)[0].apply(null); /* 101 */ } else { /* 102 */ project_arg_0 = ((scala.Function1[]) references[1] /* converters */)[0].apply(rdd_value_1); /* 103 */ } /* 104 */ /* 105 */ InternalRow project_result_0 = null; /* 106 */ try { /* 107 */ project_result_0 = (InternalRow)((scala.Function1[]) references[1] /* converters */)[1].apply(((scala.Function1) references[3] /* udf */).apply(project_arg_0)); /* 108 */ } catch (Exception e) { /* 109 */ throw new org.apache.spark.SparkException(((java.lang.String) references[2] /* errMsg */), e); /* 110 */ } /* 111 */ /* 112 */ boolean project_isNull_0 = project_result_0 == null; /* 113 */ InternalRow project_value_0 = null; /* 114 */ if (!project_isNull_0) { /* 115 */ project_value_0 = project_result_0; /* 116 */ } /* 117 */ rdd_mutableStateArray_0[2].reset(); /* 118 */ /* 119 */ rdd_mutableStateArray_0[2].zeroOutNullBytes(); /* 120 */ /* 121 */ if (project_isNull_0) { /* 122 */ rdd_mutableStateArray_0[2].setNullAt(0); /* 123 */ } else { /* 124 */ final InternalRow project_tmpInput_0 = project_value_0; /* 125 */ if (project_tmpInput_0 instanceof UnsafeRow) { /* 126 */ rdd_mutableStateArray_0[2].write(0, (UnsafeRow) project_tmpInput_0); /* 127 */ } else { /* 128 */ // Remember the current cursor so that we can calculate how many bytes are /* 129 */ // written later. /* 130 */ final int project_previousCursor_0 = rdd_mutableStateArray_0[2].cursor(); /* 131 */ /* 132 */ rdd_mutableStateArray_0[3].resetRowWriter(); /* 133 */ project_writeFields_0_0(project_tmpInput_0); /* 134 */ project_writeFields_0_1(project_tmpInput_0); /* 135 */ /* 136 */ rdd_mutableStateArray_0[2].setOffsetAndSizeFromPreviousCursor(0, project_previousCursor_0); /* 137 */ } /* 138 */ } /* 139 */ append((rdd_mutableStateArray_0[2].getRow())); /* 140 */ if (shouldStop()) return; /* 141 */ } /* 142 */ } /* 143 */ /* 144 */ private void wholestagecodegen_init_0_0() { /* 145 */ rdd_input_0 = inputs[0]; /* 146 */ rdd_mutableStateArray_0[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(3, 64); /* 147 */ rdd_mutableStateArray_1[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[0], 8); /* 148 */ rdd_mutableStateArray_0[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(rdd_mutableStateArray_0[0], 4); /* 149 */ rdd_mutableStateArray_1[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[1], 4); /* 150 */ rdd_mutableStateArray_1[2] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[1], 8); /* 151 */ rdd_mutableStateArray_0[2] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(1, 32); /* 152 */ rdd_mutableStateArray_0[3] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(rdd_mutableStateArray_0[2], 4); /* 153 */ rdd_mutableStateArray_1[3] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[3], 4); /* 154 */ /* 155 */ } /* 156 */ /* 157 */ } 21/01/17 17:43:07.301 Executor task launch worker for task 2.0 in stage 2.0 (TID 7) DEBUG LocalDiskShuffleMapOutputWriter: Writing shuffle index file for mapId 7 with length 10 21/01/17 17:43:07.302 Executor task launch worker for task 2.0 in stage 2.0 (TID 7) DEBUG IndexShuffleBlockResolver: Shuffle index for mapId 7: [248,251,0,0,0,0,0,0,0,0] 21/01/17 17:43:07.304 Executor task launch worker for task 2.0 in stage 2.0 (TID 7) INFO Executor: Finished task 2.0 in stage 2.0 (TID 7). 2337 bytes result sent to driver 21/01/17 17:43:07.304 Executor task launch worker for task 2.0 in stage 2.0 (TID 7) DEBUG ExecutorMetricsPoller: stageTCMP: (2, 0) -> 3 21/01/17 17:43:07.305 dispatcher-event-loop-3 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_2.0, runningTasks: 3 21/01/17 17:43:07.305 Thread-3 DEBUG CodeGenerator: /* 001 */ public Object generate(Object[] references) { /* 002 */ return new GeneratedIteratorForCodegenStage1(references); /* 003 */ } /* 004 */ /* 005 */ // codegenStageId=1 /* 006 */ final class GeneratedIteratorForCodegenStage1 extends org.apache.spark.sql.execution.BufferedRowIterator { /* 007 */ private Object[] references; /* 008 */ private scala.collection.Iterator[] inputs; /* 009 */ private scala.collection.Iterator rdd_input_0; /* 010 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[] rdd_mutableStateArray_0 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[4]; /* 011 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[] rdd_mutableStateArray_1 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[5]; /* 012 */ /* 013 */ public GeneratedIteratorForCodegenStage1(Object[] references) { /* 014 */ this.references = references; /* 015 */ } /* 016 */ /* 017 */ public void init(int index, scala.collection.Iterator[] inputs) { /* 018 */ partitionIndex = index; /* 019 */ this.inputs = inputs; /* 020 */ wholestagecodegen_init_0_0(); /* 021 */ wholestagecodegen_init_0_1(); /* 022 */ /* 023 */ } /* 024 */ /* 025 */ private void project_writeFields_0_1(InternalRow project_tmpInput_0) { /* 026 */ if ((project_tmpInput_0.isNullAt(3))) { /* 027 */ rdd_mutableStateArray_0[3].setNullAt(3); /* 028 */ } else { /* 029 */ // Remember the current cursor so that we can calculate how many bytes are /* 030 */ // written later. /* 031 */ final int project_previousCursor_2 = rdd_mutableStateArray_0[3].cursor(); /* 032 */ /* 033 */ final ArrayData project_tmpInput_2 = (project_tmpInput_0.getArray(3)); /* 034 */ if (project_tmpInput_2 instanceof UnsafeArrayData) { /* 035 */ rdd_mutableStateArray_0[3].write((UnsafeArrayData) project_tmpInput_2); /* 036 */ } else { /* 037 */ final int project_numElements_1 = project_tmpInput_2.numElements(); /* 038 */ rdd_mutableStateArray_1[4].initialize(project_numElements_1); /* 039 */ /* 040 */ for (int project_index_1 = 0; project_index_1 < project_numElements_1; project_index_1++) { /* 041 */ rdd_mutableStateArray_1[4].write(project_index_1, project_tmpInput_2.getDouble(project_index_1)); /* 042 */ } /* 043 */ } /* 044 */ /* 045 */ rdd_mutableStateArray_0[3].setOffsetAndSizeFromPreviousCursor(3, project_previousCursor_2); /* 046 */ } /* 047 */ /* 048 */ } /* 049 */ /* 050 */ private void wholestagecodegen_init_0_1() { /* 051 */ rdd_mutableStateArray_1[4] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[3], 8); /* 052 */ /* 053 */ } /* 054 */ /* 055 */ private void project_writeFields_0_0(InternalRow project_tmpInput_0) { /* 056 */ rdd_mutableStateArray_0[3].write(0, (project_tmpInput_0.getByte(0))); /* 057 */ /* 058 */ if ((project_tmpInput_0.isNullAt(1))) { /* 059 */ rdd_mutableStateArray_0[3].setNullAt(1); /* 060 */ } else { /* 061 */ rdd_mutableStateArray_0[3].write(1, (project_tmpInput_0.getInt(1))); /* 062 */ } /* 063 */ /* 064 */ if ((project_tmpInput_0.isNullAt(2))) { /* 065 */ rdd_mutableStateArray_0[3].setNullAt(2); /* 066 */ } else { /* 067 */ // Remember the current cursor so that we can calculate how many bytes are /* 068 */ // written later. /* 069 */ final int project_previousCursor_1 = rdd_mutableStateArray_0[3].cursor(); /* 070 */ /* 071 */ final ArrayData project_tmpInput_1 = (project_tmpInput_0.getArray(2)); /* 072 */ if (project_tmpInput_1 instanceof UnsafeArrayData) { /* 073 */ rdd_mutableStateArray_0[3].write((UnsafeArrayData) project_tmpInput_1); /* 074 */ } else { /* 075 */ final int project_numElements_0 = project_tmpInput_1.numElements(); /* 076 */ rdd_mutableStateArray_1[3].initialize(project_numElements_0); /* 077 */ /* 078 */ for (int project_index_0 = 0; project_index_0 < project_numElements_0; project_index_0++) { /* 079 */ rdd_mutableStateArray_1[3].write(project_index_0, project_tmpInput_1.getInt(project_index_0)); /* 080 */ } /* 081 */ } /* 082 */ /* 083 */ rdd_mutableStateArray_0[3].setOffsetAndSizeFromPreviousCursor(2, project_previousCursor_1); /* 084 */ } /* 085 */ /* 086 */ } /* 087 */ /* 088 */ protected void processNext() throws java.io.IOException { /* 089 */ while ( rdd_input_0.hasNext()) { /* 090 */ InternalRow rdd_row_0 = (InternalRow) rdd_input_0.next(); /* 091 */ ((org.apache.spark.sql.execution.metric.SQLMetric) references[0] /* numOutputRows */).add(1); /* 092 */ // common sub-expressions /* 093 */ /* 094 */ boolean rdd_isNull_1 = rdd_row_0.isNullAt(1); /* 095 */ ArrayData rdd_value_1 = rdd_isNull_1 ? /* 096 */ null : (rdd_row_0.getArray(1)); /* 097 */ /* 098 */ Object project_arg_0 = null; /* 099 */ if (rdd_isNull_1) { /* 100 */ project_arg_0 = ((scala.Function1[]) references[1] /* converters */)[0].apply(null); /* 101 */ } else { /* 102 */ project_arg_0 = ((scala.Function1[]) references[1] /* converters */)[0].apply(rdd_value_1); /* 103 */ } /* 104 */ /* 105 */ InternalRow project_result_0 = null; /* 106 */ try { /* 107 */ project_result_0 = (InternalRow)((scala.Function1[]) references[1] /* converters */)[1].apply(((scala.Function1) references[3] /* udf */).apply(project_arg_0)); /* 108 */ } catch (Exception e) { /* 109 */ throw new org.apache.spark.SparkException(((java.lang.String) references[2] /* errMsg */), e); /* 110 */ } /* 111 */ /* 112 */ boolean project_isNull_0 = project_result_0 == null; /* 113 */ InternalRow project_value_0 = null; /* 114 */ if (!project_isNull_0) { /* 115 */ project_value_0 = project_result_0; /* 116 */ } /* 117 */ rdd_mutableStateArray_0[2].reset(); /* 118 */ /* 119 */ rdd_mutableStateArray_0[2].zeroOutNullBytes(); /* 120 */ /* 121 */ if (project_isNull_0) { /* 122 */ rdd_mutableStateArray_0[2].setNullAt(0); /* 123 */ } else { /* 124 */ final InternalRow project_tmpInput_0 = project_value_0; /* 125 */ if (project_tmpInput_0 instanceof UnsafeRow) { /* 126 */ rdd_mutableStateArray_0[2].write(0, (UnsafeRow) project_tmpInput_0); /* 127 */ } else { /* 128 */ // Remember the current cursor so that we can calculate how many bytes are /* 129 */ // written later. /* 130 */ final int project_previousCursor_0 = rdd_mutableStateArray_0[2].cursor(); /* 131 */ /* 132 */ rdd_mutableStateArray_0[3].resetRowWriter(); /* 133 */ project_writeFields_0_0(project_tmpInput_0); /* 134 */ project_writeFields_0_1(project_tmpInput_0); /* 135 */ /* 136 */ rdd_mutableStateArray_0[2].setOffsetAndSizeFromPreviousCursor(0, project_previousCursor_0); /* 137 */ } /* 138 */ } /* 139 */ append((rdd_mutableStateArray_0[2].getRow())); /* 140 */ if (shouldStop()) return; /* 141 */ } /* 142 */ } /* 143 */ /* 144 */ private void wholestagecodegen_init_0_0() { /* 145 */ rdd_input_0 = inputs[0]; /* 146 */ rdd_mutableStateArray_0[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(3, 64); /* 147 */ rdd_mutableStateArray_1[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[0], 8); /* 148 */ rdd_mutableStateArray_0[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(rdd_mutableStateArray_0[0], 4); /* 149 */ rdd_mutableStateArray_1[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[1], 4); /* 150 */ rdd_mutableStateArray_1[2] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[1], 8); /* 151 */ rdd_mutableStateArray_0[2] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(1, 32); /* 152 */ rdd_mutableStateArray_0[3] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(rdd_mutableStateArray_0[2], 4); /* 153 */ rdd_mutableStateArray_1[3] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[3], 4); /* 154 */ /* 155 */ } /* 156 */ /* 157 */ } 21/01/17 17:43:07.305 dispatcher-event-loop-3 DEBUG TaskSetManager: No tasks for locality level NODE_LOCAL, so moving to locality level ANY 21/01/17 17:43:07.306 task-result-getter-1 INFO TaskSetManager: Finished task 2.0 in stage 2.0 (TID 7) in 163 ms on 192.168.1.225 (executor driver) (1/4) 21/01/17 17:43:07.307 dag-scheduler-event-loop DEBUG DAGScheduler: ShuffleMapTask finished on driver 21/01/17 17:43:07.311 Executor task launch worker for task 3.0 in stage 2.0 (TID 8) DEBUG ALS: Converting to local indices took 0.001353479 seconds. 21/01/17 17:43:07.312 Executor task launch worker for task 0.0 in stage 2.0 (TID 5) INFO Executor: Finished task 0.0 in stage 2.0 (TID 5). 2337 bytes result sent to driver 21/01/17 17:43:07.312 Executor task launch worker for task 0.0 in stage 2.0 (TID 5) DEBUG ExecutorMetricsPoller: stageTCMP: (2, 0) -> 2 21/01/17 17:43:07.313 task-result-getter-2 INFO TaskSetManager: Finished task 0.0 in stage 2.0 (TID 5) in 173 ms on 192.168.1.225 (executor driver) (2/4) 21/01/17 17:43:07.314 Executor task launch worker for task 1.0 in stage 2.0 (TID 6) INFO Executor: Finished task 1.0 in stage 2.0 (TID 6). 2337 bytes result sent to driver 21/01/17 17:43:07.314 Executor task launch worker for task 1.0 in stage 2.0 (TID 6) DEBUG ExecutorMetricsPoller: stageTCMP: (2, 0) -> 1 21/01/17 17:43:07.315 task-result-getter-3 INFO TaskSetManager: Finished task 1.0 in stage 2.0 (TID 6) in 172 ms on 192.168.1.225 (executor driver) (3/4) 21/01/17 17:43:07.315 dag-scheduler-event-loop DEBUG DAGScheduler: ShuffleMapTask finished on driver 21/01/17 17:43:07.316 dag-scheduler-event-loop DEBUG DAGScheduler: ShuffleMapTask finished on driver 21/01/17 17:43:07.321 Executor task launch worker for task 3.0 in stage 2.0 (TID 8) DEBUG LocalDiskShuffleMapOutputWriter: Writing shuffle index file for mapId 8 with length 10 21/01/17 17:43:07.322 Executor task launch worker for task 3.0 in stage 2.0 (TID 8) DEBUG IndexShuffleBlockResolver: Shuffle index for mapId 8: [0,247,245,0,0,0,0,0,0,0] 21/01/17 17:43:07.323 Executor task launch worker for task 3.0 in stage 2.0 (TID 8) INFO Executor: Finished task 3.0 in stage 2.0 (TID 8). 2337 bytes result sent to driver 21/01/17 17:43:07.323 Executor task launch worker for task 3.0 in stage 2.0 (TID 8) DEBUG ExecutorMetricsPoller: removing (2, 0) from stageTCMP 21/01/17 17:43:07.324 task-result-getter-0 INFO TaskSetManager: Finished task 3.0 in stage 2.0 (TID 8) in 180 ms on 192.168.1.225 (executor driver) (4/4) 21/01/17 17:43:07.324 task-result-getter-0 INFO TaskSchedulerImpl: Removed TaskSet 2.0, whose tasks have all completed, from pool 21/01/17 17:43:07.325 dag-scheduler-event-loop DEBUG DAGScheduler: ShuffleMapTask finished on driver 21/01/17 17:43:07.326 dag-scheduler-event-loop INFO DAGScheduler: ShuffleMapStage 2 (map at ALS.scala:1612) finished in 0.198 s 21/01/17 17:43:07.326 dag-scheduler-event-loop INFO DAGScheduler: looking for newly runnable stages 21/01/17 17:43:07.326 dag-scheduler-event-loop INFO DAGScheduler: running: Set() 21/01/17 17:43:07.326 dag-scheduler-event-loop INFO DAGScheduler: waiting: Set(ResultStage 3) 21/01/17 17:43:07.326 dag-scheduler-event-loop INFO DAGScheduler: failed: Set() 21/01/17 17:43:07.326 dag-scheduler-event-loop DEBUG MapOutputTrackerMaster: Increasing epoch to 2 21/01/17 17:43:07.326 dag-scheduler-event-loop DEBUG DAGScheduler: submitStage(ResultStage 3 (name=count at ALS.scala:973;jobs=1)) 21/01/17 17:43:07.327 dag-scheduler-event-loop DEBUG DAGScheduler: missing: List() 21/01/17 17:43:07.328 dag-scheduler-event-loop INFO DAGScheduler: Submitting ResultStage 3 (userOutBlocks MapPartitionsRDD[20] at mapValues at ALS.scala:1649), which has no missing parents 21/01/17 17:43:07.328 dag-scheduler-event-loop DEBUG DAGScheduler: submitMissingTasks(ResultStage 3) 21/01/17 17:43:07.334 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_3 stored as values in memory (estimated size 34.5 KiB, free 366.1 MiB) 21/01/17 17:43:07.334 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_3 locally took 1 ms 21/01/17 17:43:07.334 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_3 without replication took 1 ms 21/01/17 17:43:07.338 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_3_piece0 stored as bytes in memory (estimated size 16.2 KiB, free 366.1 MiB) 21/01/17 17:43:07.338 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master broadcast_3_piece0 for BlockManagerId(driver, 192.168.1.225, 40985, None) 21/01/17 17:43:07.339 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added broadcast_3_piece0 in memory on 192.168.1.225:40985 (size: 16.2 KiB, free: 366.2 MiB) 21/01/17 17:43:07.339 dag-scheduler-event-loop DEBUG BlockManagerMaster: Updated info of block broadcast_3_piece0 21/01/17 17:43:07.339 dag-scheduler-event-loop DEBUG BlockManager: Told master about block broadcast_3_piece0 21/01/17 17:43:07.339 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_3_piece0 locally took 4 ms 21/01/17 17:43:07.339 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_3_piece0 without replication took 4 ms 21/01/17 17:43:07.340 dag-scheduler-event-loop INFO SparkContext: Created broadcast 3 from broadcast at DAGScheduler.scala:1383 21/01/17 17:43:07.340 dag-scheduler-event-loop INFO DAGScheduler: Submitting 10 missing tasks from ResultStage 3 (userOutBlocks MapPartitionsRDD[20] at mapValues at ALS.scala:1649) (first 15 tasks are for partitions Vector(0, 1, 2, 3, 4, 5, 6, 7, 8, 9)) 21/01/17 17:43:07.340 dag-scheduler-event-loop INFO TaskSchedulerImpl: Adding task set 3.0 with 10 tasks resource profile 0 21/01/17 17:43:07.343 dag-scheduler-event-loop DEBUG TaskSetManager: Epoch for TaskSet 3.0: 2 21/01/17 17:43:07.343 dag-scheduler-event-loop DEBUG TaskSetManager: Adding pending tasks took 0 ms 21/01/17 17:43:07.343 dag-scheduler-event-loop DEBUG TaskSetManager: Valid locality levels for TaskSet 3.0: NODE_LOCAL, NO_PREF, ANY 21/01/17 17:43:07.344 dispatcher-event-loop-3 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_3.0, runningTasks: 0 21/01/17 17:43:07.345 dispatcher-event-loop-3 INFO TaskSetManager: Starting task 0.0 in stage 3.0 (TID 9) (192.168.1.225, executor driver, partition 0, NODE_LOCAL, 7143 bytes) taskResourceAssignments Map() 21/01/17 17:43:07.345 dispatcher-event-loop-3 INFO TaskSetManager: Starting task 1.0 in stage 3.0 (TID 10) (192.168.1.225, executor driver, partition 1, NODE_LOCAL, 7143 bytes) taskResourceAssignments Map() 21/01/17 17:43:07.345 dispatcher-event-loop-3 INFO TaskSetManager: Starting task 2.0 in stage 3.0 (TID 11) (192.168.1.225, executor driver, partition 2, NODE_LOCAL, 7143 bytes) taskResourceAssignments Map() 21/01/17 17:43:07.346 dispatcher-event-loop-3 DEBUG TaskSetManager: No tasks for locality level NODE_LOCAL, so moving to locality level NO_PREF 21/01/17 17:43:07.346 dispatcher-event-loop-3 DEBUG TaskSetManager: Moving to ANY after waiting for 0ms 21/01/17 17:43:07.347 dispatcher-event-loop-3 INFO TaskSetManager: Starting task 3.0 in stage 3.0 (TID 12) (192.168.1.225, executor driver, partition 3, PROCESS_LOCAL, 7143 bytes) taskResourceAssignments Map() 21/01/17 17:43:07.347 Executor task launch worker for task 1.0 in stage 3.0 (TID 10) INFO Executor: Running task 1.0 in stage 3.0 (TID 10) 21/01/17 17:43:07.347 Executor task launch worker for task 3.0 in stage 3.0 (TID 12) INFO Executor: Running task 3.0 in stage 3.0 (TID 12) 21/01/17 17:43:07.347 Executor task launch worker for task 0.0 in stage 3.0 (TID 9) INFO Executor: Running task 0.0 in stage 3.0 (TID 9) 21/01/17 17:43:07.347 Executor task launch worker for task 2.0 in stage 3.0 (TID 11) INFO Executor: Running task 2.0 in stage 3.0 (TID 11) 21/01/17 17:43:07.349 Executor task launch worker for task 2.0 in stage 3.0 (TID 11) DEBUG ExecutorMetricsPoller: stageTCMP: (3, 0) -> 1 21/01/17 17:43:07.349 Executor task launch worker for task 1.0 in stage 3.0 (TID 10) DEBUG ExecutorMetricsPoller: stageTCMP: (3, 0) -> 2 21/01/17 17:43:07.349 Executor task launch worker for task 2.0 in stage 3.0 (TID 11) DEBUG BlockManager: Getting local block broadcast_3 21/01/17 17:43:07.350 Executor task launch worker for task 2.0 in stage 3.0 (TID 11) DEBUG BlockManager: Level for block broadcast_3 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:07.351 Executor task launch worker for task 0.0 in stage 3.0 (TID 9) DEBUG ExecutorMetricsPoller: stageTCMP: (3, 0) -> 3 21/01/17 17:43:07.351 Executor task launch worker for task 3.0 in stage 3.0 (TID 12) DEBUG ExecutorMetricsPoller: stageTCMP: (3, 0) -> 4 21/01/17 17:43:07.358 Executor task launch worker for task 1.0 in stage 3.0 (TID 10) DEBUG BlockManager: Getting local block rdd_20_1 21/01/17 17:43:07.359 Executor task launch worker for task 1.0 in stage 3.0 (TID 10) DEBUG BlockManager: Block rdd_20_1 was not found 21/01/17 17:43:07.359 Executor task launch worker for task 1.0 in stage 3.0 (TID 10) DEBUG BlockManager: Getting remote block rdd_20_1 21/01/17 17:43:07.359 Executor task launch worker for task 1.0 in stage 3.0 (TID 10) DEBUG BlockManager: Block rdd_20_1 is unknown by block manager master 21/01/17 17:43:07.359 Executor task launch worker for task 1.0 in stage 3.0 (TID 10) DEBUG BlockManager: Getting local block rdd_19_1 21/01/17 17:43:07.359 Executor task launch worker for task 1.0 in stage 3.0 (TID 10) DEBUG BlockManager: Block rdd_19_1 was not found 21/01/17 17:43:07.359 Executor task launch worker for task 1.0 in stage 3.0 (TID 10) DEBUG BlockManager: Getting remote block rdd_19_1 21/01/17 17:43:07.359 Executor task launch worker for task 1.0 in stage 3.0 (TID 10) DEBUG BlockManager: Block rdd_19_1 is unknown by block manager master 21/01/17 17:43:07.360 Executor task launch worker for task 1.0 in stage 3.0 (TID 10) DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 0 21/01/17 17:43:07.360 Executor task launch worker for task 1.0 in stage 3.0 (TID 10) DEBUG MapOutputTrackerMaster: Convert map statuses for shuffle 0, mappers 0-4, partitions 1-2 21/01/17 17:43:07.360 Executor task launch worker for task 1.0 in stage 3.0 (TID 10) DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 21/01/17 17:43:07.361 Executor task launch worker for task 1.0 in stage 3.0 (TID 10) INFO ShuffleBlockFetcherIterator: Getting 2 (502.0 B) non-empty blocks including 2 (502.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 21/01/17 17:43:07.361 Executor task launch worker for task 1.0 in stage 3.0 (TID 10) INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 21/01/17 17:43:07.361 Executor task launch worker for task 1.0 in stage 3.0 (TID 10) DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_0_7_1,2), (shuffle_0_8_1,3) 21/01/17 17:43:07.361 Executor task launch worker for task 1.0 in stage 3.0 (TID 10) DEBUG BlockManager: Getting local shuffle block shuffle_0_7_1 21/01/17 17:43:07.361 Executor task launch worker for task 1.0 in stage 3.0 (TID 10) DEBUG BlockManager: Getting local shuffle block shuffle_0_8_1 21/01/17 17:43:07.361 Executor task launch worker for task 1.0 in stage 3.0 (TID 10) DEBUG ShuffleBlockFetcherIterator: Got local blocks in 1 ms 21/01/17 17:43:07.376 Thread-3 INFO CodeGenerator: Code generated in 76.029224 ms 21/01/17 17:43:07.377 Thread-3 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$doExecute$4$adapted 21/01/17 17:43:07.378 Thread-3 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$doExecute$4$adapted) is now cleaned +++ 21/01/17 17:43:07.382 Executor task launch worker for task 1.0 in stage 3.0 (TID 10) DEBUG ALS: Start sorting an uncompressed in-block of size 2. (sortId = 29840503) 21/01/17 17:43:07.382 Thread-3 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$collect$2 21/01/17 17:43:07.388 Thread-3 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$collect$2) is now cleaned +++ 21/01/17 17:43:07.391 Thread-3 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$runJob$5 21/01/17 17:43:07.394 Thread-3 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$runJob$5) is now cleaned +++ 21/01/17 17:43:07.396 Thread-3 INFO SparkContext: Starting job: at /usr/lib/python3.6/runpy.py:85 21/01/17 17:43:07.400 main DEBUG PythonGatewayServer: Exiting due to broken pipe from Python driver 21/01/17 17:43:07.401 dag-scheduler-event-loop DEBUG DAGScheduler: Merging stage rdd profiles: Set() 21/01/17 17:43:07.401 Executor task launch worker for task 3.0 in stage 3.0 (TID 12) DEBUG BlockManager: Getting local block rdd_20_3 21/01/17 17:43:07.401 Executor task launch worker for task 3.0 in stage 3.0 (TID 12) DEBUG BlockManager: Block rdd_20_3 was not found 21/01/17 17:43:07.401 Executor task launch worker for task 3.0 in stage 3.0 (TID 12) DEBUG BlockManager: Getting remote block rdd_20_3 21/01/17 17:43:07.402 dag-scheduler-event-loop INFO DAGScheduler: Got job 1 ( at /usr/lib/python3.6/runpy.py:85) with 4 output partitions 21/01/17 17:43:07.402 dag-scheduler-event-loop INFO DAGScheduler: Final stage: ResultStage 1 ( at /usr/lib/python3.6/runpy.py:85) 21/01/17 17:43:07.402 dag-scheduler-event-loop INFO DAGScheduler: Parents of final stage: List() 21/01/17 17:43:07.402 dag-scheduler-event-loop INFO DAGScheduler: Missing parents: List() 21/01/17 17:43:07.403 dag-scheduler-event-loop DEBUG DAGScheduler: submitStage(ResultStage 1 (name= at /usr/lib/python3.6/runpy.py:85;jobs=1)) 21/01/17 17:43:07.403 dag-scheduler-event-loop DEBUG DAGScheduler: missing: List() 21/01/17 17:43:07.403 dag-scheduler-event-loop INFO DAGScheduler: Submitting ResultStage 1 (MapPartitionsRDD[8] at at /usr/lib/python3.6/runpy.py:85), which has no missing parents 21/01/17 17:43:07.403 dag-scheduler-event-loop DEBUG DAGScheduler: submitMissingTasks(ResultStage 1) 21/01/17 17:43:07.405 Executor task launch worker for task 2.0 in stage 3.0 (TID 11) DEBUG BlockManager: Getting local block rdd_20_2 21/01/17 17:43:07.406 Executor task launch worker for task 0.0 in stage 3.0 (TID 9) DEBUG BlockManager: Getting local block rdd_20_0 21/01/17 17:43:07.406 Executor task launch worker for task 2.0 in stage 3.0 (TID 11) DEBUG BlockManager: Block rdd_20_2 was not found 21/01/17 17:43:07.406 Executor task launch worker for task 0.0 in stage 3.0 (TID 9) DEBUG BlockManager: Block rdd_20_0 was not found 21/01/17 17:43:07.406 Executor task launch worker for task 0.0 in stage 3.0 (TID 9) DEBUG BlockManager: Getting remote block rdd_20_0 21/01/17 17:43:07.406 Executor task launch worker for task 0.0 in stage 3.0 (TID 9) DEBUG BlockManager: Block rdd_20_0 is unknown by block manager master 21/01/17 17:43:07.407 Executor task launch worker for task 0.0 in stage 3.0 (TID 9) DEBUG BlockManager: Getting local block rdd_19_0 21/01/17 17:43:07.407 Executor task launch worker for task 0.0 in stage 3.0 (TID 9) DEBUG BlockManager: Block rdd_19_0 was not found 21/01/17 17:43:07.407 Executor task launch worker for task 0.0 in stage 3.0 (TID 9) DEBUG BlockManager: Getting remote block rdd_19_0 21/01/17 17:43:07.406 Executor task launch worker for task 2.0 in stage 3.0 (TID 11) DEBUG BlockManager: Getting remote block rdd_20_2 21/01/17 17:43:07.406 Executor task launch worker for task 3.0 in stage 3.0 (TID 12) DEBUG BlockManager: Block rdd_20_3 is unknown by block manager master 21/01/17 17:43:07.407 Executor task launch worker for task 0.0 in stage 3.0 (TID 9) DEBUG BlockManager: Block rdd_19_0 is unknown by block manager master 21/01/17 17:43:07.407 Executor task launch worker for task 3.0 in stage 3.0 (TID 12) DEBUG BlockManager: Getting local block rdd_19_3 21/01/17 17:43:07.407 Executor task launch worker for task 0.0 in stage 3.0 (TID 9) DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 0 21/01/17 17:43:07.407 Executor task launch worker for task 3.0 in stage 3.0 (TID 12) DEBUG BlockManager: Block rdd_19_3 was not found 21/01/17 17:43:07.407 Executor task launch worker for task 2.0 in stage 3.0 (TID 11) DEBUG BlockManager: Block rdd_20_2 is unknown by block manager master 21/01/17 17:43:07.407 Executor task launch worker for task 2.0 in stage 3.0 (TID 11) DEBUG BlockManager: Getting local block rdd_19_2 21/01/17 17:43:07.407 Executor task launch worker for task 2.0 in stage 3.0 (TID 11) DEBUG BlockManager: Block rdd_19_2 was not found 21/01/17 17:43:07.408 Executor task launch worker for task 2.0 in stage 3.0 (TID 11) DEBUG BlockManager: Getting remote block rdd_19_2 21/01/17 17:43:07.407 Executor task launch worker for task 0.0 in stage 3.0 (TID 9) DEBUG MapOutputTrackerMaster: Convert map statuses for shuffle 0, mappers 0-4, partitions 0-1 21/01/17 17:43:07.407 Executor task launch worker for task 3.0 in stage 3.0 (TID 12) DEBUG BlockManager: Getting remote block rdd_19_3 21/01/17 17:43:07.408 Executor task launch worker for task 3.0 in stage 3.0 (TID 12) DEBUG BlockManager: Block rdd_19_3 is unknown by block manager master 21/01/17 17:43:07.408 Executor task launch worker for task 3.0 in stage 3.0 (TID 12) DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 0 21/01/17 17:43:07.408 Executor task launch worker for task 3.0 in stage 3.0 (TID 12) DEBUG MapOutputTrackerMaster: Convert map statuses for shuffle 0, mappers 0-4, partitions 3-4 21/01/17 17:43:07.408 Executor task launch worker for task 2.0 in stage 3.0 (TID 11) DEBUG BlockManager: Block rdd_19_2 is unknown by block manager master 21/01/17 17:43:07.409 Executor task launch worker for task 1.0 in stage 3.0 (TID 10) DEBUG ALS: Sorting took 0.001881153 seconds. (sortId = 29840503) 21/01/17 17:43:07.409 Executor task launch worker for task 3.0 in stage 3.0 (TID 12) DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 21/01/17 17:43:07.409 Executor task launch worker for task 3.0 in stage 3.0 (TID 12) INFO ShuffleBlockFetcherIterator: Getting 0 (0.0 B) non-empty blocks including 0 (0.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 21/01/17 17:43:07.409 Executor task launch worker for task 2.0 in stage 3.0 (TID 11) DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 0 21/01/17 17:43:07.409 Executor task launch worker for task 3.0 in stage 3.0 (TID 12) INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 21/01/17 17:43:07.410 Executor task launch worker for task 3.0 in stage 3.0 (TID 12) DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: 21/01/17 17:43:07.410 Executor task launch worker for task 3.0 in stage 3.0 (TID 12) DEBUG ShuffleBlockFetcherIterator: Got local blocks in 0 ms 21/01/17 17:43:07.410 Executor task launch worker for task 1.0 in stage 3.0 (TID 10) DEBUG TaskMemoryManager: Task 10 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@59321627 21/01/17 17:43:07.411 Executor task launch worker for task 3.0 in stage 3.0 (TID 12) DEBUG TaskMemoryManager: Task 12 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@539c91e9 21/01/17 17:43:07.411 Executor task launch worker for task 1.0 in stage 3.0 (TID 10) INFO MemoryStore: Block rdd_19_1 stored as values in memory (estimated size 192.0 B, free 366.1 MiB) 21/01/17 17:43:07.411 Executor task launch worker for task 3.0 in stage 3.0 (TID 12) INFO MemoryStore: Block rdd_19_3 stored as values in memory (estimated size 16.0 B, free 366.1 MiB) 21/01/17 17:43:07.409 Executor task launch worker for task 0.0 in stage 3.0 (TID 9) DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 21/01/17 17:43:07.411 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_19_1 for BlockManagerId(driver, 192.168.1.225, 40985, None) 21/01/17 17:43:07.412 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_19_1 in memory on 192.168.1.225:40985 (size: 192.0 B, free: 366.2 MiB) 21/01/17 17:43:07.410 Executor task launch worker for task 2.0 in stage 3.0 (TID 11) DEBUG MapOutputTrackerMaster: Convert map statuses for shuffle 0, mappers 0-4, partitions 2-3 21/01/17 17:43:07.412 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_19_3 for BlockManagerId(driver, 192.168.1.225, 40985, None) 21/01/17 17:43:07.412 Executor task launch worker for task 2.0 in stage 3.0 (TID 11) DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 21/01/17 17:43:07.412 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_19_3 in memory on 192.168.1.225:40985 (size: 16.0 B, free: 366.2 MiB) 21/01/17 17:43:07.413 Executor task launch worker for task 3.0 in stage 3.0 (TID 12) DEBUG BlockManagerMaster: Updated info of block rdd_19_3 21/01/17 17:43:07.413 Executor task launch worker for task 3.0 in stage 3.0 (TID 12) DEBUG BlockManager: Told master about block rdd_19_3 21/01/17 17:43:07.413 Executor task launch worker for task 3.0 in stage 3.0 (TID 12) DEBUG BlockManager: Put block rdd_19_3 locally took 4 ms 21/01/17 17:43:07.413 Executor task launch worker for task 3.0 in stage 3.0 (TID 12) DEBUG BlockManager: Putting block rdd_19_3 without replication took 4 ms 21/01/17 17:43:07.413 Executor task launch worker for task 3.0 in stage 3.0 (TID 12) DEBUG BlockManager: Getting local block rdd_19_3 21/01/17 17:43:07.413 Executor task launch worker for task 3.0 in stage 3.0 (TID 12) DEBUG BlockManager: Level for block rdd_19_3 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:07.414 Executor task launch worker for task 3.0 in stage 3.0 (TID 12) INFO MemoryStore: Block rdd_20_3 stored as values in memory (estimated size 16.0 B, free 366.1 MiB) 21/01/17 17:43:07.414 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_20_3 for BlockManagerId(driver, 192.168.1.225, 40985, None) 21/01/17 17:43:07.415 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_20_3 in memory on 192.168.1.225:40985 (size: 16.0 B, free: 366.2 MiB) 21/01/17 17:43:07.415 Executor task launch worker for task 2.0 in stage 3.0 (TID 11) INFO ShuffleBlockFetcherIterator: Getting 2 (527.0 B) non-empty blocks including 2 (527.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 21/01/17 17:43:07.415 Executor task launch worker for task 2.0 in stage 3.0 (TID 11) INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 2 ms 21/01/17 17:43:07.415 Executor task launch worker for task 2.0 in stage 3.0 (TID 11) DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_0_5_2,0), (shuffle_0_8_2,3) 21/01/17 17:43:07.415 Executor task launch worker for task 2.0 in stage 3.0 (TID 11) DEBUG BlockManager: Getting local shuffle block shuffle_0_5_2 21/01/17 17:43:07.415 Executor task launch worker for task 3.0 in stage 3.0 (TID 12) DEBUG BlockManagerMaster: Updated info of block rdd_20_3 21/01/17 17:43:07.415 Executor task launch worker for task 2.0 in stage 3.0 (TID 11) DEBUG BlockManager: Getting local shuffle block shuffle_0_8_2 21/01/17 17:43:07.415 Executor task launch worker for task 2.0 in stage 3.0 (TID 11) DEBUG ShuffleBlockFetcherIterator: Got local blocks in 3 ms 21/01/17 17:43:07.417 Executor task launch worker for task 2.0 in stage 3.0 (TID 11) DEBUG ALS: Start sorting an uncompressed in-block of size 2. (sortId = -990767592) 21/01/17 17:43:07.415 Executor task launch worker for task 0.0 in stage 3.0 (TID 9) INFO ShuffleBlockFetcherIterator: Getting 2 (502.0 B) non-empty blocks including 2 (502.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 21/01/17 17:43:07.415 Executor task launch worker for task 3.0 in stage 3.0 (TID 12) DEBUG BlockManager: Told master about block rdd_20_3 21/01/17 17:43:07.419 Executor task launch worker for task 1.0 in stage 3.0 (TID 10) DEBUG BlockManagerMaster: Updated info of block rdd_19_1 21/01/17 17:43:07.419 Executor task launch worker for task 1.0 in stage 3.0 (TID 10) DEBUG BlockManager: Told master about block rdd_19_1 21/01/17 17:43:07.419 Executor task launch worker for task 1.0 in stage 3.0 (TID 10) DEBUG BlockManager: Put block rdd_19_1 locally took 59 ms 21/01/17 17:43:07.419 Executor task launch worker for task 1.0 in stage 3.0 (TID 10) DEBUG BlockManager: Putting block rdd_19_1 without replication took 59 ms 21/01/17 17:43:07.419 Executor task launch worker for task 1.0 in stage 3.0 (TID 10) DEBUG BlockManager: Getting local block rdd_19_1 21/01/17 17:43:07.419 Executor task launch worker for task 1.0 in stage 3.0 (TID 10) DEBUG BlockManager: Level for block rdd_19_1 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:07.419 Executor task launch worker for task 0.0 in stage 3.0 (TID 9) INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 10 ms 21/01/17 17:43:07.420 Executor task launch worker for task 0.0 in stage 3.0 (TID 9) DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_0_6_0,1), (shuffle_0_7_0,2) 21/01/17 17:43:07.420 Executor task launch worker for task 0.0 in stage 3.0 (TID 9) DEBUG BlockManager: Getting local shuffle block shuffle_0_6_0 21/01/17 17:43:07.420 Executor task launch worker for task 0.0 in stage 3.0 (TID 9) DEBUG BlockManager: Getting local shuffle block shuffle_0_7_0 21/01/17 17:43:07.420 Executor task launch worker for task 0.0 in stage 3.0 (TID 9) DEBUG ShuffleBlockFetcherIterator: Got local blocks in 10 ms 21/01/17 17:43:07.421 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_3 stored as values in memory (estimated size 28.1 KiB, free 366.2 MiB) 21/01/17 17:43:07.421 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_3 locally took 1 ms 21/01/17 17:43:07.421 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_3 without replication took 1 ms 21/01/17 17:43:07.422 Executor task launch worker for task 1.0 in stage 3.0 (TID 10) INFO MemoryStore: Block rdd_20_1 stored as values in memory (estimated size 296.0 B, free 366.1 MiB) 21/01/17 17:43:07.422 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_20_1 for BlockManagerId(driver, 192.168.1.225, 40985, None) 21/01/17 17:43:07.422 Executor task launch worker for task 2.0 in stage 3.0 (TID 11) DEBUG ALS: Sorting took 3.274E-5 seconds. (sortId = -990767592) 21/01/17 17:43:07.422 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_20_1 in memory on 192.168.1.225:40985 (size: 296.0 B, free: 366.2 MiB) 21/01/17 17:43:07.423 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_3_piece0 stored as bytes in memory (estimated size 12.6 KiB, free 366.2 MiB) 21/01/17 17:43:07.423 Executor task launch worker for task 1.0 in stage 3.0 (TID 10) DEBUG BlockManagerMaster: Updated info of block rdd_20_1 21/01/17 17:43:07.422 Executor task launch worker for task 3.0 in stage 3.0 (TID 12) DEBUG BlockManager: Put block rdd_20_3 locally took 15 ms 21/01/17 17:43:07.423 Executor task launch worker for task 3.0 in stage 3.0 (TID 12) DEBUG BlockManager: Putting block rdd_20_3 without replication took 15 ms 21/01/17 17:43:07.423 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master broadcast_3_piece0 for BlockManagerId(driver, 192.168.1.225, 46295, None) 21/01/17 17:43:07.423 Executor task launch worker for task 3.0 in stage 3.0 (TID 12) DEBUG BlockManager: Getting local block rdd_20_3 21/01/17 17:43:07.423 Executor task launch worker for task 3.0 in stage 3.0 (TID 12) DEBUG BlockManager: Level for block rdd_20_3 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:07.424 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added broadcast_3_piece0 in memory on 192.168.1.225:46295 (size: 12.6 KiB, free: 366.3 MiB) 21/01/17 17:43:07.424 dag-scheduler-event-loop DEBUG BlockManagerMaster: Updated info of block broadcast_3_piece0 21/01/17 17:43:07.424 dag-scheduler-event-loop DEBUG BlockManager: Told master about block broadcast_3_piece0 21/01/17 17:43:07.424 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_3_piece0 locally took 2 ms 21/01/17 17:43:07.424 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_3_piece0 without replication took 2 ms 21/01/17 17:43:07.425 Executor task launch worker for task 3.0 in stage 3.0 (TID 12) INFO Executor: Finished task 3.0 in stage 3.0 (TID 12). 2170 bytes result sent to driver 21/01/17 17:43:07.425 dag-scheduler-event-loop INFO SparkContext: Created broadcast 3 from broadcast at DAGScheduler.scala:1383 21/01/17 17:43:07.425 Executor task launch worker for task 3.0 in stage 3.0 (TID 12) DEBUG ExecutorMetricsPoller: stageTCMP: (3, 0) -> 3 21/01/17 17:43:07.425 dispatcher-event-loop-2 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_3.0, runningTasks: 3 21/01/17 17:43:07.425 dag-scheduler-event-loop INFO DAGScheduler: Submitting 4 missing tasks from ResultStage 1 (MapPartitionsRDD[8] at at /usr/lib/python3.6/runpy.py:85) (first 15 tasks are for partitions Vector(0, 1, 2, 3)) 21/01/17 17:43:07.425 dag-scheduler-event-loop INFO TaskSchedulerImpl: Adding task set 1.0 with 4 tasks resource profile 0 21/01/17 17:43:07.426 dag-scheduler-event-loop DEBUG TaskSetManager: Epoch for TaskSet 1.0: 0 21/01/17 17:43:07.426 dag-scheduler-event-loop DEBUG TaskSetManager: Adding pending tasks took 0 ms 21/01/17 17:43:07.425 Executor task launch worker for task 2.0 in stage 3.0 (TID 11) DEBUG TaskMemoryManager: Task 11 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@e15aad9 21/01/17 17:43:07.426 dag-scheduler-event-loop DEBUG TaskSetManager: Valid locality levels for TaskSet 1.0: NO_PREF, ANY 21/01/17 17:43:07.423 Executor task launch worker for task 1.0 in stage 3.0 (TID 10) DEBUG BlockManager: Told master about block rdd_20_1 21/01/17 17:43:07.427 Executor task launch worker for task 1.0 in stage 3.0 (TID 10) DEBUG BlockManager: Put block rdd_20_1 locally took 67 ms 21/01/17 17:43:07.427 Executor task launch worker for task 2.0 in stage 3.0 (TID 11) INFO MemoryStore: Block rdd_19_2 stored as values in memory (estimated size 192.0 B, free 366.1 MiB) 21/01/17 17:43:07.427 Executor task launch worker for task 0.0 in stage 3.0 (TID 9) DEBUG ALS: Start sorting an uncompressed in-block of size 2. (sortId = -1933324731) 21/01/17 17:43:07.427 dispatcher-event-loop-3 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_1.0, runningTasks: 0 21/01/17 17:43:07.427 Executor task launch worker for task 0.0 in stage 3.0 (TID 9) DEBUG ALS: Sorting took 3.163E-5 seconds. (sortId = -1933324731) 21/01/17 17:43:07.427 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_19_2 for BlockManagerId(driver, 192.168.1.225, 40985, None) 21/01/17 17:43:07.427 Executor task launch worker for task 0.0 in stage 3.0 (TID 9) DEBUG TaskMemoryManager: Task 9 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@525238e9 21/01/17 17:43:07.428 dispatcher-event-loop-3 INFO TaskSetManager: Starting task 0.0 in stage 1.0 (TID 4) (192.168.1.225, executor driver, partition 0, PROCESS_LOCAL, 7333 bytes) taskResourceAssignments Map() 21/01/17 17:43:07.428 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_19_2 in memory on 192.168.1.225:40985 (size: 192.0 B, free: 366.2 MiB) 21/01/17 17:43:07.428 dispatcher-event-loop-3 INFO TaskSetManager: Starting task 1.0 in stage 1.0 (TID 5) (192.168.1.225, executor driver, partition 1, PROCESS_LOCAL, 7421 bytes) taskResourceAssignments Map() 21/01/17 17:43:07.428 Executor task launch worker for task 2.0 in stage 3.0 (TID 11) DEBUG BlockManagerMaster: Updated info of block rdd_19_2 21/01/17 17:43:07.428 Executor task launch worker for task 2.0 in stage 3.0 (TID 11) DEBUG BlockManager: Told master about block rdd_19_2 21/01/17 17:43:07.428 Executor task launch worker for task 2.0 in stage 3.0 (TID 11) DEBUG BlockManager: Put block rdd_19_2 locally took 19 ms 21/01/17 17:43:07.428 dispatcher-event-loop-3 INFO TaskSetManager: Starting task 2.0 in stage 1.0 (TID 6) (192.168.1.225, executor driver, partition 2, PROCESS_LOCAL, 7396 bytes) taskResourceAssignments Map() 21/01/17 17:43:07.429 Executor task launch worker for task 2.0 in stage 3.0 (TID 11) DEBUG BlockManager: Putting block rdd_19_2 without replication took 20 ms 21/01/17 17:43:07.429 Executor task launch worker for task 2.0 in stage 3.0 (TID 11) DEBUG BlockManager: Getting local block rdd_19_2 21/01/17 17:43:07.429 Executor task launch worker for task 2.0 in stage 3.0 (TID 11) DEBUG BlockManager: Level for block rdd_19_2 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:07.426 dispatcher-event-loop-2 DEBUG TaskSetManager: No tasks for locality level NODE_LOCAL, so moving to locality level NO_PREF 21/01/17 17:43:07.427 Executor task launch worker for task 1.0 in stage 3.0 (TID 10) DEBUG BlockManager: Putting block rdd_20_1 without replication took 67 ms 21/01/17 17:43:07.429 dispatcher-event-loop-2 DEBUG TaskSetManager: Moving to ANY after waiting for 0ms 21/01/17 17:43:07.429 Executor task launch worker for task 1.0 in stage 3.0 (TID 10) DEBUG BlockManager: Getting local block rdd_20_1 21/01/17 17:43:07.429 Executor task launch worker for task 1.0 in stage 3.0 (TID 10) DEBUG BlockManager: Level for block rdd_20_1 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:07.429 Thread-1 INFO ShutdownHookManager: Shutdown hook called 21/01/17 17:43:07.430 dispatcher-event-loop-2 INFO TaskSetManager: Starting task 4.0 in stage 3.0 (TID 13) (192.168.1.225, executor driver, partition 4, PROCESS_LOCAL, 7143 bytes) taskResourceAssignments Map() 21/01/17 17:43:07.430 dispatcher-event-loop-3 INFO TaskSetManager: Starting task 3.0 in stage 1.0 (TID 7) (192.168.1.225, executor driver, partition 3, PROCESS_LOCAL, 7385 bytes) taskResourceAssignments Map() 21/01/17 17:43:07.430 Executor task launch worker for task 0.0 in stage 3.0 (TID 9) INFO MemoryStore: Block rdd_19_0 stored as values in memory (estimated size 192.0 B, free 366.1 MiB) 21/01/17 17:43:07.430 Executor task launch worker for task 2.0 in stage 1.0 (TID 6) INFO Executor: Running task 2.0 in stage 1.0 (TID 6) 21/01/17 17:43:07.430 Executor task launch worker for task 3.0 in stage 1.0 (TID 7) INFO Executor: Running task 3.0 in stage 1.0 (TID 7) 21/01/17 17:43:07.430 Executor task launch worker for task 1.0 in stage 1.0 (TID 5) INFO Executor: Running task 1.0 in stage 1.0 (TID 5) 21/01/17 17:43:07.431 Thread-1 INFO ShutdownHookManager: Deleting directory /home/jenkins/workspace/spark-master-test-python-arm/python/target/f344877f-8415-4e7f-bb44-a4c9c9d59bb8/spark-8f0ac86b-67bc-4e16-b3da-541fb377adc9 21/01/17 17:43:07.431 task-result-getter-1 INFO TaskSetManager: Finished task 3.0 in stage 3.0 (TID 12) in 85 ms on 192.168.1.225 (executor driver) (1/10) 21/01/17 17:43:07.431 Executor task launch worker for task 2.0 in stage 1.0 (TID 6) DEBUG ExecutorMetricsPoller: stageTCMP: (1, 0) -> 1 21/01/17 17:43:07.431 Executor task launch worker for task 4.0 in stage 3.0 (TID 13) INFO Executor: Running task 4.0 in stage 3.0 (TID 13) 21/01/17 17:43:07.432 Executor task launch worker for task 1.0 in stage 1.0 (TID 5) DEBUG ExecutorMetricsPoller: stageTCMP: (1, 0) -> 2 21/01/17 17:43:07.432 Executor task launch worker for task 2.0 in stage 1.0 (TID 6) DEBUG BlockManager: Getting local block broadcast_3 21/01/17 17:43:07.432 Executor task launch worker for task 1.0 in stage 3.0 (TID 10) INFO Executor: Finished task 1.0 in stage 3.0 (TID 10). 2170 bytes result sent to driver 21/01/17 17:43:07.432 Executor task launch worker for task 2.0 in stage 1.0 (TID 6) DEBUG BlockManager: Level for block broadcast_3 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:07.430 Executor task launch worker for task 0.0 in stage 1.0 (TID 4) INFO Executor: Running task 0.0 in stage 1.0 (TID 4) 21/01/17 17:43:07.433 Executor task launch worker for task 0.0 in stage 1.0 (TID 4) DEBUG ExecutorMetricsPoller: stageTCMP: (1, 0) -> 3 21/01/17 17:43:07.434 Executor task launch worker for task 2.0 in stage 3.0 (TID 11) INFO MemoryStore: Block rdd_20_2 stored as values in memory (estimated size 296.0 B, free 366.1 MiB) 21/01/17 17:43:07.435 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_19_0 for BlockManagerId(driver, 192.168.1.225, 40985, None) 21/01/17 17:43:07.436 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_19_0 in memory on 192.168.1.225:40985 (size: 192.0 B, free: 366.2 MiB) 21/01/17 17:43:07.436 Executor task launch worker for task 3.0 in stage 1.0 (TID 7) DEBUG ExecutorMetricsPoller: stageTCMP: (1, 0) -> 4 21/01/17 17:43:07.436 Executor task launch worker for task 0.0 in stage 3.0 (TID 9) DEBUG BlockManagerMaster: Updated info of block rdd_19_0 21/01/17 17:43:07.437 Executor task launch worker for task 0.0 in stage 3.0 (TID 9) DEBUG BlockManager: Told master about block rdd_19_0 21/01/17 17:43:07.437 Executor task launch worker for task 0.0 in stage 3.0 (TID 9) DEBUG BlockManager: Put block rdd_19_0 locally took 29 ms 21/01/17 17:43:07.437 Executor task launch worker for task 0.0 in stage 3.0 (TID 9) DEBUG BlockManager: Putting block rdd_19_0 without replication took 29 ms 21/01/17 17:43:07.437 Executor task launch worker for task 0.0 in stage 3.0 (TID 9) DEBUG BlockManager: Getting local block rdd_19_0 21/01/17 17:43:07.437 Thread-1 INFO ShutdownHookManager: Deleting directory /home/jenkins/workspace/spark-master-test-python-arm/python/target/f344877f-8415-4e7f-bb44-a4c9c9d59bb8/spark-8f0ac86b-67bc-4e16-b3da-541fb377adc9/pyspark-b38892c7-7924-430c-a709-5432b3b941a7 21/01/17 17:43:07.437 Executor task launch worker for task 0.0 in stage 3.0 (TID 9) DEBUG BlockManager: Level for block rdd_19_0 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:07.438 Executor task launch worker for task 0.0 in stage 3.0 (TID 9) INFO MemoryStore: Block rdd_20_0 stored as values in memory (estimated size 296.0 B, free 366.1 MiB) 21/01/17 17:43:07.439 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_20_2 for BlockManagerId(driver, 192.168.1.225, 40985, None) 21/01/17 17:43:07.439 Thread-1 INFO ShutdownHookManager: Deleting directory /home/jenkins/workspace/spark-master-test-python-arm/python/target/f344877f-8415-4e7f-bb44-a4c9c9d59bb8/spark-0f9def95-6dd1-45ae-8de8-86d9ab36220c 21/01/17 17:43:07.439 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_20_2 in memory on 192.168.1.225:40985 (size: 296.0 B, free: 366.2 MiB) 21/01/17 17:43:07.440 Executor task launch worker for task 2.0 in stage 3.0 (TID 11) DEBUG BlockManagerMaster: Updated info of block rdd_20_2 21/01/17 17:43:07.440 Executor task launch worker for task 2.0 in stage 3.0 (TID 11) DEBUG BlockManager: Told master about block rdd_20_2 21/01/17 17:43:07.440 Executor task launch worker for task 2.0 in stage 3.0 (TID 11) DEBUG BlockManager: Put block rdd_20_2 locally took 32 ms 21/01/17 17:43:07.440 Executor task launch worker for task 2.0 in stage 3.0 (TID 11) DEBUG BlockManager: Putting block rdd_20_2 without replication took 32 ms 21/01/17 17:43:07.440 Executor task launch worker for task 2.0 in stage 3.0 (TID 11) DEBUG BlockManager: Getting local block rdd_20_2 21/01/17 17:43:07.440 Executor task launch worker for task 2.0 in stage 3.0 (TID 11) DEBUG BlockManager: Level for block rdd_20_2 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:07.440 dispatcher-event-loop-1 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_3.0, runningTasks: 3 21/01/17 17:43:07.440 dispatcher-event-loop-1 DEBUG TaskSetManager: No tasks for locality level NODE_LOCAL, so moving to locality level NO_PREF 21/01/17 17:43:07.440 dispatcher-event-loop-1 DEBUG TaskSetManager: Moving to ANY after waiting for 0ms 21/01/17 17:43:07.441 dispatcher-event-loop-1 INFO TaskSetManager: Starting task 5.0 in stage 3.0 (TID 14) (192.168.1.225, executor driver, partition 5, PROCESS_LOCAL, 7143 bytes) taskResourceAssignments Map() 21/01/17 17:43:07.442 task-result-getter-2 INFO TaskSetManager: Finished task 1.0 in stage 3.0 (TID 10) in 97 ms on 192.168.1.225 (executor driver) (2/10) 21/01/17 17:43:07.442 Executor task launch worker for task 2.0 in stage 3.0 (TID 11) INFO Executor: Finished task 2.0 in stage 3.0 (TID 11). 2170 bytes result sent to driver 21/01/17 17:43:07.442 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_20_0 for BlockManagerId(driver, 192.168.1.225, 40985, None) 21/01/17 17:43:07.443 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_20_0 in memory on 192.168.1.225:40985 (size: 296.0 B, free: 366.2 MiB) 21/01/17 17:43:07.442 Executor task launch worker for task 1.0 in stage 3.0 (TID 10) DEBUG ExecutorMetricsPoller: stageTCMP: (3, 0) -> 2 21/01/17 17:43:07.443 Executor task launch worker for task 0.0 in stage 3.0 (TID 9) DEBUG BlockManagerMaster: Updated info of block rdd_20_0 21/01/17 17:43:07.444 Executor task launch worker for task 0.0 in stage 3.0 (TID 9) DEBUG BlockManager: Told master about block rdd_20_0 21/01/17 17:43:07.444 Executor task launch worker for task 0.0 in stage 3.0 (TID 9) DEBUG BlockManager: Put block rdd_20_0 locally took 37 ms 21/01/17 17:43:07.444 Executor task launch worker for task 0.0 in stage 3.0 (TID 9) DEBUG BlockManager: Putting block rdd_20_0 without replication took 37 ms 21/01/17 17:43:07.444 Executor task launch worker for task 4.0 in stage 3.0 (TID 13) DEBUG ExecutorMetricsPoller: stageTCMP: (3, 0) -> 3 21/01/17 17:43:07.444 Executor task launch worker for task 0.0 in stage 3.0 (TID 9) DEBUG BlockManager: Getting local block rdd_20_0 21/01/17 17:43:07.444 Executor task launch worker for task 0.0 in stage 3.0 (TID 9) DEBUG BlockManager: Level for block rdd_20_0 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:07.445 Executor task launch worker for task 0.0 in stage 3.0 (TID 9) INFO Executor: Finished task 0.0 in stage 3.0 (TID 9). 2170 bytes result sent to driver 21/01/17 17:43:07.446 Executor task launch worker for task 0.0 in stage 3.0 (TID 9) DEBUG ExecutorMetricsPoller: stageTCMP: (3, 0) -> 2 21/01/17 17:43:07.452 Executor task launch worker for task 4.0 in stage 3.0 (TID 13) DEBUG BlockManager: Getting local block rdd_20_4 21/01/17 17:43:07.452 Executor task launch worker for task 4.0 in stage 3.0 (TID 13) DEBUG BlockManager: Block rdd_20_4 was not found 21/01/17 17:43:07.452 Executor task launch worker for task 4.0 in stage 3.0 (TID 13) DEBUG BlockManager: Getting remote block rdd_20_4 21/01/17 17:43:07.452 Executor task launch worker for task 2.0 in stage 3.0 (TID 11) DEBUG ExecutorMetricsPoller: stageTCMP: (3, 0) -> 1 21/01/17 17:43:07.452 Executor task launch worker for task 4.0 in stage 3.0 (TID 13) DEBUG BlockManager: Block rdd_20_4 is unknown by block manager master 21/01/17 17:43:07.453 Executor task launch worker for task 4.0 in stage 3.0 (TID 13) DEBUG BlockManager: Getting local block rdd_19_4 21/01/17 17:43:07.453 Executor task launch worker for task 4.0 in stage 3.0 (TID 13) DEBUG BlockManager: Block rdd_19_4 was not found 21/01/17 17:43:07.453 Executor task launch worker for task 4.0 in stage 3.0 (TID 13) DEBUG BlockManager: Getting remote block rdd_19_4 21/01/17 17:43:07.453 dispatcher-event-loop-1 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_3.0, runningTasks: 3 21/01/17 17:43:07.454 dispatcher-event-loop-1 DEBUG TaskSetManager: No tasks for locality level NODE_LOCAL, so moving to locality level NO_PREF 21/01/17 17:43:07.454 dispatcher-event-loop-1 DEBUG TaskSetManager: Moving to ANY after waiting for 0ms 21/01/17 17:43:07.454 Executor task launch worker for task 5.0 in stage 3.0 (TID 14) INFO Executor: Running task 5.0 in stage 3.0 (TID 14) 21/01/17 17:43:07.455 Executor task launch worker for task 5.0 in stage 3.0 (TID 14) DEBUG ExecutorMetricsPoller: stageTCMP: (3, 0) -> 2 21/01/17 17:43:07.457 Executor task launch worker for task 4.0 in stage 3.0 (TID 13) DEBUG BlockManager: Block rdd_19_4 is unknown by block manager master 21/01/17 17:43:07.457 dispatcher-event-loop-1 INFO TaskSetManager: Starting task 6.0 in stage 3.0 (TID 15) (192.168.1.225, executor driver, partition 6, PROCESS_LOCAL, 7143 bytes) taskResourceAssignments Map() 21/01/17 17:43:07.457 Executor task launch worker for task 4.0 in stage 3.0 (TID 13) DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 0 21/01/17 17:43:07.457 Executor task launch worker for task 4.0 in stage 3.0 (TID 13) DEBUG MapOutputTrackerMaster: Convert map statuses for shuffle 0, mappers 0-4, partitions 4-5 21/01/17 17:43:07.457 Executor task launch worker for task 6.0 in stage 3.0 (TID 15) INFO Executor: Running task 6.0 in stage 3.0 (TID 15) 21/01/17 17:43:07.458 dispatcher-event-loop-1 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_3.0, runningTasks: 3 21/01/17 17:43:07.458 Executor task launch worker for task 4.0 in stage 3.0 (TID 13) DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 21/01/17 17:43:07.458 dispatcher-event-loop-1 DEBUG TaskSetManager: No tasks for locality level NODE_LOCAL, so moving to locality level NO_PREF 21/01/17 17:43:07.458 dispatcher-event-loop-1 DEBUG TaskSetManager: Moving to ANY after waiting for 0ms 21/01/17 17:43:07.458 Executor task launch worker for task 6.0 in stage 3.0 (TID 15) DEBUG ExecutorMetricsPoller: stageTCMP: (3, 0) -> 3 21/01/17 17:43:07.458 Executor task launch worker for task 4.0 in stage 3.0 (TID 13) INFO ShuffleBlockFetcherIterator: Getting 0 (0.0 B) non-empty blocks including 0 (0.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 21/01/17 17:43:07.458 Executor task launch worker for task 4.0 in stage 3.0 (TID 13) INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 21/01/17 17:43:07.458 Executor task launch worker for task 4.0 in stage 3.0 (TID 13) DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: 21/01/17 17:43:07.458 dispatcher-event-loop-1 INFO TaskSetManager: Starting task 7.0 in stage 3.0 (TID 16) (192.168.1.225, executor driver, partition 7, PROCESS_LOCAL, 7143 bytes) taskResourceAssignments Map() 21/01/17 17:43:07.458 Executor task launch worker for task 4.0 in stage 3.0 (TID 13) DEBUG ShuffleBlockFetcherIterator: Got local blocks in 0 ms 21/01/17 17:43:07.459 Executor task launch worker for task 7.0 in stage 3.0 (TID 16) INFO Executor: Running task 7.0 in stage 3.0 (TID 16) 21/01/17 17:43:07.459 task-result-getter-0 INFO TaskSetManager: Finished task 2.0 in stage 3.0 (TID 11) in 114 ms on 192.168.1.225 (executor driver) (3/10) 21/01/17 17:43:07.459 task-result-getter-3 INFO TaskSetManager: Finished task 0.0 in stage 3.0 (TID 9) in 115 ms on 192.168.1.225 (executor driver) (4/10) 21/01/17 17:43:07.459 Executor task launch worker for task 7.0 in stage 3.0 (TID 16) DEBUG ExecutorMetricsPoller: stageTCMP: (3, 0) -> 4 21/01/17 17:43:07.459 Executor task launch worker for task 4.0 in stage 3.0 (TID 13) DEBUG TaskMemoryManager: Task 13 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@19d30f73 21/01/17 17:43:07.461 Executor task launch worker for task 4.0 in stage 3.0 (TID 13) INFO MemoryStore: Block rdd_19_4 stored as values in memory (estimated size 16.0 B, free 366.1 MiB) 21/01/17 17:43:07.461 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_19_4 for BlockManagerId(driver, 192.168.1.225, 40985, None) 21/01/17 17:43:07.461 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_19_4 in memory on 192.168.1.225:40985 (size: 16.0 B, free: 366.2 MiB) 21/01/17 17:43:07.462 Executor task launch worker for task 4.0 in stage 3.0 (TID 13) DEBUG BlockManagerMaster: Updated info of block rdd_19_4 21/01/17 17:43:07.462 Executor task launch worker for task 4.0 in stage 3.0 (TID 13) DEBUG BlockManager: Told master about block rdd_19_4 21/01/17 17:43:07.462 Executor task launch worker for task 4.0 in stage 3.0 (TID 13) DEBUG BlockManager: Put block rdd_19_4 locally took 5 ms 21/01/17 17:43:07.462 Executor task launch worker for task 4.0 in stage 3.0 (TID 13) DEBUG BlockManager: Putting block rdd_19_4 without replication took 5 ms 21/01/17 17:43:07.462 Executor task launch worker for task 4.0 in stage 3.0 (TID 13) DEBUG BlockManager: Getting local block rdd_19_4 21/01/17 17:43:07.462 Executor task launch worker for task 4.0 in stage 3.0 (TID 13) DEBUG BlockManager: Level for block rdd_19_4 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:07.463 Executor task launch worker for task 4.0 in stage 3.0 (TID 13) INFO MemoryStore: Block rdd_20_4 stored as values in memory (estimated size 16.0 B, free 366.1 MiB) 21/01/17 17:43:07.463 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_20_4 for BlockManagerId(driver, 192.168.1.225, 40985, None) 21/01/17 17:43:07.464 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_20_4 in memory on 192.168.1.225:40985 (size: 16.0 B, free: 366.2 MiB) 21/01/17 17:43:07.464 Executor task launch worker for task 4.0 in stage 3.0 (TID 13) DEBUG BlockManagerMaster: Updated info of block rdd_20_4 21/01/17 17:43:07.464 Executor task launch worker for task 4.0 in stage 3.0 (TID 13) DEBUG BlockManager: Told master about block rdd_20_4 21/01/17 17:43:07.464 Executor task launch worker for task 4.0 in stage 3.0 (TID 13) DEBUG BlockManager: Put block rdd_20_4 locally took 11 ms 21/01/17 17:43:07.464 Executor task launch worker for task 4.0 in stage 3.0 (TID 13) DEBUG BlockManager: Putting block rdd_20_4 without replication took 11 ms 21/01/17 17:43:07.464 Executor task launch worker for task 4.0 in stage 3.0 (TID 13) DEBUG BlockManager: Getting local block rdd_20_4 21/01/17 17:43:07.465 Executor task launch worker for task 4.0 in stage 3.0 (TID 13) DEBUG BlockManager: Level for block rdd_20_4 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:07.466 Executor task launch worker for task 4.0 in stage 3.0 (TID 13) INFO Executor: Finished task 4.0 in stage 3.0 (TID 13). 2170 bytes result sent to driver 21/01/17 17:43:07.467 dispatcher-event-loop-0 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_3.0, runningTasks: 3 21/01/17 17:43:07.467 dispatcher-event-loop-0 DEBUG TaskSetManager: No tasks for locality level NODE_LOCAL, so moving to locality level NO_PREF 21/01/17 17:43:07.467 dispatcher-event-loop-0 DEBUG TaskSetManager: Moving to ANY after waiting for 0ms 21/01/17 17:43:07.467 dispatcher-event-loop-0 INFO TaskSetManager: Starting task 8.0 in stage 3.0 (TID 17) (192.168.1.225, executor driver, partition 8, PROCESS_LOCAL, 7143 bytes) taskResourceAssignments Map() 21/01/17 17:43:07.468 Executor task launch worker for task 8.0 in stage 3.0 (TID 17) INFO Executor: Running task 8.0 in stage 3.0 (TID 17) 21/01/17 17:43:07.468 task-result-getter-1 INFO TaskSetManager: Finished task 4.0 in stage 3.0 (TID 13) in 39 ms on 192.168.1.225 (executor driver) (5/10) 21/01/17 17:43:07.468 Executor task launch worker for task 4.0 in stage 3.0 (TID 13) DEBUG ExecutorMetricsPoller: stageTCMP: (3, 0) -> 3 21/01/17 17:43:07.469 Executor task launch worker for task 8.0 in stage 3.0 (TID 17) DEBUG ExecutorMetricsPoller: stageTCMP: (3, 0) -> 4 21/01/17 17:43:07.467 Executor task launch worker for task 5.0 in stage 3.0 (TID 14) DEBUG BlockManager: Getting local block rdd_20_5 21/01/17 17:43:07.472 Executor task launch worker for task 5.0 in stage 3.0 (TID 14) DEBUG BlockManager: Block rdd_20_5 was not found 21/01/17 17:43:07.472 Executor task launch worker for task 5.0 in stage 3.0 (TID 14) DEBUG BlockManager: Getting remote block rdd_20_5 21/01/17 17:43:07.476 Executor task launch worker for task 6.0 in stage 3.0 (TID 15) DEBUG BlockManager: Getting local block rdd_20_6 21/01/17 17:43:07.476 Executor task launch worker for task 6.0 in stage 3.0 (TID 15) DEBUG BlockManager: Block rdd_20_6 was not found 21/01/17 17:43:07.476 Executor task launch worker for task 6.0 in stage 3.0 (TID 15) DEBUG BlockManager: Getting remote block rdd_20_6 21/01/17 17:43:07.477 Executor task launch worker for task 8.0 in stage 3.0 (TID 17) DEBUG BlockManager: Getting local block rdd_20_8 21/01/17 17:43:07.477 Executor task launch worker for task 8.0 in stage 3.0 (TID 17) DEBUG BlockManager: Block rdd_20_8 was not found 21/01/17 17:43:07.477 Executor task launch worker for task 8.0 in stage 3.0 (TID 17) DEBUG BlockManager: Getting remote block rdd_20_8 21/01/17 17:43:07.477 Executor task launch worker for task 8.0 in stage 3.0 (TID 17) DEBUG BlockManager: Block rdd_20_8 is unknown by block manager master 21/01/17 17:43:07.477 Executor task launch worker for task 8.0 in stage 3.0 (TID 17) DEBUG BlockManager: Getting local block rdd_19_8 21/01/17 17:43:07.477 Executor task launch worker for task 8.0 in stage 3.0 (TID 17) DEBUG BlockManager: Block rdd_19_8 was not found 21/01/17 17:43:07.477 Executor task launch worker for task 8.0 in stage 3.0 (TID 17) DEBUG BlockManager: Getting remote block rdd_19_8 21/01/17 17:43:07.478 Executor task launch worker for task 5.0 in stage 3.0 (TID 14) DEBUG BlockManager: Block rdd_20_5 is unknown by block manager master 21/01/17 17:43:07.478 Executor task launch worker for task 5.0 in stage 3.0 (TID 14) DEBUG BlockManager: Getting local block rdd_19_5 21/01/17 17:43:07.478 Executor task launch worker for task 5.0 in stage 3.0 (TID 14) DEBUG BlockManager: Block rdd_19_5 was not found 21/01/17 17:43:07.478 Executor task launch worker for task 5.0 in stage 3.0 (TID 14) DEBUG BlockManager: Getting remote block rdd_19_5 21/01/17 17:43:07.478 Executor task launch worker for task 8.0 in stage 3.0 (TID 17) DEBUG BlockManager: Block rdd_19_8 is unknown by block manager master 21/01/17 17:43:07.478 Executor task launch worker for task 8.0 in stage 3.0 (TID 17) DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 0 21/01/17 17:43:07.478 Executor task launch worker for task 8.0 in stage 3.0 (TID 17) DEBUG MapOutputTrackerMaster: Convert map statuses for shuffle 0, mappers 0-4, partitions 8-9 21/01/17 17:43:07.479 Executor task launch worker for task 8.0 in stage 3.0 (TID 17) DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 21/01/17 17:43:07.479 Executor task launch worker for task 8.0 in stage 3.0 (TID 17) INFO ShuffleBlockFetcherIterator: Getting 0 (0.0 B) non-empty blocks including 0 (0.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 21/01/17 17:43:07.479 Executor task launch worker for task 8.0 in stage 3.0 (TID 17) INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 21/01/17 17:43:07.480 Executor task launch worker for task 8.0 in stage 3.0 (TID 17) DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: 21/01/17 17:43:07.480 Executor task launch worker for task 8.0 in stage 3.0 (TID 17) DEBUG ShuffleBlockFetcherIterator: Got local blocks in 0 ms 21/01/17 17:43:07.480 Executor task launch worker for task 7.0 in stage 3.0 (TID 16) DEBUG BlockManager: Getting local block rdd_20_7 21/01/17 17:43:07.480 Executor task launch worker for task 7.0 in stage 3.0 (TID 16) DEBUG BlockManager: Block rdd_20_7 was not found 21/01/17 17:43:07.480 Executor task launch worker for task 7.0 in stage 3.0 (TID 16) DEBUG BlockManager: Getting remote block rdd_20_7 21/01/17 17:43:07.480 Executor task launch worker for task 8.0 in stage 3.0 (TID 17) DEBUG TaskMemoryManager: Task 17 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@38bc76a6 21/01/17 17:43:07.481 Executor task launch worker for task 8.0 in stage 3.0 (TID 17) INFO MemoryStore: Block rdd_19_8 stored as values in memory (estimated size 16.0 B, free 366.1 MiB) 21/01/17 17:43:07.481 Executor task launch worker for task 5.0 in stage 3.0 (TID 14) DEBUG BlockManager: Block rdd_19_5 is unknown by block manager master 21/01/17 17:43:07.481 Executor task launch worker for task 5.0 in stage 3.0 (TID 14) DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 0 21/01/17 17:43:07.481 Executor task launch worker for task 5.0 in stage 3.0 (TID 14) DEBUG MapOutputTrackerMaster: Convert map statuses for shuffle 0, mappers 0-4, partitions 5-6 21/01/17 17:43:07.482 Executor task launch worker for task 5.0 in stage 3.0 (TID 14) DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 21/01/17 17:43:07.482 Executor task launch worker for task 5.0 in stage 3.0 (TID 14) INFO ShuffleBlockFetcherIterator: Getting 0 (0.0 B) non-empty blocks including 0 (0.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 21/01/17 17:43:07.482 Executor task launch worker for task 5.0 in stage 3.0 (TID 14) INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 21/01/17 17:43:07.482 Executor task launch worker for task 5.0 in stage 3.0 (TID 14) DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: 21/01/17 17:43:07.482 Executor task launch worker for task 5.0 in stage 3.0 (TID 14) DEBUG ShuffleBlockFetcherIterator: Got local blocks in 0 ms 21/01/17 17:43:07.483 Executor task launch worker for task 5.0 in stage 3.0 (TID 14) DEBUG TaskMemoryManager: Task 14 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@3aeb725e 21/01/17 17:43:07.484 Executor task launch worker for task 5.0 in stage 3.0 (TID 14) INFO MemoryStore: Block rdd_19_5 stored as values in memory (estimated size 16.0 B, free 366.1 MiB) 21/01/17 17:43:07.484 Executor task launch worker for task 7.0 in stage 3.0 (TID 16) DEBUG BlockManager: Block rdd_20_7 is unknown by block manager master 21/01/17 17:43:07.484 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_19_8 for BlockManagerId(driver, 192.168.1.225, 40985, None) 21/01/17 17:43:07.485 Executor task launch worker for task 7.0 in stage 3.0 (TID 16) DEBUG BlockManager: Getting local block rdd_19_7 21/01/17 17:43:07.485 Executor task launch worker for task 7.0 in stage 3.0 (TID 16) DEBUG BlockManager: Block rdd_19_7 was not found 21/01/17 17:43:07.485 Executor task launch worker for task 7.0 in stage 3.0 (TID 16) DEBUG BlockManager: Getting remote block rdd_19_7 21/01/17 17:43:07.485 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_19_8 in memory on 192.168.1.225:40985 (size: 16.0 B, free: 366.2 MiB) 21/01/17 17:43:07.485 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_19_5 for BlockManagerId(driver, 192.168.1.225, 40985, None) 21/01/17 17:43:07.485 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_19_5 in memory on 192.168.1.225:40985 (size: 16.0 B, free: 366.2 MiB) 21/01/17 17:43:07.489 Executor task launch worker for task 8.0 in stage 3.0 (TID 17) DEBUG BlockManagerMaster: Updated info of block rdd_19_8 21/01/17 17:43:07.489 Executor task launch worker for task 6.0 in stage 3.0 (TID 15) DEBUG BlockManager: Block rdd_20_6 is unknown by block manager master 21/01/17 17:43:07.489 Executor task launch worker for task 5.0 in stage 3.0 (TID 14) DEBUG BlockManagerMaster: Updated info of block rdd_19_5 21/01/17 17:43:07.489 Executor task launch worker for task 5.0 in stage 3.0 (TID 14) DEBUG BlockManager: Told master about block rdd_19_5 21/01/17 17:43:07.489 Executor task launch worker for task 5.0 in stage 3.0 (TID 14) DEBUG BlockManager: Put block rdd_19_5 locally took 7 ms 21/01/17 17:43:07.489 Executor task launch worker for task 6.0 in stage 3.0 (TID 15) DEBUG BlockManager: Getting local block rdd_19_6 21/01/17 17:43:07.489 Executor task launch worker for task 5.0 in stage 3.0 (TID 14) DEBUG BlockManager: Putting block rdd_19_5 without replication took 7 ms 21/01/17 17:43:07.489 Executor task launch worker for task 5.0 in stage 3.0 (TID 14) DEBUG BlockManager: Getting local block rdd_19_5 21/01/17 17:43:07.489 Executor task launch worker for task 5.0 in stage 3.0 (TID 14) DEBUG BlockManager: Level for block rdd_19_5 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:07.490 Executor task launch worker for task 5.0 in stage 3.0 (TID 14) INFO MemoryStore: Block rdd_20_5 stored as values in memory (estimated size 16.0 B, free 366.1 MiB) 21/01/17 17:43:07.489 Executor task launch worker for task 8.0 in stage 3.0 (TID 17) DEBUG BlockManager: Told master about block rdd_19_8 21/01/17 17:43:07.489 Executor task launch worker for task 6.0 in stage 3.0 (TID 15) DEBUG BlockManager: Block rdd_19_6 was not found 21/01/17 17:43:07.491 Executor task launch worker for task 6.0 in stage 3.0 (TID 15) DEBUG BlockManager: Getting remote block rdd_19_6 21/01/17 17:43:07.491 Executor task launch worker for task 8.0 in stage 3.0 (TID 17) DEBUG BlockManager: Put block rdd_19_8 locally took 12 ms 21/01/17 17:43:07.492 Executor task launch worker for task 8.0 in stage 3.0 (TID 17) DEBUG BlockManager: Putting block rdd_19_8 without replication took 13 ms 21/01/17 17:43:07.492 Executor task launch worker for task 8.0 in stage 3.0 (TID 17) DEBUG BlockManager: Getting local block rdd_19_8 21/01/17 17:43:07.492 Executor task launch worker for task 8.0 in stage 3.0 (TID 17) DEBUG BlockManager: Level for block rdd_19_8 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:07.492 Executor task launch worker for task 8.0 in stage 3.0 (TID 17) INFO MemoryStore: Block rdd_20_8 stored as values in memory (estimated size 16.0 B, free 366.1 MiB) 21/01/17 17:43:07.492 Executor task launch worker for task 7.0 in stage 3.0 (TID 16) DEBUG BlockManager: Block rdd_19_7 is unknown by block manager master 21/01/17 17:43:07.493 Executor task launch worker for task 7.0 in stage 3.0 (TID 16) DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 0 21/01/17 17:43:07.493 Executor task launch worker for task 7.0 in stage 3.0 (TID 16) DEBUG MapOutputTrackerMaster: Convert map statuses for shuffle 0, mappers 0-4, partitions 7-8 21/01/17 17:43:07.493 Executor task launch worker for task 7.0 in stage 3.0 (TID 16) DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 21/01/17 17:43:07.494 Executor task launch worker for task 7.0 in stage 3.0 (TID 16) INFO ShuffleBlockFetcherIterator: Getting 0 (0.0 B) non-empty blocks including 0 (0.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 21/01/17 17:43:07.494 Executor task launch worker for task 7.0 in stage 3.0 (TID 16) INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 21/01/17 17:43:07.494 Executor task launch worker for task 7.0 in stage 3.0 (TID 16) DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: 21/01/17 17:43:07.494 Executor task launch worker for task 7.0 in stage 3.0 (TID 16) DEBUG ShuffleBlockFetcherIterator: Got local blocks in 0 ms 21/01/17 17:43:07.494 Executor task launch worker for task 7.0 in stage 3.0 (TID 16) DEBUG TaskMemoryManager: Task 16 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@302f05b0 21/01/17 17:43:07.495 Executor task launch worker for task 7.0 in stage 3.0 (TID 16) INFO MemoryStore: Block rdd_19_7 stored as values in memory (estimated size 16.0 B, free 366.1 MiB) 21/01/17 17:43:07.496 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_20_5 for BlockManagerId(driver, 192.168.1.225, 40985, None) 21/01/17 17:43:07.496 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_20_5 in memory on 192.168.1.225:40985 (size: 16.0 B, free: 366.2 MiB) 21/01/17 17:43:07.497 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_19_7 for BlockManagerId(driver, 192.168.1.225, 40985, None) 21/01/17 17:43:07.497 Executor task launch worker for task 5.0 in stage 3.0 (TID 14) DEBUG BlockManagerMaster: Updated info of block rdd_20_5 21/01/17 17:43:07.497 Executor task launch worker for task 5.0 in stage 3.0 (TID 14) DEBUG BlockManager: Told master about block rdd_20_5 21/01/17 17:43:07.497 Executor task launch worker for task 5.0 in stage 3.0 (TID 14) DEBUG BlockManager: Put block rdd_20_5 locally took 19 ms 21/01/17 17:43:07.497 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_19_7 in memory on 192.168.1.225:40985 (size: 16.0 B, free: 366.2 MiB) 21/01/17 17:43:07.498 Executor task launch worker for task 7.0 in stage 3.0 (TID 16) DEBUG BlockManagerMaster: Updated info of block rdd_19_7 21/01/17 17:43:07.497 Executor task launch worker for task 5.0 in stage 3.0 (TID 14) DEBUG BlockManager: Putting block rdd_20_5 without replication took 19 ms 21/01/17 17:43:07.498 Executor task launch worker for task 7.0 in stage 3.0 (TID 16) DEBUG BlockManager: Told master about block rdd_19_7 21/01/17 17:43:07.498 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_20_8 for BlockManagerId(driver, 192.168.1.225, 40985, None) 21/01/17 17:43:07.498 Executor task launch worker for task 5.0 in stage 3.0 (TID 14) DEBUG BlockManager: Getting local block rdd_20_5 21/01/17 17:43:07.498 Executor task launch worker for task 5.0 in stage 3.0 (TID 14) DEBUG BlockManager: Level for block rdd_20_5 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:07.498 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_20_8 in memory on 192.168.1.225:40985 (size: 16.0 B, free: 366.2 MiB) 21/01/17 17:43:07.499 Executor task launch worker for task 8.0 in stage 3.0 (TID 17) DEBUG BlockManagerMaster: Updated info of block rdd_20_8 21/01/17 17:43:07.499 Executor task launch worker for task 8.0 in stage 3.0 (TID 17) DEBUG BlockManager: Told master about block rdd_20_8 21/01/17 17:43:07.499 Executor task launch worker for task 8.0 in stage 3.0 (TID 17) DEBUG BlockManager: Put block rdd_20_8 locally took 21 ms 21/01/17 17:43:07.499 Executor task launch worker for task 6.0 in stage 3.0 (TID 15) DEBUG BlockManager: Block rdd_19_6 is unknown by block manager master 21/01/17 17:43:07.499 Executor task launch worker for task 7.0 in stage 3.0 (TID 16) DEBUG BlockManager: Put block rdd_19_7 locally took 6 ms 21/01/17 17:43:07.499 Executor task launch worker for task 8.0 in stage 3.0 (TID 17) DEBUG BlockManager: Putting block rdd_20_8 without replication took 21 ms 21/01/17 17:43:07.499 Executor task launch worker for task 6.0 in stage 3.0 (TID 15) DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 0 21/01/17 17:43:07.499 Executor task launch worker for task 8.0 in stage 3.0 (TID 17) DEBUG BlockManager: Getting local block rdd_20_8 21/01/17 17:43:07.499 Executor task launch worker for task 7.0 in stage 3.0 (TID 16) DEBUG BlockManager: Putting block rdd_19_7 without replication took 6 ms 21/01/17 17:43:07.499 Executor task launch worker for task 6.0 in stage 3.0 (TID 15) DEBUG MapOutputTrackerMaster: Convert map statuses for shuffle 0, mappers 0-4, partitions 6-7 21/01/17 17:43:07.499 Executor task launch worker for task 7.0 in stage 3.0 (TID 16) DEBUG BlockManager: Getting local block rdd_19_7 21/01/17 17:43:07.499 Executor task launch worker for task 8.0 in stage 3.0 (TID 17) DEBUG BlockManager: Level for block rdd_20_8 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:07.499 Executor task launch worker for task 7.0 in stage 3.0 (TID 16) DEBUG BlockManager: Level for block rdd_19_7 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:07.500 Executor task launch worker for task 6.0 in stage 3.0 (TID 15) DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 21/01/17 17:43:07.500 Executor task launch worker for task 6.0 in stage 3.0 (TID 15) INFO ShuffleBlockFetcherIterator: Getting 0 (0.0 B) non-empty blocks including 0 (0.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 21/01/17 17:43:07.500 Executor task launch worker for task 7.0 in stage 3.0 (TID 16) INFO MemoryStore: Block rdd_20_7 stored as values in memory (estimated size 16.0 B, free 366.1 MiB) 21/01/17 17:43:07.500 Executor task launch worker for task 6.0 in stage 3.0 (TID 15) INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 21/01/17 17:43:07.500 Executor task launch worker for task 6.0 in stage 3.0 (TID 15) DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: 21/01/17 17:43:07.500 Executor task launch worker for task 6.0 in stage 3.0 (TID 15) DEBUG ShuffleBlockFetcherIterator: Got local blocks in 0 ms 21/01/17 17:43:07.500 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_20_7 for BlockManagerId(driver, 192.168.1.225, 40985, None) 21/01/17 17:43:07.500 Executor task launch worker for task 8.0 in stage 3.0 (TID 17) INFO Executor: Finished task 8.0 in stage 3.0 (TID 17). 2170 bytes result sent to driver 21/01/17 17:43:07.501 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_20_7 in memory on 192.168.1.225:40985 (size: 16.0 B, free: 366.2 MiB) 21/01/17 17:43:07.501 Executor task launch worker for task 6.0 in stage 3.0 (TID 15) DEBUG TaskMemoryManager: Task 15 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@169a952b 21/01/17 17:43:07.501 Executor task launch worker for task 7.0 in stage 3.0 (TID 16) DEBUG BlockManagerMaster: Updated info of block rdd_20_7 21/01/17 17:43:07.501 Executor task launch worker for task 7.0 in stage 3.0 (TID 16) DEBUG BlockManager: Told master about block rdd_20_7 21/01/17 17:43:07.501 Executor task launch worker for task 7.0 in stage 3.0 (TID 16) DEBUG BlockManager: Put block rdd_20_7 locally took 16 ms 21/01/17 17:43:07.501 Executor task launch worker for task 7.0 in stage 3.0 (TID 16) DEBUG BlockManager: Putting block rdd_20_7 without replication took 16 ms 21/01/17 17:43:07.501 Executor task launch worker for task 7.0 in stage 3.0 (TID 16) DEBUG BlockManager: Getting local block rdd_20_7 21/01/17 17:43:07.501 Executor task launch worker for task 6.0 in stage 3.0 (TID 15) INFO MemoryStore: Block rdd_19_6 stored as values in memory (estimated size 16.0 B, free 366.1 MiB) 21/01/17 17:43:07.501 Executor task launch worker for task 7.0 in stage 3.0 (TID 16) DEBUG BlockManager: Level for block rdd_20_7 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:07.480 Executor task launch worker for task 1.0 in stage 1.0 (TID 2) DEBUG GenerateSafeProjection: code for createexternalrow(input[0, double, true], newInstance(class org.apache.spark.ml.linalg.VectorUDT).deserialize, input[2, double, false], StructField(prediction,DoubleType,true), StructField(normalizedFeatures,org.apache.spark.ml.linalg.VectorUDT@3bfc3ba7,true), StructField(weightCol,DoubleType,false)): /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificSafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificSafeProjection extends org.apache.spark.sql.catalyst.expressions.codegen.BaseProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private InternalRow mutableRow; /* 009 */ private boolean resultIsNull_0; /* 010 */ private InternalRow[] mutableStateArray_0 = new InternalRow[1]; /* 011 */ /* 012 */ public SpecificSafeProjection(Object[] references) { /* 013 */ this.references = references; /* 014 */ mutableRow = (InternalRow) references[references.length - 1]; /* 015 */ /* 016 */ /* 017 */ } /* 018 */ /* 019 */ public void initialize(int partitionIndex) { /* 020 */ /* 021 */ } /* 022 */ /* 023 */ public java.lang.Object apply(java.lang.Object _i) { /* 024 */ InternalRow i = (InternalRow) _i; /* 025 */ Object[] values_0 = new Object[3]; /* 026 */ createExternalRow_0_0(i, values_0); /* 027 */ createExternalRow_0_1(i, values_0); /* 028 */ final org.apache.spark.sql.Row value_0 = new org.apache.spark.sql.catalyst.expressions.GenericRowWithSchema(values_0, ((org.apache.spark.sql.types.StructType) references[0] /* schema */)); /* 029 */ if (false) { /* 030 */ mutableRow.setNullAt(0); /* 031 */ } else { /* 032 */ /* 033 */ mutableRow.update(0, value_0); /* 034 */ } /* 035 */ /* 036 */ return mutableRow; /* 037 */ } /* 038 */ /* 039 */ /* 040 */ private void createExternalRow_0_1(InternalRow i, Object[] values_0) { /* 041 */ /* 042 */ double value_5 = i.getDouble(2); /* 043 */ if (false) { /* 044 */ values_0[2] = null; /* 045 */ } else { /* 046 */ values_0[2] = value_5; /* 047 */ } /* 048 */ /* 049 */ } /* 050 */ /* 051 */ /* 052 */ private void createExternalRow_0_0(InternalRow i, Object[] values_0) { /* 053 */ /* 054 */ boolean isNull_1 = i.isNullAt(0); /* 055 */ double value_1 = isNull_1 ? /* 056 */ -1.0 : (i.getDouble(0)); /* 057 */ if (isNull_1) { /* 058 */ values_0[0] = null; /* 059 */ } else { /* 060 */ values_0[0] = value_1; /* 061 */ } /* 062 */ /* 063 */ final org.apache.spark.ml.linalg.VectorUDT value_3 = false ? /* 064 */ null : new org.apache.spark.ml.linalg.VectorUDT(); /* 065 */ boolean isNull_2 = true; /* 066 */ org.apache.spark.ml.linalg.Vector value_2 = null; /* 067 */ if (!false) { /* 068 */ resultIsNull_0 = false; /* 069 */ if (!resultIsNull_0) { /* 070 */ boolean isNull_4 = i.isNullAt(1); /* 071 */ InternalRow value_4 = isNull_4 ? /* 072 */ null : (i.getStruct(1, 4)); /* 073 */ resultIsNull_0 = isNull_4; /* 074 */ mutableStateArray_0[0] = value_4; /* 075 */ } /* 076 */ /* 077 */ isNull_2 = resultIsNull_0; /* 078 */ if (!isNull_2) { /* 079 */ /* 080 */ Object funcResult_0 = null; /* 081 */ funcResult_0 = value_3.deserialize(mutableStateArray_0[0]); /* 082 */ /* 083 */ if (funcResult_0 != null) { /* 084 */ value_2 = (org.apache.spark.ml.linalg.Vector) funcResult_0; /* 085 */ } else { /* 086 */ isNull_2 = true; /* 087 */ } /* 088 */ /* 089 */ /* 090 */ } /* 091 */ } /* 092 */ if (isNull_2) { /* 093 */ values_0[1] = null; /* 094 */ } else { /* 095 */ values_0[1] = value_2; /* 096 */ } /* 097 */ /* 098 */ } /* 099 */ /* 100 */ } 21/01/17 17:43:07.502 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_19_6 for BlockManagerId(driver, 192.168.1.225, 40985, None) 21/01/17 17:43:07.502 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_19_6 in memory on 192.168.1.225:40985 (size: 16.0 B, free: 366.2 MiB) 21/01/17 17:43:07.504 Executor task launch worker for task 7.0 in stage 3.0 (TID 16) INFO Executor: Finished task 7.0 in stage 3.0 (TID 16). 2170 bytes result sent to driver 21/01/17 17:43:07.504 Executor task launch worker for task 7.0 in stage 3.0 (TID 16) DEBUG ExecutorMetricsPoller: stageTCMP: (3, 0) -> 3 21/01/17 17:43:07.504 dispatcher-event-loop-1 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_3.0, runningTasks: 3 21/01/17 17:43:07.504 Executor task launch worker for task 5.0 in stage 3.0 (TID 14) INFO Executor: Finished task 5.0 in stage 3.0 (TID 14). 2170 bytes result sent to driver 21/01/17 17:43:07.505 dispatcher-event-loop-1 DEBUG TaskSetManager: No tasks for locality level NODE_LOCAL, so moving to locality level NO_PREF 21/01/17 17:43:07.505 dispatcher-event-loop-1 DEBUG TaskSetManager: Moving to ANY after waiting for 0ms 21/01/17 17:43:07.505 Executor task launch worker for task 5.0 in stage 3.0 (TID 14) DEBUG ExecutorMetricsPoller: stageTCMP: (3, 0) -> 2 21/01/17 17:43:07.505 dispatcher-event-loop-1 INFO TaskSetManager: Starting task 9.0 in stage 3.0 (TID 18) (192.168.1.225, executor driver, partition 9, PROCESS_LOCAL, 7143 bytes) taskResourceAssignments Map() 21/01/17 17:43:07.506 dispatcher-event-loop-1 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_3.0, runningTasks: 3 21/01/17 17:43:07.506 dispatcher-event-loop-1 DEBUG TaskSetManager: No tasks for locality level NODE_LOCAL, so moving to locality level NO_PREF 21/01/17 17:43:07.506 dispatcher-event-loop-1 DEBUG TaskSetManager: No tasks for locality level NO_PREF, so moving to locality level ANY 21/01/17 17:43:07.507 Executor task launch worker for task 9.0 in stage 3.0 (TID 18) INFO Executor: Running task 9.0 in stage 3.0 (TID 18) 21/01/17 17:43:07.507 task-result-getter-2 INFO TaskSetManager: Finished task 7.0 in stage 3.0 (TID 16) in 49 ms on 192.168.1.225 (executor driver) (6/10) 21/01/17 17:43:07.507 Executor task launch worker for task 9.0 in stage 3.0 (TID 18) DEBUG ExecutorMetricsPoller: stageTCMP: (3, 0) -> 3 21/01/17 17:43:07.507 task-result-getter-0 INFO TaskSetManager: Finished task 5.0 in stage 3.0 (TID 14) in 66 ms on 192.168.1.225 (executor driver) (7/10) 21/01/17 17:43:07.508 Executor task launch worker for task 8.0 in stage 3.0 (TID 17) DEBUG ExecutorMetricsPoller: stageTCMP: (3, 0) -> 2 21/01/17 17:43:07.508 task-result-getter-3 INFO TaskSetManager: Finished task 8.0 in stage 3.0 (TID 17) in 41 ms on 192.168.1.225 (executor driver) (8/10) 21/01/17 17:43:07.509 Executor task launch worker for task 6.0 in stage 3.0 (TID 15) DEBUG BlockManagerMaster: Updated info of block rdd_19_6 21/01/17 17:43:07.509 Executor task launch worker for task 6.0 in stage 3.0 (TID 15) DEBUG BlockManager: Told master about block rdd_19_6 21/01/17 17:43:07.509 Executor task launch worker for task 6.0 in stage 3.0 (TID 15) DEBUG BlockManager: Put block rdd_19_6 locally took 10 ms 21/01/17 17:43:07.509 Executor task launch worker for task 6.0 in stage 3.0 (TID 15) DEBUG BlockManager: Putting block rdd_19_6 without replication took 10 ms 21/01/17 17:43:07.509 Executor task launch worker for task 6.0 in stage 3.0 (TID 15) DEBUG BlockManager: Getting local block rdd_19_6 21/01/17 17:43:07.509 Executor task launch worker for task 6.0 in stage 3.0 (TID 15) DEBUG BlockManager: Level for block rdd_19_6 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:07.510 Executor task launch worker for task 6.0 in stage 3.0 (TID 15) INFO MemoryStore: Block rdd_20_6 stored as values in memory (estimated size 16.0 B, free 366.1 MiB) 21/01/17 17:43:07.510 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_20_6 for BlockManagerId(driver, 192.168.1.225, 40985, None) 21/01/17 17:43:07.511 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_20_6 in memory on 192.168.1.225:40985 (size: 16.0 B, free: 366.2 MiB) 21/01/17 17:43:07.517 Executor task launch worker for task 6.0 in stage 3.0 (TID 15) DEBUG BlockManagerMaster: Updated info of block rdd_20_6 21/01/17 17:43:07.517 Executor task launch worker for task 6.0 in stage 3.0 (TID 15) DEBUG BlockManager: Told master about block rdd_20_6 21/01/17 17:43:07.517 Executor task launch worker for task 6.0 in stage 3.0 (TID 15) DEBUG BlockManager: Put block rdd_20_6 locally took 27 ms 21/01/17 17:43:07.517 Executor task launch worker for task 6.0 in stage 3.0 (TID 15) DEBUG BlockManager: Putting block rdd_20_6 without replication took 28 ms 21/01/17 17:43:07.517 Executor task launch worker for task 6.0 in stage 3.0 (TID 15) DEBUG BlockManager: Getting local block rdd_20_6 21/01/17 17:43:07.517 Executor task launch worker for task 6.0 in stage 3.0 (TID 15) DEBUG BlockManager: Level for block rdd_20_6 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:07.518 Executor task launch worker for task 9.0 in stage 3.0 (TID 18) DEBUG BlockManager: Getting local block rdd_20_9 21/01/17 17:43:07.519 Executor task launch worker for task 6.0 in stage 3.0 (TID 15) INFO Executor: Finished task 6.0 in stage 3.0 (TID 15). 2170 bytes result sent to driver 21/01/17 17:43:07.521 Executor task launch worker for task 9.0 in stage 3.0 (TID 18) DEBUG BlockManager: Block rdd_20_9 was not found 21/01/17 17:43:07.521 Executor task launch worker for task 9.0 in stage 3.0 (TID 18) DEBUG BlockManager: Getting remote block rdd_20_9 21/01/17 17:43:07.522 Executor task launch worker for task 6.0 in stage 3.0 (TID 15) DEBUG ExecutorMetricsPoller: stageTCMP: (3, 0) -> 1 21/01/17 17:43:07.522 task-result-getter-1 INFO TaskSetManager: Finished task 6.0 in stage 3.0 (TID 15) in 68 ms on 192.168.1.225 (executor driver) (9/10) 21/01/17 17:43:07.503 Executor task launch worker for task 1.0 in stage 1.0 (TID 2) DEBUG CodeGenerator: /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificSafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificSafeProjection extends org.apache.spark.sql.catalyst.expressions.codegen.BaseProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private InternalRow mutableRow; /* 009 */ private boolean resultIsNull_0; /* 010 */ private InternalRow[] mutableStateArray_0 = new InternalRow[1]; /* 011 */ /* 012 */ public SpecificSafeProjection(Object[] references) { /* 013 */ this.references = references; /* 014 */ mutableRow = (InternalRow) references[references.length - 1]; /* 015 */ /* 016 */ /* 017 */ } /* 018 */ /* 019 */ public void initialize(int partitionIndex) { /* 020 */ /* 021 */ } /* 022 */ /* 023 */ public java.lang.Object apply(java.lang.Object _i) { /* 024 */ InternalRow i = (InternalRow) _i; /* 025 */ Object[] values_0 = new Object[3]; /* 026 */ createExternalRow_0_0(i, values_0); /* 027 */ createExternalRow_0_1(i, values_0); /* 028 */ final org.apache.spark.sql.Row value_0 = new org.apache.spark.sql.catalyst.expressions.GenericRowWithSchema(values_0, ((org.apache.spark.sql.types.StructType) references[0] /* schema */)); /* 029 */ if (false) { /* 030 */ mutableRow.setNullAt(0); /* 031 */ } else { /* 032 */ /* 033 */ mutableRow.update(0, value_0); /* 034 */ } /* 035 */ /* 036 */ return mutableRow; /* 037 */ } /* 038 */ /* 039 */ /* 040 */ private void createExternalRow_0_1(InternalRow i, Object[] values_0) { /* 041 */ /* 042 */ double value_5 = i.getDouble(2); /* 043 */ if (false) { /* 044 */ values_0[2] = null; /* 045 */ } else { /* 046 */ values_0[2] = value_5; /* 047 */ } /* 048 */ /* 049 */ } /* 050 */ /* 051 */ /* 052 */ private void createExternalRow_0_0(InternalRow i, Object[] values_0) { /* 053 */ /* 054 */ boolean isNull_1 = i.isNullAt(0); /* 055 */ double value_1 = isNull_1 ? /* 056 */ -1.0 : (i.getDouble(0)); /* 057 */ if (isNull_1) { /* 058 */ values_0[0] = null; /* 059 */ } else { /* 060 */ values_0[0] = value_1; /* 061 */ } /* 062 */ /* 063 */ final org.apache.spark.ml.linalg.VectorUDT value_3 = false ? /* 064 */ null : new org.apache.spark.ml.linalg.VectorUDT(); /* 065 */ boolean isNull_2 = true; /* 066 */ org.apache.spark.ml.linalg.Vector value_2 = null; /* 067 */ if (!false) { /* 068 */ resultIsNull_0 = false; /* 069 */ if (!resultIsNull_0) { /* 070 */ boolean isNull_4 = i.isNullAt(1); /* 071 */ InternalRow value_4 = isNull_4 ? /* 072 */ null : (i.getStruct(1, 4)); /* 073 */ resultIsNull_0 = isNull_4; /* 074 */ mutableStateArray_0[0] = value_4; /* 075 */ } /* 076 */ /* 077 */ isNull_2 = resultIsNull_0; /* 078 */ if (!isNull_2) { /* 079 */ /* 080 */ Object funcResult_0 = null; /* 081 */ funcResult_0 = value_3.deserialize(mutableStateArray_0[0]); /* 082 */ /* 083 */ if (funcResult_0 != null) { /* 084 */ value_2 = (org.apache.spark.ml.linalg.Vector) funcResult_0; /* 085 */ } else { /* 086 */ isNull_2 = true; /* 087 */ } /* 088 */ /* 089 */ /* 090 */ } /* 091 */ } /* 092 */ if (isNull_2) { /* 093 */ values_0[1] = null; /* 094 */ } else { /* 095 */ values_0[1] = value_2; /* 096 */ } /* 097 */ /* 098 */ } /* 099 */ /* 100 */ } 21/01/17 17:43:07.523 Executor task launch worker for task 9.0 in stage 3.0 (TID 18) DEBUG BlockManager: Block rdd_20_9 is unknown by block manager master 21/01/17 17:43:07.523 Executor task launch worker for task 9.0 in stage 3.0 (TID 18) DEBUG BlockManager: Getting local block rdd_19_9 21/01/17 17:43:07.523 Executor task launch worker for task 9.0 in stage 3.0 (TID 18) DEBUG BlockManager: Block rdd_19_9 was not found 21/01/17 17:43:07.523 Executor task launch worker for task 9.0 in stage 3.0 (TID 18) DEBUG BlockManager: Getting remote block rdd_19_9 21/01/17 17:43:07.524 Executor task launch worker for task 9.0 in stage 3.0 (TID 18) DEBUG BlockManager: Block rdd_19_9 is unknown by block manager master 21/01/17 17:43:07.524 Executor task launch worker for task 9.0 in stage 3.0 (TID 18) DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 0 21/01/17 17:43:07.524 Executor task launch worker for task 9.0 in stage 3.0 (TID 18) DEBUG MapOutputTrackerMaster: Convert map statuses for shuffle 0, mappers 0-4, partitions 9-10 21/01/17 17:43:07.524 Executor task launch worker for task 9.0 in stage 3.0 (TID 18) DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 21/01/17 17:43:07.525 Executor task launch worker for task 9.0 in stage 3.0 (TID 18) INFO ShuffleBlockFetcherIterator: Getting 0 (0.0 B) non-empty blocks including 0 (0.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 21/01/17 17:43:07.525 Executor task launch worker for task 9.0 in stage 3.0 (TID 18) INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 21/01/17 17:43:07.525 Executor task launch worker for task 9.0 in stage 3.0 (TID 18) DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: 21/01/17 17:43:07.525 Executor task launch worker for task 9.0 in stage 3.0 (TID 18) DEBUG ShuffleBlockFetcherIterator: Got local blocks in 0 ms 21/01/17 17:43:07.526 Executor task launch worker for task 9.0 in stage 3.0 (TID 18) DEBUG TaskMemoryManager: Task 18 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@54e281ae 21/01/17 17:43:07.526 Executor task launch worker for task 9.0 in stage 3.0 (TID 18) INFO MemoryStore: Block rdd_19_9 stored as values in memory (estimated size 16.0 B, free 366.1 MiB) 21/01/17 17:43:07.526 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_19_9 for BlockManagerId(driver, 192.168.1.225, 40985, None) 21/01/17 17:43:07.527 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_19_9 in memory on 192.168.1.225:40985 (size: 16.0 B, free: 366.2 MiB) 21/01/17 17:43:07.527 Executor task launch worker for task 9.0 in stage 3.0 (TID 18) DEBUG BlockManagerMaster: Updated info of block rdd_19_9 21/01/17 17:43:07.527 Executor task launch worker for task 9.0 in stage 3.0 (TID 18) DEBUG BlockManager: Told master about block rdd_19_9 21/01/17 17:43:07.527 Executor task launch worker for task 9.0 in stage 3.0 (TID 18) DEBUG BlockManager: Put block rdd_19_9 locally took 3 ms 21/01/17 17:43:07.528 Executor task launch worker for task 9.0 in stage 3.0 (TID 18) DEBUG BlockManager: Putting block rdd_19_9 without replication took 3 ms 21/01/17 17:43:07.528 Executor task launch worker for task 9.0 in stage 3.0 (TID 18) DEBUG BlockManager: Getting local block rdd_19_9 21/01/17 17:43:07.528 Executor task launch worker for task 9.0 in stage 3.0 (TID 18) DEBUG BlockManager: Level for block rdd_19_9 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:07.528 Executor task launch worker for task 9.0 in stage 3.0 (TID 18) INFO MemoryStore: Block rdd_20_9 stored as values in memory (estimated size 16.0 B, free 366.1 MiB) 21/01/17 17:43:07.529 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_20_9 for BlockManagerId(driver, 192.168.1.225, 40985, None) 21/01/17 17:43:07.529 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_20_9 in memory on 192.168.1.225:40985 (size: 16.0 B, free: 366.2 MiB) 21/01/17 17:43:07.530 Executor task launch worker for task 9.0 in stage 3.0 (TID 18) DEBUG BlockManagerMaster: Updated info of block rdd_20_9 21/01/17 17:43:07.530 Executor task launch worker for task 9.0 in stage 3.0 (TID 18) DEBUG BlockManager: Told master about block rdd_20_9 21/01/17 17:43:07.530 Executor task launch worker for task 9.0 in stage 3.0 (TID 18) DEBUG BlockManager: Put block rdd_20_9 locally took 6 ms 21/01/17 17:43:07.530 Executor task launch worker for task 9.0 in stage 3.0 (TID 18) DEBUG BlockManager: Putting block rdd_20_9 without replication took 6 ms 21/01/17 17:43:07.530 Executor task launch worker for task 9.0 in stage 3.0 (TID 18) DEBUG BlockManager: Getting local block rdd_20_9 21/01/17 17:43:07.530 Executor task launch worker for task 9.0 in stage 3.0 (TID 18) DEBUG BlockManager: Level for block rdd_20_9 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:07.531 Executor task launch worker for task 9.0 in stage 3.0 (TID 18) INFO Executor: Finished task 9.0 in stage 3.0 (TID 18). 2170 bytes result sent to driver 21/01/17 17:43:07.532 Executor task launch worker for task 9.0 in stage 3.0 (TID 18) DEBUG ExecutorMetricsPoller: removing (3, 0) from stageTCMP 21/01/17 17:43:07.532 task-result-getter-2 INFO TaskSetManager: Finished task 9.0 in stage 3.0 (TID 18) in 27 ms on 192.168.1.225 (executor driver) (10/10) 21/01/17 17:43:07.533 task-result-getter-2 INFO TaskSchedulerImpl: Removed TaskSet 3.0, whose tasks have all completed, from pool 21/01/17 17:43:07.534 dag-scheduler-event-loop INFO DAGScheduler: ResultStage 3 (count at ALS.scala:973) finished in 0.203 s 21/01/17 17:43:07.534 dag-scheduler-event-loop DEBUG DAGScheduler: After removal of stage 2, remaining stages = 2 21/01/17 17:43:07.534 dag-scheduler-event-loop DEBUG DAGScheduler: After removal of stage 1, remaining stages = 1 21/01/17 17:43:07.534 dag-scheduler-event-loop DEBUG DAGScheduler: After removal of stage 3, remaining stages = 0 21/01/17 17:43:07.534 dag-scheduler-event-loop INFO DAGScheduler: Job 1 is finished. Cancelling potential speculative or zombie tasks for this job 21/01/17 17:43:07.534 dag-scheduler-event-loop INFO TaskSchedulerImpl: Killing all running tasks in stage 3: Stage finished 21/01/17 17:43:07.537 Executor task launch worker for task 0.0 in stage 1.0 (TID 4) INFO PythonRunner: Times: total = 85, boot = -739, init = 824, finish = 0 21/01/17 17:43:07.544 Thread-3 INFO DAGScheduler: Job 1 finished: count at ALS.scala:973, took 0.880850 s 21/01/17 17:43:07.546 Thread-3 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$train$3 21/01/17 17:43:07.548 Executor task launch worker for task 0.0 in stage 1.0 (TID 4) INFO Executor: Finished task 0.0 in stage 1.0 (TID 4). 1999 bytes result sent to driver 21/01/17 17:43:07.548 Executor task launch worker for task 0.0 in stage 1.0 (TID 4) DEBUG ExecutorMetricsPoller: stageTCMP: (1, 0) -> 3 21/01/17 17:43:07.549 Thread-3 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$train$3) is now cleaned +++ 21/01/17 17:43:07.549 dispatcher-event-loop-1 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_1.0, runningTasks: 3 21/01/17 17:43:07.549 dispatcher-event-loop-1 DEBUG TaskSetManager: No tasks for locality level NO_PREF, so moving to locality level ANY 21/01/17 17:43:07.550 task-result-getter-0 INFO TaskSetManager: Finished task 0.0 in stage 1.0 (TID 4) in 123 ms on 192.168.1.225 (executor driver) (1/4) 21/01/17 17:43:07.550 Thread-3 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$makeBlocks$1 21/01/17 17:43:07.553 Thread-3 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$makeBlocks$1) is now cleaned +++ 21/01/17 17:43:07.554 Thread-3 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$groupByKey$2 21/01/17 17:43:07.553 Executor task launch worker for task 3.0 in stage 1.0 (TID 7) DEBUG GenerateSafeProjection: code for mapobjects(lambdavariable(MapObject, StringType, true, -1), lambdavariable(MapObject, StringType, true, -1).toString, input[0, array, true], Some(interface scala.collection.Seq)): /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificSafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificSafeProjection extends org.apache.spark.sql.catalyst.expressions.codegen.BaseProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private InternalRow mutableRow; /* 009 */ private UTF8String value_MapObject_lambda_variable_1; /* 010 */ private boolean isNull_MapObject_lambda_variable_1; /* 011 */ private boolean globalIsNull_0; /* 012 */ /* 013 */ public SpecificSafeProjection(Object[] references) { /* 014 */ this.references = references; /* 015 */ mutableRow = (InternalRow) references[references.length - 1]; /* 016 */ /* 017 */ /* 018 */ } /* 019 */ /* 020 */ public void initialize(int partitionIndex) { /* 021 */ /* 022 */ } /* 023 */ /* 024 */ public java.lang.Object apply(java.lang.Object _i) { /* 025 */ InternalRow i = (InternalRow) _i; /* 026 */ scala.collection.Seq value_3 = MapObjects_0(i); /* 027 */ if (globalIsNull_0) { /* 028 */ mutableRow.setNullAt(0); /* 029 */ } else { /* 030 */ /* 031 */ mutableRow.update(0, value_3); /* 032 */ } /* 033 */ /* 034 */ return mutableRow; /* 035 */ } /* 036 */ /* 037 */ /* 038 */ private scala.collection.Seq MapObjects_0(InternalRow i) { /* 039 */ boolean isNull_1 = i.isNullAt(0); /* 040 */ ArrayData value_1 = isNull_1 ? /* 041 */ null : (i.getArray(0)); /* 042 */ scala.collection.Seq value_0 = null; /* 043 */ /* 044 */ if (!isNull_1) { /* 045 */ /* 046 */ int dataLength_0 = value_1.numElements(); /* 047 */ /* 048 */ scala.collection.mutable.Builder collectionBuilder_0 = scala.collection.Seq$.MODULE$.newBuilder(); /* 049 */ collectionBuilder_0.sizeHint(dataLength_0); /* 050 */ /* 051 */ /* 052 */ int loopIndex_0 = 0; /* 053 */ /* 054 */ while (loopIndex_0 < dataLength_0) { /* 055 */ value_MapObject_lambda_variable_1 = (UTF8String) (value_1.getUTF8String(loopIndex_0)); /* 056 */ isNull_MapObject_lambda_variable_1 = value_1.isNullAt(loopIndex_0); /* 057 */ /* 058 */ boolean isNull_2 = true; /* 059 */ java.lang.String value_2 = null; /* 060 */ if (!isNull_MapObject_lambda_variable_1) { /* 061 */ /* 062 */ isNull_2 = false; /* 063 */ if (!isNull_2) { /* 064 */ /* 065 */ Object funcResult_0 = null; /* 066 */ funcResult_0 = value_MapObject_lambda_variable_1.toString(); /* 067 */ value_2 = (java.lang.String) funcResult_0; /* 068 */ /* 069 */ } /* 070 */ } /* 071 */ if (isNull_2) { /* 072 */ collectionBuilder_0.$plus$eq(null); /* 073 */ } else { /* 074 */ collectionBuilder_0.$plus$eq(value_2); /* 075 */ } /* 076 */ /* 077 */ loopIndex_0 += 1; /* 078 */ } /* 079 */ /* 080 */ value_0 = (scala.collection.Seq) collectionBuilder_0.result(); /* 081 */ } /* 082 */ globalIsNull_0 = isNull_1; /* 083 */ return value_0; /* 084 */ } /* 085 */ /* 086 */ } 21/01/17 17:43:07.557 Thread-3 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$groupByKey$2) is now cleaned +++ 21/01/17 17:43:07.557 Executor task launch worker for task 3.0 in stage 1.0 (TID 7) DEBUG BlockManager: Getting local block broadcast_2 21/01/17 17:43:07.558 Executor task launch worker for task 3.0 in stage 1.0 (TID 7) DEBUG BlockManager: Level for block broadcast_2 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:07.559 Thread-3 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$groupByKey$3 21/01/17 17:43:07.568 Thread-3 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$groupByKey$3) is now cleaned +++ 21/01/17 17:43:07.568 Thread-3 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$groupByKey$4 21/01/17 17:43:07.573 Thread-3 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$groupByKey$4) is now cleaned +++ 21/01/17 17:43:07.574 Thread-3 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$makeBlocks$5 21/01/17 17:43:07.576 Thread-3 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$makeBlocks$5) is now cleaned +++ 21/01/17 17:43:07.578 Thread-3 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$makeBlocks$7 21/01/17 17:43:07.580 Thread-3 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$makeBlocks$7) is now cleaned +++ 21/01/17 17:43:07.581 Thread-3 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$count$1$adapted 21/01/17 17:43:07.564 Executor task launch worker for task 1.0 in stage 1.0 (TID 5) DEBUG GenerateSafeProjection: code for mapobjects(lambdavariable(MapObject, StringType, true, -1), lambdavariable(MapObject, StringType, true, -1).toString, input[0, array, true], Some(interface scala.collection.Seq)): /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificSafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificSafeProjection extends org.apache.spark.sql.catalyst.expressions.codegen.BaseProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private InternalRow mutableRow; /* 009 */ private UTF8String value_MapObject_lambda_variable_1; /* 010 */ private boolean isNull_MapObject_lambda_variable_1; /* 011 */ private boolean globalIsNull_0; /* 012 */ /* 013 */ public SpecificSafeProjection(Object[] references) { /* 014 */ this.references = references; /* 015 */ mutableRow = (InternalRow) references[references.length - 1]; /* 016 */ /* 017 */ /* 018 */ } /* 019 */ /* 020 */ public void initialize(int partitionIndex) { /* 021 */ /* 022 */ } /* 023 */ /* 024 */ public java.lang.Object apply(java.lang.Object _i) { /* 025 */ InternalRow i = (InternalRow) _i; /* 026 */ scala.collection.Seq value_3 = MapObjects_0(i); /* 027 */ if (globalIsNull_0) { /* 028 */ mutableRow.setNullAt(0); /* 029 */ } else { /* 030 */ /* 031 */ mutableRow.update(0, value_3); /* 032 */ } /* 033 */ /* 034 */ return mutableRow; /* 035 */ } /* 036 */ /* 037 */ /* 038 */ private scala.collection.Seq MapObjects_0(InternalRow i) { /* 039 */ boolean isNull_1 = i.isNullAt(0); /* 040 */ ArrayData value_1 = isNull_1 ? /* 041 */ null : (i.getArray(0)); /* 042 */ scala.collection.Seq value_0 = null; /* 043 */ /* 044 */ if (!isNull_1) { /* 045 */ /* 046 */ int dataLength_0 = value_1.numElements(); /* 047 */ /* 048 */ scala.collection.mutable.Builder collectionBuilder_0 = scala.collection.Seq$.MODULE$.newBuilder(); /* 049 */ collectionBuilder_0.sizeHint(dataLength_0); /* 050 */ /* 051 */ /* 052 */ int loopIndex_0 = 0; /* 053 */ /* 054 */ while (loopIndex_0 < dataLength_0) { /* 055 */ value_MapObject_lambda_variable_1 = (UTF8String) (value_1.getUTF8String(loopIndex_0)); /* 056 */ isNull_MapObject_lambda_variable_1 = value_1.isNullAt(loopIndex_0); /* 057 */ /* 058 */ boolean isNull_2 = true; /* 059 */ java.lang.String value_2 = null; /* 060 */ if (!isNull_MapObject_lambda_variable_1) { /* 061 */ /* 062 */ isNull_2 = false; /* 063 */ if (!isNull_2) { /* 064 */ /* 065 */ Object funcResult_0 = null; /* 066 */ funcResult_0 = value_MapObject_lambda_variable_1.toString(); /* 067 */ value_2 = (java.lang.String) funcResult_0; /* 068 */ /* 069 */ } /* 070 */ } /* 071 */ if (isNull_2) { /* 072 */ collectionBuilder_0.$plus$eq(null); /* 073 */ } else { /* 074 */ collectionBuilder_0.$plus$eq(value_2); /* 075 */ } /* 076 */ /* 077 */ loopIndex_0 += 1; /* 078 */ } /* 079 */ /* 080 */ value_0 = (scala.collection.Seq) collectionBuilder_0.result(); /* 081 */ } /* 082 */ globalIsNull_0 = isNull_1; /* 083 */ return value_0; /* 084 */ } /* 085 */ /* 086 */ } 21/01/17 17:43:07.578 Executor task launch worker for task 3.0 in stage 1.0 (TID 7) DEBUG GenerateUnsafeProjection: code for newInstance(class org.apache.spark.ml.linalg.VectorUDT).serialize: /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificUnsafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificUnsafeProjection extends org.apache.spark.sql.catalyst.expressions.UnsafeProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private boolean resultIsNull_0; /* 009 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[] mutableStateArray_1 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[2]; /* 010 */ private org.apache.spark.ml.linalg.Vector[] mutableStateArray_0 = new org.apache.spark.ml.linalg.Vector[1]; /* 011 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[] mutableStateArray_2 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[2]; /* 012 */ /* 013 */ public SpecificUnsafeProjection(Object[] references) { /* 014 */ this.references = references; /* 015 */ /* 016 */ mutableStateArray_1[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(1, 32); /* 017 */ mutableStateArray_1[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(mutableStateArray_1[0], 4); /* 018 */ mutableStateArray_2[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(mutableStateArray_1[1], 4); /* 019 */ mutableStateArray_2[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(mutableStateArray_1[1], 8); /* 020 */ /* 021 */ } /* 022 */ /* 023 */ public void initialize(int partitionIndex) { /* 024 */ /* 025 */ } /* 026 */ /* 027 */ // Scala.Function1 need this /* 028 */ public java.lang.Object apply(java.lang.Object row) { /* 029 */ return apply((InternalRow) row); /* 030 */ } /* 031 */ /* 032 */ public UnsafeRow apply(InternalRow i) { /* 033 */ mutableStateArray_1[0].reset(); /* 034 */ /* 035 */ /* 036 */ mutableStateArray_1[0].zeroOutNullBytes(); /* 037 */ /* 038 */ final org.apache.spark.ml.linalg.VectorUDT value_1 = false ? /* 039 */ null : new org.apache.spark.ml.linalg.VectorUDT(); /* 040 */ boolean isNull_0 = true; /* 041 */ InternalRow value_0 = null; /* 042 */ if (!false) { /* 043 */ resultIsNull_0 = false; /* 044 */ if (!resultIsNull_0) { /* 045 */ boolean isNull_2 = i.isNullAt(0); /* 046 */ org.apache.spark.ml.linalg.Vector value_2 = isNull_2 ? /* 047 */ null : ((org.apache.spark.ml.linalg.Vector)i.get(0, null)); /* 048 */ resultIsNull_0 = isNull_2; /* 049 */ mutableStateArray_0[0] = value_2; /* 050 */ } /* 051 */ /* 052 */ isNull_0 = resultIsNull_0; /* 053 */ if (!isNull_0) { /* 054 */ /* 055 */ Object funcResult_0 = null; /* 056 */ funcResult_0 = value_1.serialize(mutableStateArray_0[0]); /* 057 */ /* 058 */ if (funcResult_0 != null) { /* 059 */ value_0 = (InternalRow) funcResult_0; /* 060 */ } else { /* 061 */ isNull_0 = true; /* 062 */ } /* 063 */ /* 064 */ /* 065 */ } /* 066 */ } /* 067 */ if (isNull_0) { /* 068 */ mutableStateArray_1[0].setNullAt(0); /* 069 */ } else { /* 070 */ final InternalRow tmpInput_0 = value_0; /* 071 */ if (tmpInput_0 instanceof UnsafeRow) { /* 072 */ mutableStateArray_1[0].write(0, (UnsafeRow) tmpInput_0); /* 073 */ } else { /* 074 */ // Remember the current cursor so that we can calculate how many bytes are /* 075 */ // written later. /* 076 */ final int previousCursor_0 = mutableStateArray_1[0].cursor(); /* 077 */ /* 078 */ mutableStateArray_1[1].resetRowWriter(); /* 079 */ /* 080 */ /* 081 */ mutableStateArray_1[1].write(0, (tmpInput_0.getByte(0))); /* 082 */ /* 083 */ /* 084 */ if ((tmpInput_0.isNullAt(1))) { /* 085 */ mutableStateArray_1[1].setNullAt(1); /* 086 */ } else { /* 087 */ mutableStateArray_1[1].write(1, (tmpInput_0.getInt(1))); /* 088 */ } /* 089 */ /* 090 */ /* 091 */ if ((tmpInput_0.isNullAt(2))) { /* 092 */ mutableStateArray_1[1].setNullAt(2); /* 093 */ } else { /* 094 */ // Remember the current cursor so that we can calculate how many bytes are /* 095 */ // written later. /* 096 */ final int previousCursor_1 = mutableStateArray_1[1].cursor(); /* 097 */ /* 098 */ final ArrayData tmpInput_1 = (tmpInput_0.getArray(2)); /* 099 */ if (tmpInput_1 instanceof UnsafeArrayData) { /* 100 */ mutableStateArray_1[1].write((UnsafeArrayData) tmpInput_1); /* 101 */ } else { /* 102 */ final int numElements_0 = tmpInput_1.numElements(); /* 103 */ mutableStateArray_2[0].initialize(numElements_0); /* 104 */ /* 105 */ for (int index_0 = 0; index_0 < numElements_0; index_0++) { /* 106 */ mutableStateArray_2[0].write(index_0, tmpInput_1.getInt(index_0)); /* 107 */ } /* 108 */ } /* 109 */ /* 110 */ mutableStateArray_1[1].setOffsetAndSizeFromPreviousCursor(2, previousCursor_1); /* 111 */ } /* 112 */ /* 113 */ /* 114 */ if ((tmpInput_0.isNullAt(3))) { /* 115 */ mutableStateArray_1[1].setNullAt(3); /* 116 */ } else { /* 117 */ // Remember the current cursor so that we can calculate how many bytes are /* 118 */ // written later. /* 119 */ final int previousCursor_2 = mutableStateArray_1[1].cursor(); /* 120 */ /* 121 */ final ArrayData tmpInput_2 = (tmpInput_0.getArray(3)); /* 122 */ if (tmpInput_2 instanceof UnsafeArrayData) { /* 123 */ mutableStateArray_1[1].write((UnsafeArrayData) tmpInput_2); /* 124 */ } else { /* 125 */ final int numElements_1 = tmpInput_2.numElements(); /* 126 */ mutableStateArray_2[1].initialize(numElements_1); /* 127 */ /* 128 */ for (int index_1 = 0; index_1 < numElements_1; index_1++) { /* 129 */ mutableStateArray_2[1].write(index_1, tmpInput_2.getDouble(index_1)); /* 130 */ } /* 131 */ } /* 132 */ /* 133 */ mutableStateArray_1[1].setOffsetAndSizeFromPreviousCursor(3, previousCursor_2); /* 134 */ } /* 135 */ /* 136 */ /* 137 */ mutableStateArray_1[0].setOffsetAndSizeFromPreviousCursor(0, previousCursor_0); /* 138 */ } /* 139 */ } /* 140 */ return (mutableStateArray_1[0].getRow()); /* 141 */ } /* 142 */ /* 143 */ /* 144 */ } 21/01/17 17:43:07.577 Executor task launch worker for task 2.0 in stage 1.0 (TID 6) DEBUG GenerateSafeProjection: code for mapobjects(lambdavariable(MapObject, StringType, true, -1), lambdavariable(MapObject, StringType, true, -1).toString, input[0, array, true], Some(interface scala.collection.Seq)): /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificSafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificSafeProjection extends org.apache.spark.sql.catalyst.expressions.codegen.BaseProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private InternalRow mutableRow; /* 009 */ private UTF8String value_MapObject_lambda_variable_1; /* 010 */ private boolean isNull_MapObject_lambda_variable_1; /* 011 */ private boolean globalIsNull_0; /* 012 */ /* 013 */ public SpecificSafeProjection(Object[] references) { /* 014 */ this.references = references; /* 015 */ mutableRow = (InternalRow) references[references.length - 1]; /* 016 */ /* 017 */ /* 018 */ } /* 019 */ /* 020 */ public void initialize(int partitionIndex) { /* 021 */ /* 022 */ } /* 023 */ /* 024 */ public java.lang.Object apply(java.lang.Object _i) { /* 025 */ InternalRow i = (InternalRow) _i; /* 026 */ scala.collection.Seq value_3 = MapObjects_0(i); /* 027 */ if (globalIsNull_0) { /* 028 */ mutableRow.setNullAt(0); /* 029 */ } else { /* 030 */ /* 031 */ mutableRow.update(0, value_3); /* 032 */ } /* 033 */ /* 034 */ return mutableRow; /* 035 */ } /* 036 */ /* 037 */ /* 038 */ private scala.collection.Seq MapObjects_0(InternalRow i) { /* 039 */ boolean isNull_1 = i.isNullAt(0); /* 040 */ ArrayData value_1 = isNull_1 ? /* 041 */ null : (i.getArray(0)); /* 042 */ scala.collection.Seq value_0 = null; /* 043 */ /* 044 */ if (!isNull_1) { /* 045 */ /* 046 */ int dataLength_0 = value_1.numElements(); /* 047 */ /* 048 */ scala.collection.mutable.Builder collectionBuilder_0 = scala.collection.Seq$.MODULE$.newBuilder(); /* 049 */ collectionBuilder_0.sizeHint(dataLength_0); /* 050 */ /* 051 */ /* 052 */ int loopIndex_0 = 0; /* 053 */ /* 054 */ while (loopIndex_0 < dataLength_0) { /* 055 */ value_MapObject_lambda_variable_1 = (UTF8String) (value_1.getUTF8String(loopIndex_0)); /* 056 */ isNull_MapObject_lambda_variable_1 = value_1.isNullAt(loopIndex_0); /* 057 */ /* 058 */ boolean isNull_2 = true; /* 059 */ java.lang.String value_2 = null; /* 060 */ if (!isNull_MapObject_lambda_variable_1) { /* 061 */ /* 062 */ isNull_2 = false; /* 063 */ if (!isNull_2) { /* 064 */ /* 065 */ Object funcResult_0 = null; /* 066 */ funcResult_0 = value_MapObject_lambda_variable_1.toString(); /* 067 */ value_2 = (java.lang.String) funcResult_0; /* 068 */ /* 069 */ } /* 070 */ } /* 071 */ if (isNull_2) { /* 072 */ collectionBuilder_0.$plus$eq(null); /* 073 */ } else { /* 074 */ collectionBuilder_0.$plus$eq(value_2); /* 075 */ } /* 076 */ /* 077 */ loopIndex_0 += 1; /* 078 */ } /* 079 */ /* 080 */ value_0 = (scala.collection.Seq) collectionBuilder_0.result(); /* 081 */ } /* 082 */ globalIsNull_0 = isNull_1; /* 083 */ return value_0; /* 084 */ } /* 085 */ /* 086 */ } 21/01/17 17:43:07.586 Executor task launch worker for task 3.0 in stage 1.0 (TID 7) INFO PythonRunner: Times: total = 57, boot = -733, init = 790, finish = 0 21/01/17 17:43:07.566 Executor task launch worker for task 3.0 in stage 1.0 (TID 4) DEBUG GenerateSafeProjection: code for createexternalrow(input[0, double, true], newInstance(class org.apache.spark.ml.linalg.VectorUDT).deserialize, input[2, double, false], StructField(prediction,DoubleType,true), StructField(normalizedFeatures,org.apache.spark.ml.linalg.VectorUDT@3bfc3ba7,true), StructField(weightCol,DoubleType,false)): /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificSafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificSafeProjection extends org.apache.spark.sql.catalyst.expressions.codegen.BaseProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private InternalRow mutableRow; /* 009 */ private boolean resultIsNull_0; /* 010 */ private InternalRow[] mutableStateArray_0 = new InternalRow[1]; /* 011 */ /* 012 */ public SpecificSafeProjection(Object[] references) { /* 013 */ this.references = references; /* 014 */ mutableRow = (InternalRow) references[references.length - 1]; /* 015 */ /* 016 */ /* 017 */ } /* 018 */ /* 019 */ public void initialize(int partitionIndex) { /* 020 */ /* 021 */ } /* 022 */ /* 023 */ public java.lang.Object apply(java.lang.Object _i) { /* 024 */ InternalRow i = (InternalRow) _i; /* 025 */ Object[] values_0 = new Object[3]; /* 026 */ createExternalRow_0_0(i, values_0); /* 027 */ createExternalRow_0_1(i, values_0); /* 028 */ final org.apache.spark.sql.Row value_0 = new org.apache.spark.sql.catalyst.expressions.GenericRowWithSchema(values_0, ((org.apache.spark.sql.types.StructType) references[0] /* schema */)); /* 029 */ if (false) { /* 030 */ mutableRow.setNullAt(0); /* 031 */ } else { /* 032 */ /* 033 */ mutableRow.update(0, value_0); /* 034 */ } /* 035 */ /* 036 */ return mutableRow; /* 037 */ } /* 038 */ /* 039 */ /* 040 */ private void createExternalRow_0_1(InternalRow i, Object[] values_0) { /* 041 */ /* 042 */ double value_5 = i.getDouble(2); /* 043 */ if (false) { /* 044 */ values_0[2] = null; /* 045 */ } else { /* 046 */ values_0[2] = value_5; /* 047 */ } /* 048 */ /* 049 */ } /* 050 */ /* 051 */ /* 052 */ private void createExternalRow_0_0(InternalRow i, Object[] values_0) { /* 053 */ /* 054 */ boolean isNull_1 = i.isNullAt(0); /* 055 */ double value_1 = isNull_1 ? /* 056 */ -1.0 : (i.getDouble(0)); /* 057 */ if (isNull_1) { /* 058 */ values_0[0] = null; /* 059 */ } else { /* 060 */ values_0[0] = value_1; /* 061 */ } /* 062 */ /* 063 */ final org.apache.spark.ml.linalg.VectorUDT value_3 = false ? /* 064 */ null : new org.apache.spark.ml.linalg.VectorUDT(); /* 065 */ boolean isNull_2 = true; /* 066 */ org.apache.spark.ml.linalg.Vector value_2 = null; /* 067 */ if (!false) { /* 068 */ resultIsNull_0 = false; /* 069 */ if (!resultIsNull_0) { /* 070 */ boolean isNull_4 = i.isNullAt(1); /* 071 */ InternalRow value_4 = isNull_4 ? /* 072 */ null : (i.getStruct(1, 4)); /* 073 */ resultIsNull_0 = isNull_4; /* 074 */ mutableStateArray_0[0] = value_4; /* 075 */ } /* 076 */ /* 077 */ isNull_2 = resultIsNull_0; /* 078 */ if (!isNull_2) { /* 079 */ /* 080 */ Object funcResult_0 = null; /* 081 */ funcResult_0 = value_3.deserialize(mutableStateArray_0[0]); /* 082 */ /* 083 */ if (funcResult_0 != null) { /* 084 */ value_2 = (org.apache.spark.ml.linalg.Vector) funcResult_0; /* 085 */ } else { /* 086 */ isNull_2 = true; /* 087 */ } /* 088 */ /* 089 */ /* 090 */ } /* 091 */ } /* 092 */ if (isNull_2) { /* 093 */ values_0[1] = null; /* 094 */ } else { /* 095 */ values_0[1] = value_2; /* 096 */ } /* 097 */ /* 098 */ } /* 099 */ /* 100 */ } 21/01/17 17:43:07.590 Executor task launch worker for task 3.0 in stage 1.0 (TID 7) INFO Executor: Finished task 3.0 in stage 1.0 (TID 7). 2063 bytes result sent to driver 21/01/17 17:43:07.590 Executor task launch worker for task 3.0 in stage 1.0 (TID 7) DEBUG ExecutorMetricsPoller: stageTCMP: (1, 0) -> 2 21/01/17 17:43:07.591 task-result-getter-2 INFO TaskSetManager: Finished task 3.0 in stage 1.0 (TID 7) in 162 ms on 192.168.1.225 (executor driver) (2/4) 21/01/17 17:43:07.571 Executor task launch worker for task 0.0 in stage 1.0 (TID 1) DEBUG GenerateSafeProjection: code for createexternalrow(input[0, double, true], newInstance(class org.apache.spark.ml.linalg.VectorUDT).deserialize, input[2, double, false], StructField(prediction,DoubleType,true), StructField(normalizedFeatures,org.apache.spark.ml.linalg.VectorUDT@3bfc3ba7,true), StructField(weightCol,DoubleType,false)): /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificSafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificSafeProjection extends org.apache.spark.sql.catalyst.expressions.codegen.BaseProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private InternalRow mutableRow; /* 009 */ private boolean resultIsNull_0; /* 010 */ private InternalRow[] mutableStateArray_0 = new InternalRow[1]; /* 011 */ /* 012 */ public SpecificSafeProjection(Object[] references) { /* 013 */ this.references = references; /* 014 */ mutableRow = (InternalRow) references[references.length - 1]; /* 015 */ /* 016 */ /* 017 */ } /* 018 */ /* 019 */ public void initialize(int partitionIndex) { /* 020 */ /* 021 */ } /* 022 */ /* 023 */ public java.lang.Object apply(java.lang.Object _i) { /* 024 */ InternalRow i = (InternalRow) _i; /* 025 */ Object[] values_0 = new Object[3]; /* 026 */ createExternalRow_0_0(i, values_0); /* 027 */ createExternalRow_0_1(i, values_0); /* 028 */ final org.apache.spark.sql.Row value_0 = new org.apache.spark.sql.catalyst.expressions.GenericRowWithSchema(values_0, ((org.apache.spark.sql.types.StructType) references[0] /* schema */)); /* 029 */ if (false) { /* 030 */ mutableRow.setNullAt(0); /* 031 */ } else { /* 032 */ /* 033 */ mutableRow.update(0, value_0); /* 034 */ } /* 035 */ /* 036 */ return mutableRow; /* 037 */ } /* 038 */ /* 039 */ /* 040 */ private void createExternalRow_0_1(InternalRow i, Object[] values_0) { /* 041 */ /* 042 */ double value_5 = i.getDouble(2); /* 043 */ if (false) { /* 044 */ values_0[2] = null; /* 045 */ } else { /* 046 */ values_0[2] = value_5; /* 047 */ } /* 048 */ /* 049 */ } /* 050 */ /* 051 */ /* 052 */ private void createExternalRow_0_0(InternalRow i, Object[] values_0) { /* 053 */ /* 054 */ boolean isNull_1 = i.isNullAt(0); /* 055 */ double value_1 = isNull_1 ? /* 056 */ -1.0 : (i.getDouble(0)); /* 057 */ if (isNull_1) { /* 058 */ values_0[0] = null; /* 059 */ } else { /* 060 */ values_0[0] = value_1; /* 061 */ } /* 062 */ /* 063 */ final org.apache.spark.ml.linalg.VectorUDT value_3 = false ? /* 064 */ null : new org.apache.spark.ml.linalg.VectorUDT(); /* 065 */ boolean isNull_2 = true; /* 066 */ org.apache.spark.ml.linalg.Vector value_2 = null; /* 067 */ if (!false) { /* 068 */ resultIsNull_0 = false; /* 069 */ if (!resultIsNull_0) { /* 070 */ boolean isNull_4 = i.isNullAt(1); /* 071 */ InternalRow value_4 = isNull_4 ? /* 072 */ null : (i.getStruct(1, 4)); /* 073 */ resultIsNull_0 = isNull_4; /* 074 */ mutableStateArray_0[0] = value_4; /* 075 */ } /* 076 */ /* 077 */ isNull_2 = resultIsNull_0; /* 078 */ if (!isNull_2) { /* 079 */ /* 080 */ Object funcResult_0 = null; /* 081 */ funcResult_0 = value_3.deserialize(mutableStateArray_0[0]); /* 082 */ /* 083 */ if (funcResult_0 != null) { /* 084 */ value_2 = (org.apache.spark.ml.linalg.Vector) funcResult_0; /* 085 */ } else { /* 086 */ isNull_2 = true; /* 087 */ } /* 088 */ /* 089 */ /* 090 */ } /* 091 */ } /* 092 */ if (isNull_2) { /* 093 */ values_0[1] = null; /* 094 */ } else { /* 095 */ values_0[1] = value_2; /* 096 */ } /* 097 */ /* 098 */ } /* 099 */ /* 100 */ } 21/01/17 17:43:07.597 Executor task launch worker for task 2.0 in stage 1.0 (TID 6) DEBUG GenerateUnsafeProjection: code for newInstance(class org.apache.spark.ml.linalg.VectorUDT).serialize: /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificUnsafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificUnsafeProjection extends org.apache.spark.sql.catalyst.expressions.UnsafeProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private boolean resultIsNull_0; /* 009 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[] mutableStateArray_1 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[2]; /* 010 */ private org.apache.spark.ml.linalg.Vector[] mutableStateArray_0 = new org.apache.spark.ml.linalg.Vector[1]; /* 011 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[] mutableStateArray_2 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[2]; /* 012 */ /* 013 */ public SpecificUnsafeProjection(Object[] references) { /* 014 */ this.references = references; /* 015 */ /* 016 */ mutableStateArray_1[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(1, 32); /* 017 */ mutableStateArray_1[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(mutableStateArray_1[0], 4); /* 018 */ mutableStateArray_2[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(mutableStateArray_1[1], 4); /* 019 */ mutableStateArray_2[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(mutableStateArray_1[1], 8); /* 020 */ /* 021 */ } /* 022 */ /* 023 */ public void initialize(int partitionIndex) { /* 024 */ /* 025 */ } /* 026 */ /* 027 */ // Scala.Function1 need this /* 028 */ public java.lang.Object apply(java.lang.Object row) { /* 029 */ return apply((InternalRow) row); /* 030 */ } /* 031 */ /* 032 */ public UnsafeRow apply(InternalRow i) { /* 033 */ mutableStateArray_1[0].reset(); /* 034 */ /* 035 */ /* 036 */ mutableStateArray_1[0].zeroOutNullBytes(); /* 037 */ /* 038 */ final org.apache.spark.ml.linalg.VectorUDT value_1 = false ? /* 039 */ null : new org.apache.spark.ml.linalg.VectorUDT(); /* 040 */ boolean isNull_0 = true; /* 041 */ InternalRow value_0 = null; /* 042 */ if (!false) { /* 043 */ resultIsNull_0 = false; /* 044 */ if (!resultIsNull_0) { /* 045 */ boolean isNull_2 = i.isNullAt(0); /* 046 */ org.apache.spark.ml.linalg.Vector value_2 = isNull_2 ? /* 047 */ null : ((org.apache.spark.ml.linalg.Vector)i.get(0, null)); /* 048 */ resultIsNull_0 = isNull_2; /* 049 */ mutableStateArray_0[0] = value_2; /* 050 */ } /* 051 */ /* 052 */ isNull_0 = resultIsNull_0; /* 053 */ if (!isNull_0) { /* 054 */ /* 055 */ Object funcResult_0 = null; /* 056 */ funcResult_0 = value_1.serialize(mutableStateArray_0[0]); /* 057 */ /* 058 */ if (funcResult_0 != null) { /* 059 */ value_0 = (InternalRow) funcResult_0; /* 060 */ } else { /* 061 */ isNull_0 = true; /* 062 */ } /* 063 */ /* 064 */ /* 065 */ } /* 066 */ } /* 067 */ if (isNull_0) { /* 068 */ mutableStateArray_1[0].setNullAt(0); /* 069 */ } else { /* 070 */ final InternalRow tmpInput_0 = value_0; /* 071 */ if (tmpInput_0 instanceof UnsafeRow) { /* 072 */ mutableStateArray_1[0].write(0, (UnsafeRow) tmpInput_0); /* 073 */ } else { /* 074 */ // Remember the current cursor so that we can calculate how many bytes are /* 075 */ // written later. /* 076 */ final int previousCursor_0 = mutableStateArray_1[0].cursor(); /* 077 */ /* 078 */ mutableStateArray_1[1].resetRowWriter(); /* 079 */ /* 080 */ /* 081 */ mutableStateArray_1[1].write(0, (tmpInput_0.getByte(0))); /* 082 */ /* 083 */ /* 084 */ if ((tmpInput_0.isNullAt(1))) { /* 085 */ mutableStateArray_1[1].setNullAt(1); /* 086 */ } else { /* 087 */ mutableStateArray_1[1].write(1, (tmpInput_0.getInt(1))); /* 088 */ } /* 089 */ /* 090 */ /* 091 */ if ((tmpInput_0.isNullAt(2))) { /* 092 */ mutableStateArray_1[1].setNullAt(2); /* 093 */ } else { /* 094 */ // Remember the current cursor so that we can calculate how many bytes are /* 095 */ // written later. /* 096 */ final int previousCursor_1 = mutableStateArray_1[1].cursor(); /* 097 */ /* 098 */ final ArrayData tmpInput_1 = (tmpInput_0.getArray(2)); /* 099 */ if (tmpInput_1 instanceof UnsafeArrayData) { /* 100 */ mutableStateArray_1[1].write((UnsafeArrayData) tmpInput_1); /* 101 */ } else { /* 102 */ final int numElements_0 = tmpInput_1.numElements(); /* 103 */ mutableStateArray_2[0].initialize(numElements_0); /* 104 */ /* 105 */ for (int index_0 = 0; index_0 < numElements_0; index_0++) { /* 106 */ mutableStateArray_2[0].write(index_0, tmpInput_1.getInt(index_0)); /* 107 */ } /* 108 */ } /* 109 */ /* 110 */ mutableStateArray_1[1].setOffsetAndSizeFromPreviousCursor(2, previousCursor_1); /* 111 */ } /* 112 */ /* 113 */ /* 114 */ if ((tmpInput_0.isNullAt(3))) { /* 115 */ mutableStateArray_1[1].setNullAt(3); /* 116 */ } else { /* 117 */ // Remember the current cursor so that we can calculate how many bytes are /* 118 */ // written later. /* 119 */ final int previousCursor_2 = mutableStateArray_1[1].cursor(); /* 120 */ /* 121 */ final ArrayData tmpInput_2 = (tmpInput_0.getArray(3)); /* 122 */ if (tmpInput_2 instanceof UnsafeArrayData) { /* 123 */ mutableStateArray_1[1].write((UnsafeArrayData) tmpInput_2); /* 124 */ } else { /* 125 */ final int numElements_1 = tmpInput_2.numElements(); /* 126 */ mutableStateArray_2[1].initialize(numElements_1); /* 127 */ /* 128 */ for (int index_1 = 0; index_1 < numElements_1; index_1++) { /* 129 */ mutableStateArray_2[1].write(index_1, tmpInput_2.getDouble(index_1)); /* 130 */ } /* 131 */ } /* 132 */ /* 133 */ mutableStateArray_1[1].setOffsetAndSizeFromPreviousCursor(3, previousCursor_2); /* 134 */ } /* 135 */ /* 136 */ /* 137 */ mutableStateArray_1[0].setOffsetAndSizeFromPreviousCursor(0, previousCursor_0); /* 138 */ } /* 139 */ } /* 140 */ return (mutableStateArray_1[0].getRow()); /* 141 */ } /* 142 */ /* 143 */ /* 144 */ } 21/01/17 17:43:07.599 Executor task launch worker for task 1.0 in stage 1.0 (TID 5) DEBUG GenerateUnsafeProjection: code for newInstance(class org.apache.spark.ml.linalg.VectorUDT).serialize: /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificUnsafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificUnsafeProjection extends org.apache.spark.sql.catalyst.expressions.UnsafeProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private boolean resultIsNull_0; /* 009 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[] mutableStateArray_1 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[2]; /* 010 */ private org.apache.spark.ml.linalg.Vector[] mutableStateArray_0 = new org.apache.spark.ml.linalg.Vector[1]; /* 011 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[] mutableStateArray_2 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[2]; /* 012 */ /* 013 */ public SpecificUnsafeProjection(Object[] references) { /* 014 */ this.references = references; /* 015 */ /* 016 */ mutableStateArray_1[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(1, 32); /* 017 */ mutableStateArray_1[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(mutableStateArray_1[0], 4); /* 018 */ mutableStateArray_2[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(mutableStateArray_1[1], 4); /* 019 */ mutableStateArray_2[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(mutableStateArray_1[1], 8); /* 020 */ /* 021 */ } /* 022 */ /* 023 */ public void initialize(int partitionIndex) { /* 024 */ /* 025 */ } /* 026 */ /* 027 */ // Scala.Function1 need this /* 028 */ public java.lang.Object apply(java.lang.Object row) { /* 029 */ return apply((InternalRow) row); /* 030 */ } /* 031 */ /* 032 */ public UnsafeRow apply(InternalRow i) { /* 033 */ mutableStateArray_1[0].reset(); /* 034 */ /* 035 */ /* 036 */ mutableStateArray_1[0].zeroOutNullBytes(); /* 037 */ /* 038 */ final org.apache.spark.ml.linalg.VectorUDT value_1 = false ? /* 039 */ null : new org.apache.spark.ml.linalg.VectorUDT(); /* 040 */ boolean isNull_0 = true; /* 041 */ InternalRow value_0 = null; /* 042 */ if (!false) { /* 043 */ resultIsNull_0 = false; /* 044 */ if (!resultIsNull_0) { /* 045 */ boolean isNull_2 = i.isNullAt(0); /* 046 */ org.apache.spark.ml.linalg.Vector value_2 = isNull_2 ? /* 047 */ null : ((org.apache.spark.ml.linalg.Vector)i.get(0, null)); /* 048 */ resultIsNull_0 = isNull_2; /* 049 */ mutableStateArray_0[0] = value_2; /* 050 */ } /* 051 */ /* 052 */ isNull_0 = resultIsNull_0; /* 053 */ if (!isNull_0) { /* 054 */ /* 055 */ Object funcResult_0 = null; /* 056 */ funcResult_0 = value_1.serialize(mutableStateArray_0[0]); /* 057 */ /* 058 */ if (funcResult_0 != null) { /* 059 */ value_0 = (InternalRow) funcResult_0; /* 060 */ } else { /* 061 */ isNull_0 = true; /* 062 */ } /* 063 */ /* 064 */ /* 065 */ } /* 066 */ } /* 067 */ if (isNull_0) { /* 068 */ mutableStateArray_1[0].setNullAt(0); /* 069 */ } else { /* 070 */ final InternalRow tmpInput_0 = value_0; /* 071 */ if (tmpInput_0 instanceof UnsafeRow) { /* 072 */ mutableStateArray_1[0].write(0, (UnsafeRow) tmpInput_0); /* 073 */ } else { /* 074 */ // Remember the current cursor so that we can calculate how many bytes are /* 075 */ // written later. /* 076 */ final int previousCursor_0 = mutableStateArray_1[0].cursor(); /* 077 */ /* 078 */ mutableStateArray_1[1].resetRowWriter(); /* 079 */ /* 080 */ /* 081 */ mutableStateArray_1[1].write(0, (tmpInput_0.getByte(0))); /* 082 */ /* 083 */ /* 084 */ if ((tmpInput_0.isNullAt(1))) { /* 085 */ mutableStateArray_1[1].setNullAt(1); /* 086 */ } else { /* 087 */ mutableStateArray_1[1].write(1, (tmpInput_0.getInt(1))); /* 088 */ } /* 089 */ /* 090 */ /* 091 */ if ((tmpInput_0.isNullAt(2))) { /* 092 */ mutableStateArray_1[1].setNullAt(2); /* 093 */ } else { /* 094 */ // Remember the current cursor so that we can calculate how many bytes are /* 095 */ // written later. /* 096 */ final int previousCursor_1 = mutableStateArray_1[1].cursor(); /* 097 */ /* 098 */ final ArrayData tmpInput_1 = (tmpInput_0.getArray(2)); /* 099 */ if (tmpInput_1 instanceof UnsafeArrayData) { /* 100 */ mutableStateArray_1[1].write((UnsafeArrayData) tmpInput_1); /* 101 */ } else { /* 102 */ final int numElements_0 = tmpInput_1.numElements(); /* 103 */ mutableStateArray_2[0].initialize(numElements_0); /* 104 */ /* 105 */ for (int index_0 = 0; index_0 < numElements_0; index_0++) { /* 106 */ mutableStateArray_2[0].write(index_0, tmpInput_1.getInt(index_0)); /* 107 */ } /* 108 */ } /* 109 */ /* 110 */ mutableStateArray_1[1].setOffsetAndSizeFromPreviousCursor(2, previousCursor_1); /* 111 */ } /* 112 */ /* 113 */ /* 114 */ if ((tmpInput_0.isNullAt(3))) { /* 115 */ mutableStateArray_1[1].setNullAt(3); /* 116 */ } else { /* 117 */ // Remember the current cursor so that we can calculate how many bytes are /* 118 */ // written later. /* 119 */ final int previousCursor_2 = mutableStateArray_1[1].cursor(); /* 120 */ /* 121 */ final ArrayData tmpInput_2 = (tmpInput_0.getArray(3)); /* 122 */ if (tmpInput_2 instanceof UnsafeArrayData) { /* 123 */ mutableStateArray_1[1].write((UnsafeArrayData) tmpInput_2); /* 124 */ } else { /* 125 */ final int numElements_1 = tmpInput_2.numElements(); /* 126 */ mutableStateArray_2[1].initialize(numElements_1); /* 127 */ /* 128 */ for (int index_1 = 0; index_1 < numElements_1; index_1++) { /* 129 */ mutableStateArray_2[1].write(index_1, tmpInput_2.getDouble(index_1)); /* 130 */ } /* 131 */ } /* 132 */ /* 133 */ mutableStateArray_1[1].setOffsetAndSizeFromPreviousCursor(3, previousCursor_2); /* 134 */ } /* 135 */ /* 136 */ /* 137 */ mutableStateArray_1[0].setOffsetAndSizeFromPreviousCursor(0, previousCursor_0); /* 138 */ } /* 139 */ } /* 140 */ return (mutableStateArray_1[0].getRow()); /* 141 */ } /* 142 */ /* 143 */ /* 144 */ } 21/01/17 17:43:07.601 Executor task launch worker for task 2.0 in stage 1.0 (TID 3) DEBUG GenerateSafeProjection: code for createexternalrow(input[0, double, true], newInstance(class org.apache.spark.ml.linalg.VectorUDT).deserialize, input[2, double, false], StructField(prediction,DoubleType,true), StructField(normalizedFeatures,org.apache.spark.ml.linalg.VectorUDT@3bfc3ba7,true), StructField(weightCol,DoubleType,false)): /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificSafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificSafeProjection extends org.apache.spark.sql.catalyst.expressions.codegen.BaseProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private InternalRow mutableRow; /* 009 */ private boolean resultIsNull_0; /* 010 */ private InternalRow[] mutableStateArray_0 = new InternalRow[1]; /* 011 */ /* 012 */ public SpecificSafeProjection(Object[] references) { /* 013 */ this.references = references; /* 014 */ mutableRow = (InternalRow) references[references.length - 1]; /* 015 */ /* 016 */ /* 017 */ } /* 018 */ /* 019 */ public void initialize(int partitionIndex) { /* 020 */ /* 021 */ } /* 022 */ /* 023 */ public java.lang.Object apply(java.lang.Object _i) { /* 024 */ InternalRow i = (InternalRow) _i; /* 025 */ Object[] values_0 = new Object[3]; /* 026 */ createExternalRow_0_0(i, values_0); /* 027 */ createExternalRow_0_1(i, values_0); /* 028 */ final org.apache.spark.sql.Row value_0 = new org.apache.spark.sql.catalyst.expressions.GenericRowWithSchema(values_0, ((org.apache.spark.sql.types.StructType) references[0] /* schema */)); /* 029 */ if (false) { /* 030 */ mutableRow.setNullAt(0); /* 031 */ } else { /* 032 */ /* 033 */ mutableRow.update(0, value_0); /* 034 */ } /* 035 */ /* 036 */ return mutableRow; /* 037 */ } /* 038 */ /* 039 */ /* 040 */ private void createExternalRow_0_1(InternalRow i, Object[] values_0) { /* 041 */ /* 042 */ double value_5 = i.getDouble(2); /* 043 */ if (false) { /* 044 */ values_0[2] = null; /* 045 */ } else { /* 046 */ values_0[2] = value_5; /* 047 */ } /* 048 */ /* 049 */ } /* 050 */ /* 051 */ /* 052 */ private void createExternalRow_0_0(InternalRow i, Object[] values_0) { /* 053 */ /* 054 */ boolean isNull_1 = i.isNullAt(0); /* 055 */ double value_1 = isNull_1 ? /* 056 */ -1.0 : (i.getDouble(0)); /* 057 */ if (isNull_1) { /* 058 */ values_0[0] = null; /* 059 */ } else { /* 060 */ values_0[0] = value_1; /* 061 */ } /* 062 */ /* 063 */ final org.apache.spark.ml.linalg.VectorUDT value_3 = false ? /* 064 */ null : new org.apache.spark.ml.linalg.VectorUDT(); /* 065 */ boolean isNull_2 = true; /* 066 */ org.apache.spark.ml.linalg.Vector value_2 = null; /* 067 */ if (!false) { /* 068 */ resultIsNull_0 = false; /* 069 */ if (!resultIsNull_0) { /* 070 */ boolean isNull_4 = i.isNullAt(1); /* 071 */ InternalRow value_4 = isNull_4 ? /* 072 */ null : (i.getStruct(1, 4)); /* 073 */ resultIsNull_0 = isNull_4; /* 074 */ mutableStateArray_0[0] = value_4; /* 075 */ } /* 076 */ /* 077 */ isNull_2 = resultIsNull_0; /* 078 */ if (!isNull_2) { /* 079 */ /* 080 */ Object funcResult_0 = null; /* 081 */ funcResult_0 = value_3.deserialize(mutableStateArray_0[0]); /* 082 */ /* 083 */ if (funcResult_0 != null) { /* 084 */ value_2 = (org.apache.spark.ml.linalg.Vector) funcResult_0; /* 085 */ } else { /* 086 */ isNull_2 = true; /* 087 */ } /* 088 */ /* 089 */ /* 090 */ } /* 091 */ } /* 092 */ if (isNull_2) { /* 093 */ values_0[1] = null; /* 094 */ } else { /* 095 */ values_0[1] = value_2; /* 096 */ } /* 097 */ /* 098 */ } /* 099 */ /* 100 */ } 21/01/17 17:43:07.605 Executor task launch worker for task 2.0 in stage 1.0 (TID 6) INFO PythonRunner: Times: total = 55, boot = -702, init = 757, finish = 0 21/01/17 17:43:07.605 Executor task launch worker for task 1.0 in stage 1.0 (TID 2) INFO CodeGenerator: Code generated in 103.637931 ms 21/01/17 17:43:07.607 Executor task launch worker for task 2.0 in stage 1.0 (TID 6) INFO Executor: Finished task 2.0 in stage 1.0 (TID 6). 2047 bytes result sent to driver 21/01/17 17:43:07.607 Executor task launch worker for task 2.0 in stage 1.0 (TID 6) DEBUG ExecutorMetricsPoller: stageTCMP: (1, 0) -> 1 21/01/17 17:43:07.605 Executor task launch worker for task 1.0 in stage 1.0 (TID 5) INFO PythonRunner: Times: total = 94, boot = -746, init = 840, finish = 0 21/01/17 17:43:07.610 task-result-getter-1 INFO TaskSetManager: Finished task 2.0 in stage 1.0 (TID 6) in 182 ms on 192.168.1.225 (executor driver) (3/4) 21/01/17 17:43:07.613 Thread-3 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$count$1$adapted) is now cleaned +++ 21/01/17 17:43:07.613 Thread-3 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$runJob$5 21/01/17 17:43:07.618 Thread-3 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$runJob$5) is now cleaned +++ 21/01/17 17:43:07.619 Executor task launch worker for task 1.0 in stage 1.0 (TID 5) INFO Executor: Finished task 1.0 in stage 1.0 (TID 5). 2062 bytes result sent to driver 21/01/17 17:43:07.619 Executor task launch worker for task 1.0 in stage 1.0 (TID 5) DEBUG ExecutorMetricsPoller: removing (1, 0) from stageTCMP 21/01/17 17:43:07.619 Thread-3 INFO SparkContext: Starting job: count at ALS.scala:980 21/01/17 17:43:07.620 task-result-getter-3 INFO TaskSetManager: Finished task 1.0 in stage 1.0 (TID 5) in 192 ms on 192.168.1.225 (executor driver) (4/4) 21/01/17 17:43:07.620 task-result-getter-3 INFO TaskSchedulerImpl: Removed TaskSet 1.0, whose tasks have all completed, from pool 21/01/17 17:43:07.620 dag-scheduler-event-loop DEBUG DAGScheduler: Merging stage rdd profiles: Set() 21/01/17 17:43:07.621 dag-scheduler-event-loop DEBUG DAGScheduler: Merging stage rdd profiles: Set() 21/01/17 17:43:07.621 dag-scheduler-event-loop INFO DAGScheduler: ResultStage 1 ( at /usr/lib/python3.6/runpy.py:85) finished in 0.216 s 21/01/17 17:43:07.621 dag-scheduler-event-loop DEBUG DAGScheduler: After removal of stage 1, remaining stages = 0 21/01/17 17:43:07.621 dag-scheduler-event-loop INFO DAGScheduler: Job 1 is finished. Cancelling potential speculative or zombie tasks for this job 21/01/17 17:43:07.621 dag-scheduler-event-loop INFO TaskSchedulerImpl: Killing all running tasks in stage 1: Stage finished 21/01/17 17:43:07.622 dag-scheduler-event-loop DEBUG DAGScheduler: Merging stage rdd profiles: Set() 21/01/17 17:43:07.622 dag-scheduler-event-loop INFO DAGScheduler: Registering RDD 22 (map at ALS.scala:1612) as input to shuffle 2 21/01/17 17:43:07.622 dag-scheduler-event-loop INFO DAGScheduler: Got job 2 (count at ALS.scala:980) with 10 output partitions 21/01/17 17:43:07.622 dag-scheduler-event-loop INFO DAGScheduler: Final stage: ResultStage 6 (count at ALS.scala:980) 21/01/17 17:43:07.623 dag-scheduler-event-loop INFO DAGScheduler: Parents of final stage: List(ShuffleMapStage 5) 21/01/17 17:43:07.626 dag-scheduler-event-loop INFO DAGScheduler: Missing parents: List(ShuffleMapStage 5) 21/01/17 17:43:07.626 dag-scheduler-event-loop DEBUG DAGScheduler: submitStage(ResultStage 6 (name=count at ALS.scala:980;jobs=2)) 21/01/17 17:43:07.626 dag-scheduler-event-loop DEBUG DAGScheduler: missing: List(ShuffleMapStage 5) 21/01/17 17:43:07.626 dag-scheduler-event-loop DEBUG DAGScheduler: submitStage(ShuffleMapStage 5 (name=map at ALS.scala:1612;jobs=2)) 21/01/17 17:43:07.630 dag-scheduler-event-loop DEBUG DAGScheduler: missing: List() 21/01/17 17:43:07.630 dag-scheduler-event-loop INFO DAGScheduler: Submitting ShuffleMapStage 5 (MapPartitionsRDD[22] at map at ALS.scala:1612), which has no missing parents 21/01/17 17:43:07.630 dag-scheduler-event-loop DEBUG DAGScheduler: submitMissingTasks(ShuffleMapStage 5) 21/01/17 17:43:07.632 Thread-3 INFO DAGScheduler: Job 1 finished: at /usr/lib/python3.6/runpy.py:85, took 0.236127 s 21/01/17 17:43:07.632 Executor task launch worker for task 3.0 in stage 1.0 (TID 4) DEBUG GenerateSafeProjection: code for newInstance(class org.apache.spark.ml.linalg.VectorUDT).deserialize: /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificSafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificSafeProjection extends org.apache.spark.sql.catalyst.expressions.codegen.BaseProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private InternalRow mutableRow; /* 009 */ private boolean resultIsNull_0; /* 010 */ private InternalRow[] mutableStateArray_0 = new InternalRow[1]; /* 011 */ /* 012 */ public SpecificSafeProjection(Object[] references) { /* 013 */ this.references = references; /* 014 */ mutableRow = (InternalRow) references[references.length - 1]; /* 015 */ /* 016 */ /* 017 */ } /* 018 */ /* 019 */ public void initialize(int partitionIndex) { /* 020 */ /* 021 */ } /* 022 */ /* 023 */ public java.lang.Object apply(java.lang.Object _i) { /* 024 */ InternalRow i = (InternalRow) _i; /* 025 */ final org.apache.spark.ml.linalg.VectorUDT value_1 = false ? /* 026 */ null : new org.apache.spark.ml.linalg.VectorUDT(); /* 027 */ boolean isNull_0 = true; /* 028 */ org.apache.spark.ml.linalg.Vector value_0 = null; /* 029 */ if (!false) { /* 030 */ resultIsNull_0 = false; /* 031 */ if (!resultIsNull_0) { /* 032 */ boolean isNull_2 = i.isNullAt(0); /* 033 */ InternalRow value_2 = isNull_2 ? /* 034 */ null : (i.getStruct(0, 4)); /* 035 */ resultIsNull_0 = isNull_2; /* 036 */ mutableStateArray_0[0] = value_2; /* 037 */ } /* 038 */ /* 039 */ isNull_0 = resultIsNull_0; /* 040 */ if (!isNull_0) { /* 041 */ /* 042 */ Object funcResult_0 = null; /* 043 */ funcResult_0 = value_1.deserialize(mutableStateArray_0[0]); /* 044 */ /* 045 */ if (funcResult_0 != null) { /* 046 */ value_0 = (org.apache.spark.ml.linalg.Vector) funcResult_0; /* 047 */ } else { /* 048 */ isNull_0 = true; /* 049 */ } /* 050 */ /* 051 */ /* 052 */ } /* 053 */ } /* 054 */ if (isNull_0) { /* 055 */ mutableRow.setNullAt(0); /* 056 */ } else { /* 057 */ /* 058 */ mutableRow.update(0, value_0); /* 059 */ } /* 060 */ /* 061 */ return mutableRow; /* 062 */ } /* 063 */ /* 064 */ /* 065 */ } 21/01/17 17:43:07.639 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_4 stored as values in memory (estimated size 33.5 KiB, free 366.1 MiB) 21/01/17 17:43:07.639 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_4 locally took 1 ms 21/01/17 17:43:07.639 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_4 without replication took 1 ms 21/01/17 17:43:07.640 Executor task launch worker for task 3.0 in stage 1.0 (TID 4) DEBUG CodeGenerator: /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificSafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificSafeProjection extends org.apache.spark.sql.catalyst.expressions.codegen.BaseProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private InternalRow mutableRow; /* 009 */ private boolean resultIsNull_0; /* 010 */ private InternalRow[] mutableStateArray_0 = new InternalRow[1]; /* 011 */ /* 012 */ public SpecificSafeProjection(Object[] references) { /* 013 */ this.references = references; /* 014 */ mutableRow = (InternalRow) references[references.length - 1]; /* 015 */ /* 016 */ /* 017 */ } /* 018 */ /* 019 */ public void initialize(int partitionIndex) { /* 020 */ /* 021 */ } /* 022 */ /* 023 */ public java.lang.Object apply(java.lang.Object _i) { /* 024 */ InternalRow i = (InternalRow) _i; /* 025 */ final org.apache.spark.ml.linalg.VectorUDT value_1 = false ? /* 026 */ null : new org.apache.spark.ml.linalg.VectorUDT(); /* 027 */ boolean isNull_0 = true; /* 028 */ org.apache.spark.ml.linalg.Vector value_0 = null; /* 029 */ if (!false) { /* 030 */ resultIsNull_0 = false; /* 031 */ if (!resultIsNull_0) { /* 032 */ boolean isNull_2 = i.isNullAt(0); /* 033 */ InternalRow value_2 = isNull_2 ? /* 034 */ null : (i.getStruct(0, 4)); /* 035 */ resultIsNull_0 = isNull_2; /* 036 */ mutableStateArray_0[0] = value_2; /* 037 */ } /* 038 */ /* 039 */ isNull_0 = resultIsNull_0; /* 040 */ if (!isNull_0) { /* 041 */ /* 042 */ Object funcResult_0 = null; /* 043 */ funcResult_0 = value_1.deserialize(mutableStateArray_0[0]); /* 044 */ /* 045 */ if (funcResult_0 != null) { /* 046 */ value_0 = (org.apache.spark.ml.linalg.Vector) funcResult_0; /* 047 */ } else { /* 048 */ isNull_0 = true; /* 049 */ } /* 050 */ /* 051 */ /* 052 */ } /* 053 */ } /* 054 */ if (isNull_0) { /* 055 */ mutableRow.setNullAt(0); /* 056 */ } else { /* 057 */ /* 058 */ mutableRow.update(0, value_0); /* 059 */ } /* 060 */ /* 061 */ return mutableRow; /* 062 */ } /* 063 */ /* 064 */ /* 065 */ } 21/01/17 17:43:07.641 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_4_piece0 stored as bytes in memory (estimated size 15.9 KiB, free 366.1 MiB) 21/01/17 17:43:07.642 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master broadcast_4_piece0 for BlockManagerId(driver, 192.168.1.225, 40985, None) 21/01/17 17:43:07.643 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added broadcast_4_piece0 in memory on 192.168.1.225:40985 (size: 15.9 KiB, free: 366.2 MiB) 21/01/17 17:43:07.643 dag-scheduler-event-loop DEBUG BlockManagerMaster: Updated info of block broadcast_4_piece0 21/01/17 17:43:07.643 dag-scheduler-event-loop DEBUG BlockManager: Told master about block broadcast_4_piece0 21/01/17 17:43:07.643 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_4_piece0 locally took 2 ms 21/01/17 17:43:07.643 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_4_piece0 without replication took 3 ms 21/01/17 17:43:07.644 dag-scheduler-event-loop INFO SparkContext: Created broadcast 4 from broadcast at DAGScheduler.scala:1383 21/01/17 17:43:07.644 dag-scheduler-event-loop INFO DAGScheduler: Submitting 4 missing tasks from ShuffleMapStage 5 (MapPartitionsRDD[22] at map at ALS.scala:1612) (first 15 tasks are for partitions Vector(0, 1, 2, 3)) 21/01/17 17:43:07.644 dag-scheduler-event-loop INFO TaskSchedulerImpl: Adding task set 5.0 with 4 tasks resource profile 0 21/01/17 17:43:07.643 Executor task launch worker for task 1.0 in stage 1.0 (TID 2) DEBUG GenerateSafeProjection: code for newInstance(class org.apache.spark.ml.linalg.VectorUDT).deserialize: /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificSafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificSafeProjection extends org.apache.spark.sql.catalyst.expressions.codegen.BaseProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private InternalRow mutableRow; /* 009 */ private boolean resultIsNull_0; /* 010 */ private InternalRow[] mutableStateArray_0 = new InternalRow[1]; /* 011 */ /* 012 */ public SpecificSafeProjection(Object[] references) { /* 013 */ this.references = references; /* 014 */ mutableRow = (InternalRow) references[references.length - 1]; /* 015 */ /* 016 */ /* 017 */ } /* 018 */ /* 019 */ public void initialize(int partitionIndex) { /* 020 */ /* 021 */ } /* 022 */ /* 023 */ public java.lang.Object apply(java.lang.Object _i) { /* 024 */ InternalRow i = (InternalRow) _i; /* 025 */ final org.apache.spark.ml.linalg.VectorUDT value_1 = false ? /* 026 */ null : new org.apache.spark.ml.linalg.VectorUDT(); /* 027 */ boolean isNull_0 = true; /* 028 */ org.apache.spark.ml.linalg.Vector value_0 = null; /* 029 */ if (!false) { /* 030 */ resultIsNull_0 = false; /* 031 */ if (!resultIsNull_0) { /* 032 */ boolean isNull_2 = i.isNullAt(0); /* 033 */ InternalRow value_2 = isNull_2 ? /* 034 */ null : (i.getStruct(0, 4)); /* 035 */ resultIsNull_0 = isNull_2; /* 036 */ mutableStateArray_0[0] = value_2; /* 037 */ } /* 038 */ /* 039 */ isNull_0 = resultIsNull_0; /* 040 */ if (!isNull_0) { /* 041 */ /* 042 */ Object funcResult_0 = null; /* 043 */ funcResult_0 = value_1.deserialize(mutableStateArray_0[0]); /* 044 */ /* 045 */ if (funcResult_0 != null) { /* 046 */ value_0 = (org.apache.spark.ml.linalg.Vector) funcResult_0; /* 047 */ } else { /* 048 */ isNull_0 = true; /* 049 */ } /* 050 */ /* 051 */ /* 052 */ } /* 053 */ } /* 054 */ if (isNull_0) { /* 055 */ mutableRow.setNullAt(0); /* 056 */ } else { /* 057 */ /* 058 */ mutableRow.update(0, value_0); /* 059 */ } /* 060 */ /* 061 */ return mutableRow; /* 062 */ } /* 063 */ /* 064 */ /* 065 */ } 21/01/17 17:43:07.663 dag-scheduler-event-loop DEBUG TaskSetManager: Epoch for TaskSet 5.0: 2 21/01/17 17:43:07.664 dag-scheduler-event-loop DEBUG TaskSetManager: Adding pending tasks took 0 ms 21/01/17 17:43:07.665 dag-scheduler-event-loop DEBUG TaskSetManager: Valid locality levels for TaskSet 5.0: PROCESS_LOCAL, NODE_LOCAL, ANY 21/01/17 17:43:07.647 Executor task launch worker for task 0.0 in stage 1.0 (TID 1) DEBUG GenerateSafeProjection: code for newInstance(class org.apache.spark.ml.linalg.VectorUDT).deserialize: /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificSafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificSafeProjection extends org.apache.spark.sql.catalyst.expressions.codegen.BaseProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private InternalRow mutableRow; /* 009 */ private boolean resultIsNull_0; /* 010 */ private InternalRow[] mutableStateArray_0 = new InternalRow[1]; /* 011 */ /* 012 */ public SpecificSafeProjection(Object[] references) { /* 013 */ this.references = references; /* 014 */ mutableRow = (InternalRow) references[references.length - 1]; /* 015 */ /* 016 */ /* 017 */ } /* 018 */ /* 019 */ public void initialize(int partitionIndex) { /* 020 */ /* 021 */ } /* 022 */ /* 023 */ public java.lang.Object apply(java.lang.Object _i) { /* 024 */ InternalRow i = (InternalRow) _i; /* 025 */ final org.apache.spark.ml.linalg.VectorUDT value_1 = false ? /* 026 */ null : new org.apache.spark.ml.linalg.VectorUDT(); /* 027 */ boolean isNull_0 = true; /* 028 */ org.apache.spark.ml.linalg.Vector value_0 = null; /* 029 */ if (!false) { /* 030 */ resultIsNull_0 = false; /* 031 */ if (!resultIsNull_0) { /* 032 */ boolean isNull_2 = i.isNullAt(0); /* 033 */ InternalRow value_2 = isNull_2 ? /* 034 */ null : (i.getStruct(0, 4)); /* 035 */ resultIsNull_0 = isNull_2; /* 036 */ mutableStateArray_0[0] = value_2; /* 037 */ } /* 038 */ /* 039 */ isNull_0 = resultIsNull_0; /* 040 */ if (!isNull_0) { /* 041 */ /* 042 */ Object funcResult_0 = null; /* 043 */ funcResult_0 = value_1.deserialize(mutableStateArray_0[0]); /* 044 */ /* 045 */ if (funcResult_0 != null) { /* 046 */ value_0 = (org.apache.spark.ml.linalg.Vector) funcResult_0; /* 047 */ } else { /* 048 */ isNull_0 = true; /* 049 */ } /* 050 */ /* 051 */ /* 052 */ } /* 053 */ } /* 054 */ if (isNull_0) { /* 055 */ mutableRow.setNullAt(0); /* 056 */ } else { /* 057 */ /* 058 */ mutableRow.update(0, value_0); /* 059 */ } /* 060 */ /* 061 */ return mutableRow; /* 062 */ } /* 063 */ /* 064 */ /* 065 */ } 21/01/17 17:43:07.666 dispatcher-event-loop-0 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_5.0, runningTasks: 0 21/01/17 17:43:07.666 dispatcher-event-loop-0 INFO TaskSetManager: Starting task 0.0 in stage 5.0 (TID 19) (192.168.1.225, executor driver, partition 0, PROCESS_LOCAL, 7132 bytes) taskResourceAssignments Map() 21/01/17 17:43:07.667 dispatcher-event-loop-0 INFO TaskSetManager: Starting task 1.0 in stage 5.0 (TID 20) (192.168.1.225, executor driver, partition 1, PROCESS_LOCAL, 7132 bytes) taskResourceAssignments Map() 21/01/17 17:43:07.667 dispatcher-event-loop-0 INFO TaskSetManager: Starting task 2.0 in stage 5.0 (TID 21) (192.168.1.225, executor driver, partition 2, PROCESS_LOCAL, 7132 bytes) taskResourceAssignments Map() 21/01/17 17:43:07.667 dispatcher-event-loop-0 INFO TaskSetManager: Starting task 3.0 in stage 5.0 (TID 22) (192.168.1.225, executor driver, partition 3, PROCESS_LOCAL, 7132 bytes) taskResourceAssignments Map() 21/01/17 17:43:07.668 Executor task launch worker for task 0.0 in stage 5.0 (TID 19) INFO Executor: Running task 0.0 in stage 5.0 (TID 19) 21/01/17 17:43:07.668 Executor task launch worker for task 2.0 in stage 5.0 (TID 21) INFO Executor: Running task 2.0 in stage 5.0 (TID 21) 21/01/17 17:43:07.668 Executor task launch worker for task 0.0 in stage 5.0 (TID 19) DEBUG ExecutorMetricsPoller: stageTCMP: (5, 0) -> 1 21/01/17 17:43:07.668 Executor task launch worker for task 1.0 in stage 5.0 (TID 20) INFO Executor: Running task 1.0 in stage 5.0 (TID 20) 21/01/17 17:43:07.669 Executor task launch worker for task 3.0 in stage 5.0 (TID 22) INFO Executor: Running task 3.0 in stage 5.0 (TID 22) 21/01/17 17:43:07.669 Executor task launch worker for task 1.0 in stage 5.0 (TID 20) DEBUG ExecutorMetricsPoller: stageTCMP: (5, 0) -> 3 21/01/17 17:43:07.669 Executor task launch worker for task 0.0 in stage 5.0 (TID 19) DEBUG BlockManager: Getting local block broadcast_4 21/01/17 17:43:07.669 Executor task launch worker for task 0.0 in stage 5.0 (TID 19) DEBUG BlockManager: Level for block broadcast_4 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:07.668 Executor task launch worker for task 2.0 in stage 5.0 (TID 21) DEBUG ExecutorMetricsPoller: stageTCMP: (5, 0) -> 2 21/01/17 17:43:07.669 Executor task launch worker for task 3.0 in stage 5.0 (TID 22) DEBUG ExecutorMetricsPoller: stageTCMP: (5, 0) -> 4 21/01/17 17:43:07.683 Executor task launch worker for task 2.0 in stage 5.0 (TID 21) DEBUG BlockManager: Getting local block rdd_16_2 21/01/17 17:43:07.684 Executor task launch worker for task 2.0 in stage 5.0 (TID 21) DEBUG BlockManager: Level for block rdd_16_2 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:07.684 Executor task launch worker for task 2.0 in stage 5.0 (TID 21) INFO BlockManager: Found block rdd_16_2 locally 21/01/17 17:43:07.670 Executor task launch worker for task 2.0 in stage 1.0 (TID 3) DEBUG GenerateSafeProjection: code for newInstance(class org.apache.spark.ml.linalg.VectorUDT).deserialize: /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificSafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificSafeProjection extends org.apache.spark.sql.catalyst.expressions.codegen.BaseProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private InternalRow mutableRow; /* 009 */ private boolean resultIsNull_0; /* 010 */ private InternalRow[] mutableStateArray_0 = new InternalRow[1]; /* 011 */ /* 012 */ public SpecificSafeProjection(Object[] references) { /* 013 */ this.references = references; /* 014 */ mutableRow = (InternalRow) references[references.length - 1]; /* 015 */ /* 016 */ /* 017 */ } /* 018 */ /* 019 */ public void initialize(int partitionIndex) { /* 020 */ /* 021 */ } /* 022 */ /* 023 */ public java.lang.Object apply(java.lang.Object _i) { /* 024 */ InternalRow i = (InternalRow) _i; /* 025 */ final org.apache.spark.ml.linalg.VectorUDT value_1 = false ? /* 026 */ null : new org.apache.spark.ml.linalg.VectorUDT(); /* 027 */ boolean isNull_0 = true; /* 028 */ org.apache.spark.ml.linalg.Vector value_0 = null; /* 029 */ if (!false) { /* 030 */ resultIsNull_0 = false; /* 031 */ if (!resultIsNull_0) { /* 032 */ boolean isNull_2 = i.isNullAt(0); /* 033 */ InternalRow value_2 = isNull_2 ? /* 034 */ null : (i.getStruct(0, 4)); /* 035 */ resultIsNull_0 = isNull_2; /* 036 */ mutableStateArray_0[0] = value_2; /* 037 */ } /* 038 */ /* 039 */ isNull_0 = resultIsNull_0; /* 040 */ if (!isNull_0) { /* 041 */ /* 042 */ Object funcResult_0 = null; /* 043 */ funcResult_0 = value_1.deserialize(mutableStateArray_0[0]); /* 044 */ /* 045 */ if (funcResult_0 != null) { /* 046 */ value_0 = (org.apache.spark.ml.linalg.Vector) funcResult_0; /* 047 */ } else { /* 048 */ isNull_0 = true; /* 049 */ } /* 050 */ /* 051 */ /* 052 */ } /* 053 */ } /* 054 */ if (isNull_0) { /* 055 */ mutableRow.setNullAt(0); /* 056 */ } else { /* 057 */ /* 058 */ mutableRow.update(0, value_0); /* 059 */ } /* 060 */ /* 061 */ return mutableRow; /* 062 */ } /* 063 */ /* 064 */ /* 065 */ } 21/01/17 17:43:07.689 Executor task launch worker for task 0.0 in stage 5.0 (TID 19) DEBUG BlockManager: Getting local block rdd_16_0 21/01/17 17:43:07.689 Executor task launch worker for task 0.0 in stage 5.0 (TID 19) DEBUG BlockManager: Level for block rdd_16_0 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:07.689 Executor task launch worker for task 0.0 in stage 5.0 (TID 19) INFO BlockManager: Found block rdd_16_0 locally 21/01/17 17:43:07.690 Executor task launch worker for task 3.0 in stage 5.0 (TID 22) DEBUG BlockManager: Getting local block rdd_16_3 21/01/17 17:43:07.690 Executor task launch worker for task 3.0 in stage 5.0 (TID 22) DEBUG BlockManager: Level for block rdd_16_3 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:07.690 Executor task launch worker for task 3.0 in stage 5.0 (TID 22) INFO BlockManager: Found block rdd_16_3 locally 21/01/17 17:43:07.692 Executor task launch worker for task 3.0 in stage 5.0 (TID 22) DEBUG ALS: Converting to local indices took 0.001182998 seconds. 21/01/17 17:43:07.693 Executor task launch worker for task 3.0 in stage 5.0 (TID 22) DEBUG ALS: Converting to local indices took 5.36414E-4 seconds. 21/01/17 17:43:07.693 Executor task launch worker for task 2.0 in stage 5.0 (TID 21) DEBUG ALS: Converting to local indices took 5.13143E-4 seconds. 21/01/17 17:43:07.694 Executor task launch worker for task 0.0 in stage 5.0 (TID 19) DEBUG ALS: Converting to local indices took 5.51464E-4 seconds. 21/01/17 17:43:07.694 Executor task launch worker for task 1.0 in stage 5.0 (TID 20) DEBUG BlockManager: Getting local block rdd_16_1 21/01/17 17:43:07.694 Executor task launch worker for task 2.0 in stage 5.0 (TID 21) DEBUG ALS: Converting to local indices took 5.06194E-4 seconds. 21/01/17 17:43:07.694 Executor task launch worker for task 1.0 in stage 5.0 (TID 20) DEBUG BlockManager: Level for block rdd_16_1 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:07.694 Executor task launch worker for task 1.0 in stage 5.0 (TID 20) INFO BlockManager: Found block rdd_16_1 locally 21/01/17 17:43:07.697 Executor task launch worker for task 0.0 in stage 5.0 (TID 19) DEBUG LocalDiskShuffleMapOutputWriter: Writing shuffle index file for mapId 19 with length 10 21/01/17 17:43:07.697 Executor task launch worker for task 0.0 in stage 5.0 (TID 19) DEBUG IndexShuffleBlockResolver: Shuffle index for mapId 19: [0,251,0,0,0,0,0,0,0,0] 21/01/17 17:43:07.698 Executor task launch worker for task 1.0 in stage 5.0 (TID 20) DEBUG ALS: Converting to local indices took 5.44494E-4 seconds. 21/01/17 17:43:07.699 Executor task launch worker for task 0.0 in stage 5.0 (TID 19) INFO Executor: Finished task 0.0 in stage 5.0 (TID 19). 1993 bytes result sent to driver 21/01/17 17:43:07.699 Thread-3 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$pythonToJava$1 21/01/17 17:43:07.700 Thread-3 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$pythonToJava$1) is now cleaned +++ 21/01/17 17:43:07.696 Executor task launch worker for task 3.0 in stage 1.0 (TID 4) INFO CodeGenerator: Code generated in 57.60461 ms 21/01/17 17:43:07.709 Thread-3 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$toJavaArray$1 21/01/17 17:43:07.709 Thread-3 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$toJavaArray$1) is now cleaned +++ 21/01/17 17:43:07.710 Executor task launch worker for task 2.0 in stage 5.0 (TID 21) DEBUG LocalDiskShuffleMapOutputWriter: Writing shuffle index file for mapId 21 with length 10 21/01/17 17:43:07.710 Executor task launch worker for task 2.0 in stage 5.0 (TID 21) DEBUG IndexShuffleBlockResolver: Shuffle index for mapId 21: [0,245,252,0,0,0,0,0,0,0] 21/01/17 17:43:07.712 Executor task launch worker for task 2.0 in stage 5.0 (TID 21) INFO Executor: Finished task 2.0 in stage 5.0 (TID 21). 1993 bytes result sent to driver 21/01/17 17:43:07.712 Executor task launch worker for task 2.0 in stage 5.0 (TID 21) DEBUG ExecutorMetricsPoller: stageTCMP: (5, 0) -> 3 21/01/17 17:43:07.712 Executor task launch worker for task 0.0 in stage 5.0 (TID 19) DEBUG ExecutorMetricsPoller: stageTCMP: (5, 0) -> 2 21/01/17 17:43:07.713 dispatcher-event-loop-2 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_5.0, runningTasks: 3 21/01/17 17:43:07.713 Executor task launch worker for task 3.0 in stage 5.0 (TID 22) DEBUG LocalDiskShuffleMapOutputWriter: Writing shuffle index file for mapId 22 with length 10 21/01/17 17:43:07.713 dispatcher-event-loop-2 DEBUG TaskSetManager: No tasks for locality level PROCESS_LOCAL, so moving to locality level NODE_LOCAL 21/01/17 17:43:07.713 dispatcher-event-loop-2 DEBUG TaskSetManager: No tasks for locality level NODE_LOCAL, so moving to locality level ANY 21/01/17 17:43:07.713 Executor task launch worker for task 3.0 in stage 5.0 (TID 22) DEBUG IndexShuffleBlockResolver: Shuffle index for mapId 22: [0,247,245,0,0,0,0,0,0,0] 21/01/17 17:43:07.714 Executor task launch worker for task 2.0 in stage 1.0 (TID 3) WARN BLAS: Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS 21/01/17 17:43:07.714 task-result-getter-3 INFO TaskSetManager: Finished task 0.0 in stage 5.0 (TID 19) in 48 ms on 192.168.1.225 (executor driver) (1/4) 21/01/17 17:43:07.714 Executor task launch worker for task 3.0 in stage 5.0 (TID 22) INFO Executor: Finished task 3.0 in stage 5.0 (TID 22). 1993 bytes result sent to driver 21/01/17 17:43:07.714 Executor task launch worker for task 2.0 in stage 1.0 (TID 3) WARN BLAS: Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS 21/01/17 17:43:07.715 Executor task launch worker for task 3.0 in stage 5.0 (TID 22) DEBUG ExecutorMetricsPoller: stageTCMP: (5, 0) -> 1 21/01/17 17:43:07.715 task-result-getter-1 INFO TaskSetManager: Finished task 3.0 in stage 5.0 (TID 22) in 48 ms on 192.168.1.225 (executor driver) (2/4) 21/01/17 17:43:07.715 Executor task launch worker for task 1.0 in stage 5.0 (TID 20) DEBUG LocalDiskShuffleMapOutputWriter: Writing shuffle index file for mapId 20 with length 10 21/01/17 17:43:07.715 dag-scheduler-event-loop DEBUG DAGScheduler: ShuffleMapTask finished on driver 21/01/17 17:43:07.716 Executor task launch worker for task 1.0 in stage 5.0 (TID 20) DEBUG IndexShuffleBlockResolver: Shuffle index for mapId 20: [244,0,0,0,0,0,0,0,0,0] 21/01/17 17:43:07.716 task-result-getter-0 INFO TaskSetManager: Finished task 2.0 in stage 5.0 (TID 21) in 49 ms on 192.168.1.225 (executor driver) (3/4) 21/01/17 17:43:07.717 Executor task launch worker for task 1.0 in stage 5.0 (TID 20) INFO Executor: Finished task 1.0 in stage 5.0 (TID 20). 1993 bytes result sent to driver 21/01/17 17:43:07.717 Executor task launch worker for task 1.0 in stage 5.0 (TID 20) DEBUG ExecutorMetricsPoller: removing (5, 0) from stageTCMP 21/01/17 17:43:07.717 dag-scheduler-event-loop DEBUG DAGScheduler: ShuffleMapTask finished on driver 21/01/17 17:43:07.719 dag-scheduler-event-loop DEBUG DAGScheduler: ShuffleMapTask finished on driver 21/01/17 17:43:07.722 task-result-getter-2 INFO TaskSetManager: Finished task 1.0 in stage 5.0 (TID 20) in 55 ms on 192.168.1.225 (executor driver) (4/4) 21/01/17 17:43:07.722 task-result-getter-2 INFO TaskSchedulerImpl: Removed TaskSet 5.0, whose tasks have all completed, from pool 21/01/17 17:43:07.724 dag-scheduler-event-loop DEBUG DAGScheduler: ShuffleMapTask finished on driver 21/01/17 17:43:07.725 dag-scheduler-event-loop INFO DAGScheduler: ShuffleMapStage 5 (map at ALS.scala:1612) finished in 0.092 s 21/01/17 17:43:07.729 Executor task launch worker for task 0.0 in stage 1.0 (TID 1) DEBUG GenerateUnsafeProjection: code for newInstance(class org.apache.spark.ml.linalg.VectorUDT).serialize: /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificUnsafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificUnsafeProjection extends org.apache.spark.sql.catalyst.expressions.UnsafeProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private boolean resultIsNull_0; /* 009 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[] mutableStateArray_1 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[2]; /* 010 */ private org.apache.spark.ml.linalg.Vector[] mutableStateArray_0 = new org.apache.spark.ml.linalg.Vector[1]; /* 011 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[] mutableStateArray_2 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[2]; /* 012 */ /* 013 */ public SpecificUnsafeProjection(Object[] references) { /* 014 */ this.references = references; /* 015 */ /* 016 */ mutableStateArray_1[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(1, 32); /* 017 */ mutableStateArray_1[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(mutableStateArray_1[0], 4); /* 018 */ mutableStateArray_2[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(mutableStateArray_1[1], 4); /* 019 */ mutableStateArray_2[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(mutableStateArray_1[1], 8); /* 020 */ /* 021 */ } /* 022 */ /* 023 */ public void initialize(int partitionIndex) { /* 024 */ /* 025 */ } /* 026 */ /* 027 */ // Scala.Function1 need this /* 028 */ public java.lang.Object apply(java.lang.Object row) { /* 029 */ return apply((InternalRow) row); /* 030 */ } /* 031 */ /* 032 */ public UnsafeRow apply(InternalRow i) { /* 033 */ mutableStateArray_1[0].reset(); /* 034 */ /* 035 */ /* 036 */ mutableStateArray_1[0].zeroOutNullBytes(); /* 037 */ /* 038 */ final org.apache.spark.ml.linalg.VectorUDT value_1 = false ? /* 039 */ null : new org.apache.spark.ml.linalg.VectorUDT(); /* 040 */ boolean isNull_0 = true; /* 041 */ InternalRow value_0 = null; /* 042 */ if (!false) { /* 043 */ resultIsNull_0 = false; /* 044 */ if (!resultIsNull_0) { /* 045 */ boolean isNull_2 = i.isNullAt(0); /* 046 */ org.apache.spark.ml.linalg.Vector value_2 = isNull_2 ? /* 047 */ null : ((org.apache.spark.ml.linalg.Vector)i.get(0, null)); /* 048 */ resultIsNull_0 = isNull_2; /* 049 */ mutableStateArray_0[0] = value_2; /* 050 */ } /* 051 */ /* 052 */ isNull_0 = resultIsNull_0; /* 053 */ if (!isNull_0) { /* 054 */ /* 055 */ Object funcResult_0 = null; /* 056 */ funcResult_0 = value_1.serialize(mutableStateArray_0[0]); /* 057 */ /* 058 */ if (funcResult_0 != null) { /* 059 */ value_0 = (InternalRow) funcResult_0; /* 060 */ } else { /* 061 */ isNull_0 = true; /* 062 */ } /* 063 */ /* 064 */ /* 065 */ } /* 066 */ } /* 067 */ if (isNull_0) { /* 068 */ mutableStateArray_1[0].setNullAt(0); /* 069 */ } else { /* 070 */ final InternalRow tmpInput_0 = value_0; /* 071 */ if (tmpInput_0 instanceof UnsafeRow) { /* 072 */ mutableStateArray_1[0].write(0, (UnsafeRow) tmpInput_0); /* 073 */ } else { /* 074 */ // Remember the current cursor so that we can calculate how many bytes are /* 075 */ // written later. /* 076 */ final int previousCursor_0 = mutableStateArray_1[0].cursor(); /* 077 */ /* 078 */ mutableStateArray_1[1].resetRowWriter(); /* 079 */ /* 080 */ /* 081 */ mutableStateArray_1[1].write(0, (tmpInput_0.getByte(0))); /* 082 */ /* 083 */ /* 084 */ if ((tmpInput_0.isNullAt(1))) { /* 085 */ mutableStateArray_1[1].setNullAt(1); /* 086 */ } else { /* 087 */ mutableStateArray_1[1].write(1, (tmpInput_0.getInt(1))); /* 088 */ } /* 089 */ /* 090 */ /* 091 */ if ((tmpInput_0.isNullAt(2))) { /* 092 */ mutableStateArray_1[1].setNullAt(2); /* 093 */ } else { /* 094 */ // Remember the current cursor so that we can calculate how many bytes are /* 095 */ // written later. /* 096 */ final int previousCursor_1 = mutableStateArray_1[1].cursor(); /* 097 */ /* 098 */ final ArrayData tmpInput_1 = (tmpInput_0.getArray(2)); /* 099 */ if (tmpInput_1 instanceof UnsafeArrayData) { /* 100 */ mutableStateArray_1[1].write((UnsafeArrayData) tmpInput_1); /* 101 */ } else { /* 102 */ final int numElements_0 = tmpInput_1.numElements(); /* 103 */ mutableStateArray_2[0].initialize(numElements_0); /* 104 */ /* 105 */ for (int index_0 = 0; index_0 < numElements_0; index_0++) { /* 106 */ mutableStateArray_2[0].write(index_0, tmpInput_1.getInt(index_0)); /* 107 */ } /* 108 */ } /* 109 */ /* 110 */ mutableStateArray_1[1].setOffsetAndSizeFromPreviousCursor(2, previousCursor_1); /* 111 */ } /* 112 */ /* 113 */ /* 114 */ if ((tmpInput_0.isNullAt(3))) { /* 115 */ mutableStateArray_1[1].setNullAt(3); /* 116 */ } else { /* 117 */ // Remember the current cursor so that we can calculate how many bytes are /* 118 */ // written later. /* 119 */ final int previousCursor_2 = mutableStateArray_1[1].cursor(); /* 120 */ /* 121 */ final ArrayData tmpInput_2 = (tmpInput_0.getArray(3)); /* 122 */ if (tmpInput_2 instanceof UnsafeArrayData) { /* 123 */ mutableStateArray_1[1].write((UnsafeArrayData) tmpInput_2); /* 124 */ } else { /* 125 */ final int numElements_1 = tmpInput_2.numElements(); /* 126 */ mutableStateArray_2[1].initialize(numElements_1); /* 127 */ /* 128 */ for (int index_1 = 0; index_1 < numElements_1; index_1++) { /* 129 */ mutableStateArray_2[1].write(index_1, tmpInput_2.getDouble(index_1)); /* 130 */ } /* 131 */ } /* 132 */ /* 133 */ mutableStateArray_1[1].setOffsetAndSizeFromPreviousCursor(3, previousCursor_2); /* 134 */ } /* 135 */ /* 136 */ /* 137 */ mutableStateArray_1[0].setOffsetAndSizeFromPreviousCursor(0, previousCursor_0); /* 138 */ } /* 139 */ } /* 140 */ return (mutableStateArray_1[0].getRow()); /* 141 */ } /* 142 */ /* 143 */ /* 144 */ } 21/01/17 17:43:07.729 dag-scheduler-event-loop INFO DAGScheduler: looking for newly runnable stages 21/01/17 17:43:07.730 dag-scheduler-event-loop INFO DAGScheduler: running: Set() 21/01/17 17:43:07.730 dag-scheduler-event-loop INFO DAGScheduler: waiting: Set(ResultStage 6) 21/01/17 17:43:07.730 dag-scheduler-event-loop INFO DAGScheduler: failed: Set() 21/01/17 17:43:07.730 dag-scheduler-event-loop DEBUG MapOutputTrackerMaster: Increasing epoch to 3 21/01/17 17:43:07.730 dag-scheduler-event-loop DEBUG DAGScheduler: submitStage(ResultStage 6 (name=count at ALS.scala:980;jobs=2)) 21/01/17 17:43:07.731 Thread-3 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$applySchemaToPythonRDD$1 21/01/17 17:43:07.731 dag-scheduler-event-loop DEBUG DAGScheduler: missing: List() 21/01/17 17:43:07.731 dag-scheduler-event-loop INFO DAGScheduler: Submitting ResultStage 6 (itemOutBlocks MapPartitionsRDD[25] at mapValues at ALS.scala:1649), which has no missing parents 21/01/17 17:43:07.731 dag-scheduler-event-loop DEBUG DAGScheduler: submitMissingTasks(ResultStage 6) 21/01/17 17:43:07.737 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_5 stored as values in memory (estimated size 34.8 KiB, free 366.0 MiB) 21/01/17 17:43:07.737 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_5 locally took 1 ms 21/01/17 17:43:07.737 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_5 without replication took 1 ms 21/01/17 17:43:07.738 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_5_piece0 stored as bytes in memory (estimated size 16.2 KiB, free 366.0 MiB) 21/01/17 17:43:07.738 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master broadcast_5_piece0 for BlockManagerId(driver, 192.168.1.225, 40985, None) 21/01/17 17:43:07.736 Executor task launch worker for task 1.0 in stage 1.0 (TID 2) DEBUG GenerateUnsafeProjection: code for newInstance(class org.apache.spark.ml.linalg.VectorUDT).serialize: /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificUnsafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificUnsafeProjection extends org.apache.spark.sql.catalyst.expressions.UnsafeProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private boolean resultIsNull_0; /* 009 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[] mutableStateArray_1 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[2]; /* 010 */ private org.apache.spark.ml.linalg.Vector[] mutableStateArray_0 = new org.apache.spark.ml.linalg.Vector[1]; /* 011 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[] mutableStateArray_2 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[2]; /* 012 */ /* 013 */ public SpecificUnsafeProjection(Object[] references) { /* 014 */ this.references = references; /* 015 */ /* 016 */ mutableStateArray_1[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(1, 32); /* 017 */ mutableStateArray_1[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(mutableStateArray_1[0], 4); /* 018 */ mutableStateArray_2[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(mutableStateArray_1[1], 4); /* 019 */ mutableStateArray_2[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(mutableStateArray_1[1], 8); /* 020 */ /* 021 */ } /* 022 */ /* 023 */ public void initialize(int partitionIndex) { /* 024 */ /* 025 */ } /* 026 */ /* 027 */ // Scala.Function1 need this /* 028 */ public java.lang.Object apply(java.lang.Object row) { /* 029 */ return apply((InternalRow) row); /* 030 */ } /* 031 */ /* 032 */ public UnsafeRow apply(InternalRow i) { /* 033 */ mutableStateArray_1[0].reset(); /* 034 */ /* 035 */ /* 036 */ mutableStateArray_1[0].zeroOutNullBytes(); /* 037 */ /* 038 */ final org.apache.spark.ml.linalg.VectorUDT value_1 = false ? /* 039 */ null : new org.apache.spark.ml.linalg.VectorUDT(); /* 040 */ boolean isNull_0 = true; /* 041 */ InternalRow value_0 = null; /* 042 */ if (!false) { /* 043 */ resultIsNull_0 = false; /* 044 */ if (!resultIsNull_0) { /* 045 */ boolean isNull_2 = i.isNullAt(0); /* 046 */ org.apache.spark.ml.linalg.Vector value_2 = isNull_2 ? /* 047 */ null : ((org.apache.spark.ml.linalg.Vector)i.get(0, null)); /* 048 */ resultIsNull_0 = isNull_2; /* 049 */ mutableStateArray_0[0] = value_2; /* 050 */ } /* 051 */ /* 052 */ isNull_0 = resultIsNull_0; /* 053 */ if (!isNull_0) { /* 054 */ /* 055 */ Object funcResult_0 = null; /* 056 */ funcResult_0 = value_1.serialize(mutableStateArray_0[0]); /* 057 */ /* 058 */ if (funcResult_0 != null) { /* 059 */ value_0 = (InternalRow) funcResult_0; /* 060 */ } else { /* 061 */ isNull_0 = true; /* 062 */ } /* 063 */ /* 064 */ /* 065 */ } /* 066 */ } /* 067 */ if (isNull_0) { /* 068 */ mutableStateArray_1[0].setNullAt(0); /* 069 */ } else { /* 070 */ final InternalRow tmpInput_0 = value_0; /* 071 */ if (tmpInput_0 instanceof UnsafeRow) { /* 072 */ mutableStateArray_1[0].write(0, (UnsafeRow) tmpInput_0); /* 073 */ } else { /* 074 */ // Remember the current cursor so that we can calculate how many bytes are /* 075 */ // written later. /* 076 */ final int previousCursor_0 = mutableStateArray_1[0].cursor(); /* 077 */ /* 078 */ mutableStateArray_1[1].resetRowWriter(); /* 079 */ /* 080 */ /* 081 */ mutableStateArray_1[1].write(0, (tmpInput_0.getByte(0))); /* 082 */ /* 083 */ /* 084 */ if ((tmpInput_0.isNullAt(1))) { /* 085 */ mutableStateArray_1[1].setNullAt(1); /* 086 */ } else { /* 087 */ mutableStateArray_1[1].write(1, (tmpInput_0.getInt(1))); /* 088 */ } /* 089 */ /* 090 */ /* 091 */ if ((tmpInput_0.isNullAt(2))) { /* 092 */ mutableStateArray_1[1].setNullAt(2); /* 093 */ } else { /* 094 */ // Remember the current cursor so that we can calculate how many bytes are /* 095 */ // written later. /* 096 */ final int previousCursor_1 = mutableStateArray_1[1].cursor(); /* 097 */ /* 098 */ final ArrayData tmpInput_1 = (tmpInput_0.getArray(2)); /* 099 */ if (tmpInput_1 instanceof UnsafeArrayData) { /* 100 */ mutableStateArray_1[1].write((UnsafeArrayData) tmpInput_1); /* 101 */ } else { /* 102 */ final int numElements_0 = tmpInput_1.numElements(); /* 103 */ mutableStateArray_2[0].initialize(numElements_0); /* 104 */ /* 105 */ for (int index_0 = 0; index_0 < numElements_0; index_0++) { /* 106 */ mutableStateArray_2[0].write(index_0, tmpInput_1.getInt(index_0)); /* 107 */ } /* 108 */ } /* 109 */ /* 110 */ mutableStateArray_1[1].setOffsetAndSizeFromPreviousCursor(2, previousCursor_1); /* 111 */ } /* 112 */ /* 113 */ /* 114 */ if ((tmpInput_0.isNullAt(3))) { /* 115 */ mutableStateArray_1[1].setNullAt(3); /* 116 */ } else { /* 117 */ // Remember the current cursor so that we can calculate how many bytes are /* 118 */ // written later. /* 119 */ final int previousCursor_2 = mutableStateArray_1[1].cursor(); /* 120 */ /* 121 */ final ArrayData tmpInput_2 = (tmpInput_0.getArray(3)); /* 122 */ if (tmpInput_2 instanceof UnsafeArrayData) { /* 123 */ mutableStateArray_1[1].write((UnsafeArrayData) tmpInput_2); /* 124 */ } else { /* 125 */ final int numElements_1 = tmpInput_2.numElements(); /* 126 */ mutableStateArray_2[1].initialize(numElements_1); /* 127 */ /* 128 */ for (int index_1 = 0; index_1 < numElements_1; index_1++) { /* 129 */ mutableStateArray_2[1].write(index_1, tmpInput_2.getDouble(index_1)); /* 130 */ } /* 131 */ } /* 132 */ /* 133 */ mutableStateArray_1[1].setOffsetAndSizeFromPreviousCursor(3, previousCursor_2); /* 134 */ } /* 135 */ /* 136 */ /* 137 */ mutableStateArray_1[0].setOffsetAndSizeFromPreviousCursor(0, previousCursor_0); /* 138 */ } /* 139 */ } /* 140 */ return (mutableStateArray_1[0].getRow()); /* 141 */ } /* 142 */ /* 143 */ /* 144 */ } 21/01/17 17:43:07.739 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added broadcast_5_piece0 in memory on 192.168.1.225:40985 (size: 16.2 KiB, free: 366.2 MiB) 21/01/17 17:43:07.739 dag-scheduler-event-loop DEBUG BlockManagerMaster: Updated info of block broadcast_5_piece0 21/01/17 17:43:07.739 dag-scheduler-event-loop DEBUG BlockManager: Told master about block broadcast_5_piece0 21/01/17 17:43:07.739 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_5_piece0 locally took 1 ms 21/01/17 17:43:07.739 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_5_piece0 without replication took 1 ms 21/01/17 17:43:07.740 dag-scheduler-event-loop INFO SparkContext: Created broadcast 5 from broadcast at DAGScheduler.scala:1383 21/01/17 17:43:07.740 dag-scheduler-event-loop INFO DAGScheduler: Submitting 10 missing tasks from ResultStage 6 (itemOutBlocks MapPartitionsRDD[25] at mapValues at ALS.scala:1649) (first 15 tasks are for partitions Vector(0, 1, 2, 3, 4, 5, 6, 7, 8, 9)) 21/01/17 17:43:07.740 dag-scheduler-event-loop INFO TaskSchedulerImpl: Adding task set 6.0 with 10 tasks resource profile 0 21/01/17 17:43:07.741 dag-scheduler-event-loop DEBUG TaskSetManager: Epoch for TaskSet 6.0: 3 21/01/17 17:43:07.741 dag-scheduler-event-loop DEBUG TaskSetManager: Adding pending tasks took 0 ms 21/01/17 17:43:07.741 dag-scheduler-event-loop DEBUG TaskSetManager: Valid locality levels for TaskSet 6.0: NODE_LOCAL, NO_PREF, ANY 21/01/17 17:43:07.742 dispatcher-event-loop-3 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_6.0, runningTasks: 0 21/01/17 17:43:07.742 dispatcher-event-loop-3 INFO TaskSetManager: Starting task 0.0 in stage 6.0 (TID 23) (192.168.1.225, executor driver, partition 0, NODE_LOCAL, 7143 bytes) taskResourceAssignments Map() 21/01/17 17:43:07.743 dispatcher-event-loop-3 INFO TaskSetManager: Starting task 1.0 in stage 6.0 (TID 24) (192.168.1.225, executor driver, partition 1, NODE_LOCAL, 7143 bytes) taskResourceAssignments Map() 21/01/17 17:43:07.743 dispatcher-event-loop-3 INFO TaskSetManager: Starting task 2.0 in stage 6.0 (TID 25) (192.168.1.225, executor driver, partition 2, NODE_LOCAL, 7143 bytes) taskResourceAssignments Map() 21/01/17 17:43:07.743 dispatcher-event-loop-3 DEBUG TaskSetManager: No tasks for locality level NODE_LOCAL, so moving to locality level NO_PREF 21/01/17 17:43:07.743 dispatcher-event-loop-3 DEBUG TaskSetManager: Moving to ANY after waiting for 0ms 21/01/17 17:43:07.743 dispatcher-event-loop-3 INFO TaskSetManager: Starting task 3.0 in stage 6.0 (TID 26) (192.168.1.225, executor driver, partition 3, PROCESS_LOCAL, 7143 bytes) taskResourceAssignments Map() 21/01/17 17:43:07.744 Executor task launch worker for task 0.0 in stage 6.0 (TID 23) INFO Executor: Running task 0.0 in stage 6.0 (TID 23) 21/01/17 17:43:07.744 Executor task launch worker for task 3.0 in stage 6.0 (TID 26) INFO Executor: Running task 3.0 in stage 6.0 (TID 26) 21/01/17 17:43:07.745 Executor task launch worker for task 0.0 in stage 6.0 (TID 23) DEBUG ExecutorMetricsPoller: stageTCMP: (6, 0) -> 1 21/01/17 17:43:07.744 Executor task launch worker for task 2.0 in stage 6.0 (TID 25) INFO Executor: Running task 2.0 in stage 6.0 (TID 25) 21/01/17 17:43:07.745 Executor task launch worker for task 2.0 in stage 6.0 (TID 25) DEBUG ExecutorMetricsPoller: stageTCMP: (6, 0) -> 3 21/01/17 17:43:07.745 Executor task launch worker for task 0.0 in stage 6.0 (TID 23) DEBUG BlockManager: Getting local block broadcast_5 21/01/17 17:43:07.746 Executor task launch worker for task 0.0 in stage 6.0 (TID 23) DEBUG BlockManager: Level for block broadcast_5 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:07.745 Executor task launch worker for task 3.0 in stage 6.0 (TID 26) DEBUG ExecutorMetricsPoller: stageTCMP: (6, 0) -> 2 21/01/17 17:43:07.747 Executor task launch worker for task 1.0 in stage 6.0 (TID 24) INFO Executor: Running task 1.0 in stage 6.0 (TID 24) 21/01/17 17:43:07.748 Executor task launch worker for task 1.0 in stage 6.0 (TID 24) DEBUG ExecutorMetricsPoller: stageTCMP: (6, 0) -> 4 21/01/17 17:43:07.753 Executor task launch worker for task 0.0 in stage 6.0 (TID 23) DEBUG BlockManager: Getting local block rdd_25_0 21/01/17 17:43:07.753 Executor task launch worker for task 0.0 in stage 6.0 (TID 23) DEBUG BlockManager: Block rdd_25_0 was not found 21/01/17 17:43:07.753 Executor task launch worker for task 0.0 in stage 6.0 (TID 23) DEBUG BlockManager: Getting remote block rdd_25_0 21/01/17 17:43:07.753 Executor task launch worker for task 3.0 in stage 6.0 (TID 26) DEBUG BlockManager: Getting local block rdd_25_3 21/01/17 17:43:07.753 Thread-3 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$applySchemaToPythonRDD$1) is now cleaned +++ 21/01/17 17:43:07.754 Executor task launch worker for task 1.0 in stage 6.0 (TID 24) DEBUG BlockManager: Getting local block rdd_25_1 21/01/17 17:43:07.744 Executor task launch worker for task 3.0 in stage 1.0 (TID 4) DEBUG GenerateUnsafeProjection: code for newInstance(class org.apache.spark.ml.linalg.VectorUDT).serialize: /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificUnsafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificUnsafeProjection extends org.apache.spark.sql.catalyst.expressions.UnsafeProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private boolean resultIsNull_0; /* 009 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[] mutableStateArray_1 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[2]; /* 010 */ private org.apache.spark.ml.linalg.Vector[] mutableStateArray_0 = new org.apache.spark.ml.linalg.Vector[1]; /* 011 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[] mutableStateArray_2 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[2]; /* 012 */ /* 013 */ public SpecificUnsafeProjection(Object[] references) { /* 014 */ this.references = references; /* 015 */ /* 016 */ mutableStateArray_1[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(1, 32); /* 017 */ mutableStateArray_1[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(mutableStateArray_1[0], 4); /* 018 */ mutableStateArray_2[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(mutableStateArray_1[1], 4); /* 019 */ mutableStateArray_2[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(mutableStateArray_1[1], 8); /* 020 */ /* 021 */ } /* 022 */ /* 023 */ public void initialize(int partitionIndex) { /* 024 */ /* 025 */ } /* 026 */ /* 027 */ // Scala.Function1 need this /* 028 */ public java.lang.Object apply(java.lang.Object row) { /* 029 */ return apply((InternalRow) row); /* 030 */ } /* 031 */ /* 032 */ public UnsafeRow apply(InternalRow i) { /* 033 */ mutableStateArray_1[0].reset(); /* 034 */ /* 035 */ /* 036 */ mutableStateArray_1[0].zeroOutNullBytes(); /* 037 */ /* 038 */ final org.apache.spark.ml.linalg.VectorUDT value_1 = false ? /* 039 */ null : new org.apache.spark.ml.linalg.VectorUDT(); /* 040 */ boolean isNull_0 = true; /* 041 */ InternalRow value_0 = null; /* 042 */ if (!false) { /* 043 */ resultIsNull_0 = false; /* 044 */ if (!resultIsNull_0) { /* 045 */ boolean isNull_2 = i.isNullAt(0); /* 046 */ org.apache.spark.ml.linalg.Vector value_2 = isNull_2 ? /* 047 */ null : ((org.apache.spark.ml.linalg.Vector)i.get(0, null)); /* 048 */ resultIsNull_0 = isNull_2; /* 049 */ mutableStateArray_0[0] = value_2; /* 050 */ } /* 051 */ /* 052 */ isNull_0 = resultIsNull_0; /* 053 */ if (!isNull_0) { /* 054 */ /* 055 */ Object funcResult_0 = null; /* 056 */ funcResult_0 = value_1.serialize(mutableStateArray_0[0]); /* 057 */ /* 058 */ if (funcResult_0 != null) { /* 059 */ value_0 = (InternalRow) funcResult_0; /* 060 */ } else { /* 061 */ isNull_0 = true; /* 062 */ } /* 063 */ /* 064 */ /* 065 */ } /* 066 */ } /* 067 */ if (isNull_0) { /* 068 */ mutableStateArray_1[0].setNullAt(0); /* 069 */ } else { /* 070 */ final InternalRow tmpInput_0 = value_0; /* 071 */ if (tmpInput_0 instanceof UnsafeRow) { /* 072 */ mutableStateArray_1[0].write(0, (UnsafeRow) tmpInput_0); /* 073 */ } else { /* 074 */ // Remember the current cursor so that we can calculate how many bytes are /* 075 */ // written later. /* 076 */ final int previousCursor_0 = mutableStateArray_1[0].cursor(); /* 077 */ /* 078 */ mutableStateArray_1[1].resetRowWriter(); /* 079 */ /* 080 */ /* 081 */ mutableStateArray_1[1].write(0, (tmpInput_0.getByte(0))); /* 082 */ /* 083 */ /* 084 */ if ((tmpInput_0.isNullAt(1))) { /* 085 */ mutableStateArray_1[1].setNullAt(1); /* 086 */ } else { /* 087 */ mutableStateArray_1[1].write(1, (tmpInput_0.getInt(1))); /* 088 */ } /* 089 */ /* 090 */ /* 091 */ if ((tmpInput_0.isNullAt(2))) { /* 092 */ mutableStateArray_1[1].setNullAt(2); /* 093 */ } else { /* 094 */ // Remember the current cursor so that we can calculate how many bytes are /* 095 */ // written later. /* 096 */ final int previousCursor_1 = mutableStateArray_1[1].cursor(); /* 097 */ /* 098 */ final ArrayData tmpInput_1 = (tmpInput_0.getArray(2)); /* 099 */ if (tmpInput_1 instanceof UnsafeArrayData) { /* 100 */ mutableStateArray_1[1].write((UnsafeArrayData) tmpInput_1); /* 101 */ } else { /* 102 */ final int numElements_0 = tmpInput_1.numElements(); /* 103 */ mutableStateArray_2[0].initialize(numElements_0); /* 104 */ /* 105 */ for (int index_0 = 0; index_0 < numElements_0; index_0++) { /* 106 */ mutableStateArray_2[0].write(index_0, tmpInput_1.getInt(index_0)); /* 107 */ } /* 108 */ } /* 109 */ /* 110 */ mutableStateArray_1[1].setOffsetAndSizeFromPreviousCursor(2, previousCursor_1); /* 111 */ } /* 112 */ /* 113 */ /* 114 */ if ((tmpInput_0.isNullAt(3))) { /* 115 */ mutableStateArray_1[1].setNullAt(3); /* 116 */ } else { /* 117 */ // Remember the current cursor so that we can calculate how many bytes are /* 118 */ // written later. /* 119 */ final int previousCursor_2 = mutableStateArray_1[1].cursor(); /* 120 */ /* 121 */ final ArrayData tmpInput_2 = (tmpInput_0.getArray(3)); /* 122 */ if (tmpInput_2 instanceof UnsafeArrayData) { /* 123 */ mutableStateArray_1[1].write((UnsafeArrayData) tmpInput_2); /* 124 */ } else { /* 125 */ final int numElements_1 = tmpInput_2.numElements(); /* 126 */ mutableStateArray_2[1].initialize(numElements_1); /* 127 */ /* 128 */ for (int index_1 = 0; index_1 < numElements_1; index_1++) { /* 129 */ mutableStateArray_2[1].write(index_1, tmpInput_2.getDouble(index_1)); /* 130 */ } /* 131 */ } /* 132 */ /* 133 */ mutableStateArray_1[1].setOffsetAndSizeFromPreviousCursor(3, previousCursor_2); /* 134 */ } /* 135 */ /* 136 */ /* 137 */ mutableStateArray_1[0].setOffsetAndSizeFromPreviousCursor(0, previousCursor_0); /* 138 */ } /* 139 */ } /* 140 */ return (mutableStateArray_1[0].getRow()); /* 141 */ } /* 142 */ /* 143 */ /* 144 */ } 21/01/17 17:43:07.759 Executor task launch worker for task 2.0 in stage 6.0 (TID 25) DEBUG BlockManager: Getting local block rdd_25_2 21/01/17 17:43:07.751 Executor task launch worker for task 0.0 in stage 1.0 (TID 1) DEBUG CodeGenerator: /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificUnsafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificUnsafeProjection extends org.apache.spark.sql.catalyst.expressions.UnsafeProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private boolean resultIsNull_0; /* 009 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[] mutableStateArray_1 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[2]; /* 010 */ private org.apache.spark.ml.linalg.Vector[] mutableStateArray_0 = new org.apache.spark.ml.linalg.Vector[1]; /* 011 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[] mutableStateArray_2 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[2]; /* 012 */ /* 013 */ public SpecificUnsafeProjection(Object[] references) { /* 014 */ this.references = references; /* 015 */ /* 016 */ mutableStateArray_1[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(1, 32); /* 017 */ mutableStateArray_1[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(mutableStateArray_1[0], 4); /* 018 */ mutableStateArray_2[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(mutableStateArray_1[1], 4); /* 019 */ mutableStateArray_2[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(mutableStateArray_1[1], 8); /* 020 */ /* 021 */ } /* 022 */ /* 023 */ public void initialize(int partitionIndex) { /* 024 */ /* 025 */ } /* 026 */ /* 027 */ // Scala.Function1 need this /* 028 */ public java.lang.Object apply(java.lang.Object row) { /* 029 */ return apply((InternalRow) row); /* 030 */ } /* 031 */ /* 032 */ public UnsafeRow apply(InternalRow i) { /* 033 */ mutableStateArray_1[0].reset(); /* 034 */ /* 035 */ /* 036 */ mutableStateArray_1[0].zeroOutNullBytes(); /* 037 */ /* 038 */ final org.apache.spark.ml.linalg.VectorUDT value_1 = false ? /* 039 */ null : new org.apache.spark.ml.linalg.VectorUDT(); /* 040 */ boolean isNull_0 = true; /* 041 */ InternalRow value_0 = null; /* 042 */ if (!false) { /* 043 */ resultIsNull_0 = false; /* 044 */ if (!resultIsNull_0) { /* 045 */ boolean isNull_2 = i.isNullAt(0); /* 046 */ org.apache.spark.ml.linalg.Vector value_2 = isNull_2 ? /* 047 */ null : ((org.apache.spark.ml.linalg.Vector)i.get(0, null)); /* 048 */ resultIsNull_0 = isNull_2; /* 049 */ mutableStateArray_0[0] = value_2; /* 050 */ } /* 051 */ /* 052 */ isNull_0 = resultIsNull_0; /* 053 */ if (!isNull_0) { /* 054 */ /* 055 */ Object funcResult_0 = null; /* 056 */ funcResult_0 = value_1.serialize(mutableStateArray_0[0]); /* 057 */ /* 058 */ if (funcResult_0 != null) { /* 059 */ value_0 = (InternalRow) funcResult_0; /* 060 */ } else { /* 061 */ isNull_0 = true; /* 062 */ } /* 063 */ /* 064 */ /* 065 */ } /* 066 */ } /* 067 */ if (isNull_0) { /* 068 */ mutableStateArray_1[0].setNullAt(0); /* 069 */ } else { /* 070 */ final InternalRow tmpInput_0 = value_0; /* 071 */ if (tmpInput_0 instanceof UnsafeRow) { /* 072 */ mutableStateArray_1[0].write(0, (UnsafeRow) tmpInput_0); /* 073 */ } else { /* 074 */ // Remember the current cursor so that we can calculate how many bytes are /* 075 */ // written later. /* 076 */ final int previousCursor_0 = mutableStateArray_1[0].cursor(); /* 077 */ /* 078 */ mutableStateArray_1[1].resetRowWriter(); /* 079 */ /* 080 */ /* 081 */ mutableStateArray_1[1].write(0, (tmpInput_0.getByte(0))); /* 082 */ /* 083 */ /* 084 */ if ((tmpInput_0.isNullAt(1))) { /* 085 */ mutableStateArray_1[1].setNullAt(1); /* 086 */ } else { /* 087 */ mutableStateArray_1[1].write(1, (tmpInput_0.getInt(1))); /* 088 */ } /* 089 */ /* 090 */ /* 091 */ if ((tmpInput_0.isNullAt(2))) { /* 092 */ mutableStateArray_1[1].setNullAt(2); /* 093 */ } else { /* 094 */ // Remember the current cursor so that we can calculate how many bytes are /* 095 */ // written later. /* 096 */ final int previousCursor_1 = mutableStateArray_1[1].cursor(); /* 097 */ /* 098 */ final ArrayData tmpInput_1 = (tmpInput_0.getArray(2)); /* 099 */ if (tmpInput_1 instanceof UnsafeArrayData) { /* 100 */ mutableStateArray_1[1].write((UnsafeArrayData) tmpInput_1); /* 101 */ } else { /* 102 */ final int numElements_0 = tmpInput_1.numElements(); /* 103 */ mutableStateArray_2[0].initialize(numElements_0); /* 104 */ /* 105 */ for (int index_0 = 0; index_0 < numElements_0; index_0++) { /* 106 */ mutableStateArray_2[0].write(index_0, tmpInput_1.getInt(index_0)); /* 107 */ } /* 108 */ } /* 109 */ /* 110 */ mutableStateArray_1[1].setOffsetAndSizeFromPreviousCursor(2, previousCursor_1); /* 111 */ } /* 112 */ /* 113 */ /* 114 */ if ((tmpInput_0.isNullAt(3))) { /* 115 */ mutableStateArray_1[1].setNullAt(3); /* 116 */ } else { /* 117 */ // Remember the current cursor so that we can calculate how many bytes are /* 118 */ // written later. /* 119 */ final int previousCursor_2 = mutableStateArray_1[1].cursor(); /* 120 */ /* 121 */ final ArrayData tmpInput_2 = (tmpInput_0.getArray(3)); /* 122 */ if (tmpInput_2 instanceof UnsafeArrayData) { /* 123 */ mutableStateArray_1[1].write((UnsafeArrayData) tmpInput_2); /* 124 */ } else { /* 125 */ final int numElements_1 = tmpInput_2.numElements(); /* 126 */ mutableStateArray_2[1].initialize(numElements_1); /* 127 */ /* 128 */ for (int index_1 = 0; index_1 < numElements_1; index_1++) { /* 129 */ mutableStateArray_2[1].write(index_1, tmpInput_2.getDouble(index_1)); /* 130 */ } /* 131 */ } /* 132 */ /* 133 */ mutableStateArray_1[1].setOffsetAndSizeFromPreviousCursor(3, previousCursor_2); /* 134 */ } /* 135 */ /* 136 */ /* 137 */ mutableStateArray_1[0].setOffsetAndSizeFromPreviousCursor(0, previousCursor_0); /* 138 */ } /* 139 */ } /* 140 */ return (mutableStateArray_1[0].getRow()); /* 141 */ } /* 142 */ /* 143 */ /* 144 */ } 21/01/17 17:43:07.759 Executor task launch worker for task 3.0 in stage 6.0 (TID 26) DEBUG BlockManager: Block rdd_25_3 was not found 21/01/17 17:43:07.759 Executor task launch worker for task 3.0 in stage 6.0 (TID 26) DEBUG BlockManager: Getting remote block rdd_25_3 21/01/17 17:43:07.759 Executor task launch worker for task 2.0 in stage 6.0 (TID 25) DEBUG BlockManager: Block rdd_25_2 was not found 21/01/17 17:43:07.759 Executor task launch worker for task 2.0 in stage 6.0 (TID 25) DEBUG BlockManager: Getting remote block rdd_25_2 21/01/17 17:43:07.759 Executor task launch worker for task 1.0 in stage 6.0 (TID 24) DEBUG BlockManager: Block rdd_25_1 was not found 21/01/17 17:43:07.760 Executor task launch worker for task 1.0 in stage 6.0 (TID 24) DEBUG BlockManager: Getting remote block rdd_25_1 21/01/17 17:43:07.760 Executor task launch worker for task 3.0 in stage 6.0 (TID 26) DEBUG BlockManager: Block rdd_25_3 is unknown by block manager master 21/01/17 17:43:07.760 Executor task launch worker for task 3.0 in stage 6.0 (TID 26) DEBUG BlockManager: Getting local block rdd_24_3 21/01/17 17:43:07.760 Executor task launch worker for task 1.0 in stage 6.0 (TID 24) DEBUG BlockManager: Block rdd_25_1 is unknown by block manager master 21/01/17 17:43:07.760 Executor task launch worker for task 2.0 in stage 6.0 (TID 25) DEBUG BlockManager: Block rdd_25_2 is unknown by block manager master 21/01/17 17:43:07.760 Executor task launch worker for task 1.0 in stage 6.0 (TID 24) DEBUG BlockManager: Getting local block rdd_24_1 21/01/17 17:43:07.760 Executor task launch worker for task 3.0 in stage 6.0 (TID 26) DEBUG BlockManager: Block rdd_24_3 was not found 21/01/17 17:43:07.760 Executor task launch worker for task 1.0 in stage 6.0 (TID 24) DEBUG BlockManager: Block rdd_24_1 was not found 21/01/17 17:43:07.760 Executor task launch worker for task 2.0 in stage 6.0 (TID 25) DEBUG BlockManager: Getting local block rdd_24_2 21/01/17 17:43:07.760 Executor task launch worker for task 1.0 in stage 6.0 (TID 24) DEBUG BlockManager: Getting remote block rdd_24_1 21/01/17 17:43:07.760 Executor task launch worker for task 3.0 in stage 6.0 (TID 26) DEBUG BlockManager: Getting remote block rdd_24_3 21/01/17 17:43:07.760 Executor task launch worker for task 2.0 in stage 6.0 (TID 25) DEBUG BlockManager: Block rdd_24_2 was not found 21/01/17 17:43:07.760 Executor task launch worker for task 2.0 in stage 6.0 (TID 25) DEBUG BlockManager: Getting remote block rdd_24_2 21/01/17 17:43:07.760 Executor task launch worker for task 1.0 in stage 6.0 (TID 24) DEBUG BlockManager: Block rdd_24_1 is unknown by block manager master 21/01/17 17:43:07.760 Executor task launch worker for task 3.0 in stage 6.0 (TID 26) DEBUG BlockManager: Block rdd_24_3 is unknown by block manager master 21/01/17 17:43:07.761 Executor task launch worker for task 3.0 in stage 6.0 (TID 26) DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 2 21/01/17 17:43:07.761 Executor task launch worker for task 1.0 in stage 6.0 (TID 24) DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 2 21/01/17 17:43:07.761 Executor task launch worker for task 2.0 in stage 6.0 (TID 25) DEBUG BlockManager: Block rdd_24_2 is unknown by block manager master 21/01/17 17:43:07.761 Executor task launch worker for task 1.0 in stage 6.0 (TID 24) DEBUG MapOutputTrackerMaster: Convert map statuses for shuffle 2, mappers 0-4, partitions 1-2 21/01/17 17:43:07.761 Executor task launch worker for task 2.0 in stage 6.0 (TID 25) DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 2 21/01/17 17:43:07.761 Executor task launch worker for task 2.0 in stage 6.0 (TID 25) DEBUG MapOutputTrackerMaster: Convert map statuses for shuffle 2, mappers 0-4, partitions 2-3 21/01/17 17:43:07.761 Executor task launch worker for task 1.0 in stage 6.0 (TID 24) DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 21/01/17 17:43:07.762 Executor task launch worker for task 1.0 in stage 6.0 (TID 24) INFO ShuffleBlockFetcherIterator: Getting 3 (753.0 B) non-empty blocks including 3 (753.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 21/01/17 17:43:07.762 Executor task launch worker for task 1.0 in stage 6.0 (TID 24) INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 21/01/17 17:43:07.762 Executor task launch worker for task 1.0 in stage 6.0 (TID 24) DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_2_19_1,0), (shuffle_2_21_1,2), (shuffle_2_22_1,3) 21/01/17 17:43:07.762 Executor task launch worker for task 1.0 in stage 6.0 (TID 24) DEBUG BlockManager: Getting local shuffle block shuffle_2_19_1 21/01/17 17:43:07.762 Executor task launch worker for task 1.0 in stage 6.0 (TID 24) DEBUG BlockManager: Getting local shuffle block shuffle_2_21_1 21/01/17 17:43:07.762 Executor task launch worker for task 1.0 in stage 6.0 (TID 24) DEBUG BlockManager: Getting local shuffle block shuffle_2_22_1 21/01/17 17:43:07.762 Executor task launch worker for task 1.0 in stage 6.0 (TID 24) DEBUG ShuffleBlockFetcherIterator: Got local blocks in 1 ms 21/01/17 17:43:07.765 Executor task launch worker for task 1.0 in stage 6.0 (TID 24) DEBUG ALS: Start sorting an uncompressed in-block of size 3. (sortId = -1691086670) 21/01/17 17:43:07.765 Executor task launch worker for task 1.0 in stage 6.0 (TID 24) DEBUG ALS: Sorting took 3.8271E-5 seconds. (sortId = -1691086670) 21/01/17 17:43:07.765 Executor task launch worker for task 1.0 in stage 6.0 (TID 24) DEBUG TaskMemoryManager: Task 24 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@67c61142 21/01/17 17:43:07.766 Executor task launch worker for task 1.0 in stage 6.0 (TID 24) INFO MemoryStore: Block rdd_24_1 stored as values in memory (estimated size 208.0 B, free 366.0 MiB) 21/01/17 17:43:07.767 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_24_1 for BlockManagerId(driver, 192.168.1.225, 40985, None) 21/01/17 17:43:07.761 Executor task launch worker for task 3.0 in stage 6.0 (TID 26) DEBUG MapOutputTrackerMaster: Convert map statuses for shuffle 2, mappers 0-4, partitions 3-4 21/01/17 17:43:07.767 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_24_1 in memory on 192.168.1.225:40985 (size: 208.0 B, free: 366.2 MiB) 21/01/17 17:43:07.761 Executor task launch worker for task 2.0 in stage 6.0 (TID 25) DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 21/01/17 17:43:07.768 Executor task launch worker for task 2.0 in stage 6.0 (TID 25) INFO ShuffleBlockFetcherIterator: Getting 2 (527.0 B) non-empty blocks including 2 (527.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 21/01/17 17:43:07.768 Executor task launch worker for task 2.0 in stage 6.0 (TID 25) INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 6 ms 21/01/17 17:43:07.768 Executor task launch worker for task 2.0 in stage 6.0 (TID 25) DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_2_21_2,2), (shuffle_2_22_2,3) 21/01/17 17:43:07.768 Executor task launch worker for task 2.0 in stage 6.0 (TID 25) DEBUG BlockManager: Getting local shuffle block shuffle_2_21_2 21/01/17 17:43:07.769 Executor task launch worker for task 2.0 in stage 6.0 (TID 25) DEBUG BlockManager: Getting local shuffle block shuffle_2_22_2 21/01/17 17:43:07.769 Executor task launch worker for task 2.0 in stage 6.0 (TID 25) DEBUG ShuffleBlockFetcherIterator: Got local blocks in 7 ms 21/01/17 17:43:07.771 Executor task launch worker for task 2.0 in stage 6.0 (TID 25) DEBUG ALS: Start sorting an uncompressed in-block of size 2. (sortId = 546257062) 21/01/17 17:43:07.749 Executor task launch worker for task 2.0 in stage 1.0 (TID 3) DEBUG GenerateUnsafeProjection: code for newInstance(class org.apache.spark.ml.linalg.VectorUDT).serialize: /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificUnsafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificUnsafeProjection extends org.apache.spark.sql.catalyst.expressions.UnsafeProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private boolean resultIsNull_0; /* 009 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[] mutableStateArray_1 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[2]; /* 010 */ private org.apache.spark.ml.linalg.Vector[] mutableStateArray_0 = new org.apache.spark.ml.linalg.Vector[1]; /* 011 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[] mutableStateArray_2 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[2]; /* 012 */ /* 013 */ public SpecificUnsafeProjection(Object[] references) { /* 014 */ this.references = references; /* 015 */ /* 016 */ mutableStateArray_1[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(1, 32); /* 017 */ mutableStateArray_1[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(mutableStateArray_1[0], 4); /* 018 */ mutableStateArray_2[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(mutableStateArray_1[1], 4); /* 019 */ mutableStateArray_2[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(mutableStateArray_1[1], 8); /* 020 */ /* 021 */ } /* 022 */ /* 023 */ public void initialize(int partitionIndex) { /* 024 */ /* 025 */ } /* 026 */ /* 027 */ // Scala.Function1 need this /* 028 */ public java.lang.Object apply(java.lang.Object row) { /* 029 */ return apply((InternalRow) row); /* 030 */ } /* 031 */ /* 032 */ public UnsafeRow apply(InternalRow i) { /* 033 */ mutableStateArray_1[0].reset(); /* 034 */ /* 035 */ /* 036 */ mutableStateArray_1[0].zeroOutNullBytes(); /* 037 */ /* 038 */ final org.apache.spark.ml.linalg.VectorUDT value_1 = false ? /* 039 */ null : new org.apache.spark.ml.linalg.VectorUDT(); /* 040 */ boolean isNull_0 = true; /* 041 */ InternalRow value_0 = null; /* 042 */ if (!false) { /* 043 */ resultIsNull_0 = false; /* 044 */ if (!resultIsNull_0) { /* 045 */ boolean isNull_2 = i.isNullAt(0); /* 046 */ org.apache.spark.ml.linalg.Vector value_2 = isNull_2 ? /* 047 */ null : ((org.apache.spark.ml.linalg.Vector)i.get(0, null)); /* 048 */ resultIsNull_0 = isNull_2; /* 049 */ mutableStateArray_0[0] = value_2; /* 050 */ } /* 051 */ /* 052 */ isNull_0 = resultIsNull_0; /* 053 */ if (!isNull_0) { /* 054 */ /* 055 */ Object funcResult_0 = null; /* 056 */ funcResult_0 = value_1.serialize(mutableStateArray_0[0]); /* 057 */ /* 058 */ if (funcResult_0 != null) { /* 059 */ value_0 = (InternalRow) funcResult_0; /* 060 */ } else { /* 061 */ isNull_0 = true; /* 062 */ } /* 063 */ /* 064 */ /* 065 */ } /* 066 */ } /* 067 */ if (isNull_0) { /* 068 */ mutableStateArray_1[0].setNullAt(0); /* 069 */ } else { /* 070 */ final InternalRow tmpInput_0 = value_0; /* 071 */ if (tmpInput_0 instanceof UnsafeRow) { /* 072 */ mutableStateArray_1[0].write(0, (UnsafeRow) tmpInput_0); /* 073 */ } else { /* 074 */ // Remember the current cursor so that we can calculate how many bytes are /* 075 */ // written later. /* 076 */ final int previousCursor_0 = mutableStateArray_1[0].cursor(); /* 077 */ /* 078 */ mutableStateArray_1[1].resetRowWriter(); /* 079 */ /* 080 */ /* 081 */ mutableStateArray_1[1].write(0, (tmpInput_0.getByte(0))); /* 082 */ /* 083 */ /* 084 */ if ((tmpInput_0.isNullAt(1))) { /* 085 */ mutableStateArray_1[1].setNullAt(1); /* 086 */ } else { /* 087 */ mutableStateArray_1[1].write(1, (tmpInput_0.getInt(1))); /* 088 */ } /* 089 */ /* 090 */ /* 091 */ if ((tmpInput_0.isNullAt(2))) { /* 092 */ mutableStateArray_1[1].setNullAt(2); /* 093 */ } else { /* 094 */ // Remember the current cursor so that we can calculate how many bytes are /* 095 */ // written later. /* 096 */ final int previousCursor_1 = mutableStateArray_1[1].cursor(); /* 097 */ /* 098 */ final ArrayData tmpInput_1 = (tmpInput_0.getArray(2)); /* 099 */ if (tmpInput_1 instanceof UnsafeArrayData) { /* 100 */ mutableStateArray_1[1].write((UnsafeArrayData) tmpInput_1); /* 101 */ } else { /* 102 */ final int numElements_0 = tmpInput_1.numElements(); /* 103 */ mutableStateArray_2[0].initialize(numElements_0); /* 104 */ /* 105 */ for (int index_0 = 0; index_0 < numElements_0; index_0++) { /* 106 */ mutableStateArray_2[0].write(index_0, tmpInput_1.getInt(index_0)); /* 107 */ } /* 108 */ } /* 109 */ /* 110 */ mutableStateArray_1[1].setOffsetAndSizeFromPreviousCursor(2, previousCursor_1); /* 111 */ } /* 112 */ /* 113 */ /* 114 */ if ((tmpInput_0.isNullAt(3))) { /* 115 */ mutableStateArray_1[1].setNullAt(3); /* 116 */ } else { /* 117 */ // Remember the current cursor so that we can calculate how many bytes are /* 118 */ // written later. /* 119 */ final int previousCursor_2 = mutableStateArray_1[1].cursor(); /* 120 */ /* 121 */ final ArrayData tmpInput_2 = (tmpInput_0.getArray(3)); /* 122 */ if (tmpInput_2 instanceof UnsafeArrayData) { /* 123 */ mutableStateArray_1[1].write((UnsafeArrayData) tmpInput_2); /* 124 */ } else { /* 125 */ final int numElements_1 = tmpInput_2.numElements(); /* 126 */ mutableStateArray_2[1].initialize(numElements_1); /* 127 */ /* 128 */ for (int index_1 = 0; index_1 < numElements_1; index_1++) { /* 129 */ mutableStateArray_2[1].write(index_1, tmpInput_2.getDouble(index_1)); /* 130 */ } /* 131 */ } /* 132 */ /* 133 */ mutableStateArray_1[1].setOffsetAndSizeFromPreviousCursor(3, previousCursor_2); /* 134 */ } /* 135 */ /* 136 */ /* 137 */ mutableStateArray_1[0].setOffsetAndSizeFromPreviousCursor(0, previousCursor_0); /* 138 */ } /* 139 */ } /* 140 */ return (mutableStateArray_1[0].getRow()); /* 141 */ } /* 142 */ /* 143 */ /* 144 */ } 21/01/17 17:43:07.771 Executor task launch worker for task 2.0 in stage 6.0 (TID 25) DEBUG ALS: Sorting took 3.3031E-5 seconds. (sortId = 546257062) 21/01/17 17:43:07.771 Executor task launch worker for task 2.0 in stage 6.0 (TID 25) DEBUG TaskMemoryManager: Task 25 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@5fd731b8 21/01/17 17:43:07.772 Executor task launch worker for task 2.0 in stage 6.0 (TID 25) INFO MemoryStore: Block rdd_24_2 stored as values in memory (estimated size 192.0 B, free 366.0 MiB) 21/01/17 17:43:07.772 Executor task launch worker for task 3.0 in stage 6.0 (TID 26) DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 21/01/17 17:43:07.773 Executor task launch worker for task 3.0 in stage 6.0 (TID 26) INFO ShuffleBlockFetcherIterator: Getting 0 (0.0 B) non-empty blocks including 0 (0.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 21/01/17 17:43:07.773 Executor task launch worker for task 3.0 in stage 6.0 (TID 26) INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 21/01/17 17:43:07.773 Executor task launch worker for task 3.0 in stage 6.0 (TID 26) DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: 21/01/17 17:43:07.773 Executor task launch worker for task 3.0 in stage 6.0 (TID 26) DEBUG ShuffleBlockFetcherIterator: Got local blocks in 0 ms 21/01/17 17:43:07.773 Executor task launch worker for task 3.0 in stage 6.0 (TID 26) DEBUG TaskMemoryManager: Task 26 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@22ca2d50 21/01/17 17:43:07.774 Executor task launch worker for task 3.0 in stage 6.0 (TID 26) INFO MemoryStore: Block rdd_24_3 stored as values in memory (estimated size 16.0 B, free 366.0 MiB) 21/01/17 17:43:07.777 Executor task launch worker for task 1.0 in stage 6.0 (TID 24) DEBUG BlockManagerMaster: Updated info of block rdd_24_1 21/01/17 17:43:07.777 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_24_3 for BlockManagerId(driver, 192.168.1.225, 40985, None) 21/01/17 17:43:07.778 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_24_3 in memory on 192.168.1.225:40985 (size: 16.0 B, free: 366.2 MiB) 21/01/17 17:43:07.778 Executor task launch worker for task 3.0 in stage 6.0 (TID 26) DEBUG BlockManagerMaster: Updated info of block rdd_24_3 21/01/17 17:43:07.778 Executor task launch worker for task 3.0 in stage 6.0 (TID 26) DEBUG BlockManager: Told master about block rdd_24_3 21/01/17 17:43:07.778 Executor task launch worker for task 3.0 in stage 6.0 (TID 26) DEBUG BlockManager: Put block rdd_24_3 locally took 17 ms 21/01/17 17:43:07.778 Executor task launch worker for task 3.0 in stage 6.0 (TID 26) DEBUG BlockManager: Putting block rdd_24_3 without replication took 17 ms 21/01/17 17:43:07.778 Executor task launch worker for task 3.0 in stage 6.0 (TID 26) DEBUG BlockManager: Getting local block rdd_24_3 21/01/17 17:43:07.778 Executor task launch worker for task 3.0 in stage 6.0 (TID 26) DEBUG BlockManager: Level for block rdd_24_3 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:07.778 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_24_2 for BlockManagerId(driver, 192.168.1.225, 40985, None) 21/01/17 17:43:07.778 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_24_2 in memory on 192.168.1.225:40985 (size: 192.0 B, free: 366.2 MiB) 21/01/17 17:43:07.779 Executor task launch worker for task 0.0 in stage 6.0 (TID 23) DEBUG BlockManager: Block rdd_25_0 is unknown by block manager master 21/01/17 17:43:07.779 Executor task launch worker for task 3.0 in stage 6.0 (TID 26) INFO MemoryStore: Block rdd_25_3 stored as values in memory (estimated size 16.0 B, free 366.0 MiB) 21/01/17 17:43:07.779 Executor task launch worker for task 0.0 in stage 6.0 (TID 23) DEBUG BlockManager: Getting local block rdd_24_0 21/01/17 17:43:07.779 Executor task launch worker for task 0.0 in stage 6.0 (TID 23) DEBUG BlockManager: Block rdd_24_0 was not found 21/01/17 17:43:07.779 Executor task launch worker for task 0.0 in stage 6.0 (TID 23) DEBUG BlockManager: Getting remote block rdd_24_0 21/01/17 17:43:07.779 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_25_3 for BlockManagerId(driver, 192.168.1.225, 40985, None) 21/01/17 17:43:07.779 Executor task launch worker for task 2.0 in stage 6.0 (TID 25) DEBUG BlockManagerMaster: Updated info of block rdd_24_2 21/01/17 17:43:07.779 Executor task launch worker for task 2.0 in stage 6.0 (TID 25) DEBUG BlockManager: Told master about block rdd_24_2 21/01/17 17:43:07.779 Executor task launch worker for task 2.0 in stage 6.0 (TID 25) DEBUG BlockManager: Put block rdd_24_2 locally took 18 ms 21/01/17 17:43:07.779 Executor task launch worker for task 2.0 in stage 6.0 (TID 25) DEBUG BlockManager: Putting block rdd_24_2 without replication took 18 ms 21/01/17 17:43:07.779 Executor task launch worker for task 2.0 in stage 6.0 (TID 25) DEBUG BlockManager: Getting local block rdd_24_2 21/01/17 17:43:07.779 Executor task launch worker for task 1.0 in stage 6.0 (TID 24) DEBUG BlockManager: Told master about block rdd_24_1 21/01/17 17:43:07.779 Executor task launch worker for task 2.0 in stage 6.0 (TID 25) DEBUG BlockManager: Level for block rdd_24_2 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:07.779 Executor task launch worker for task 1.0 in stage 6.0 (TID 24) DEBUG BlockManager: Put block rdd_24_1 locally took 19 ms 21/01/17 17:43:07.780 Executor task launch worker for task 2.0 in stage 6.0 (TID 25) INFO MemoryStore: Block rdd_25_2 stored as values in memory (estimated size 296.0 B, free 366.0 MiB) 21/01/17 17:43:07.779 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_25_3 in memory on 192.168.1.225:40985 (size: 16.0 B, free: 366.2 MiB) 21/01/17 17:43:07.786 Executor task launch worker for task 3.0 in stage 6.0 (TID 26) DEBUG BlockManagerMaster: Updated info of block rdd_25_3 21/01/17 17:43:07.786 Executor task launch worker for task 3.0 in stage 6.0 (TID 26) DEBUG BlockManager: Told master about block rdd_25_3 21/01/17 17:43:07.786 Executor task launch worker for task 3.0 in stage 6.0 (TID 26) DEBUG BlockManager: Put block rdd_25_3 locally took 26 ms 21/01/17 17:43:07.786 Executor task launch worker for task 3.0 in stage 6.0 (TID 26) DEBUG BlockManager: Putting block rdd_25_3 without replication took 26 ms 21/01/17 17:43:07.786 Executor task launch worker for task 3.0 in stage 6.0 (TID 26) DEBUG BlockManager: Getting local block rdd_25_3 21/01/17 17:43:07.786 Executor task launch worker for task 3.0 in stage 6.0 (TID 26) DEBUG BlockManager: Level for block rdd_25_3 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:07.786 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_25_2 for BlockManagerId(driver, 192.168.1.225, 40985, None) 21/01/17 17:43:07.786 Executor task launch worker for task 0.0 in stage 6.0 (TID 23) DEBUG BlockManager: Block rdd_24_0 is unknown by block manager master 21/01/17 17:43:07.787 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_25_2 in memory on 192.168.1.225:40985 (size: 296.0 B, free: 366.2 MiB) 21/01/17 17:43:07.787 Executor task launch worker for task 0.0 in stage 6.0 (TID 23) DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 2 21/01/17 17:43:07.787 Executor task launch worker for task 2.0 in stage 6.0 (TID 25) DEBUG BlockManagerMaster: Updated info of block rdd_25_2 21/01/17 17:43:07.787 Executor task launch worker for task 0.0 in stage 6.0 (TID 23) DEBUG MapOutputTrackerMaster: Convert map statuses for shuffle 2, mappers 0-4, partitions 0-1 21/01/17 17:43:07.794 Executor task launch worker for task 1.0 in stage 6.0 (TID 24) DEBUG BlockManager: Putting block rdd_24_1 without replication took 33 ms 21/01/17 17:43:07.794 Executor task launch worker for task 2.0 in stage 6.0 (TID 25) DEBUG BlockManager: Told master about block rdd_25_2 21/01/17 17:43:07.794 Executor task launch worker for task 2.0 in stage 6.0 (TID 25) DEBUG BlockManager: Put block rdd_25_2 locally took 34 ms 21/01/17 17:43:07.794 Executor task launch worker for task 2.0 in stage 6.0 (TID 25) DEBUG BlockManager: Putting block rdd_25_2 without replication took 34 ms 21/01/17 17:43:07.794 Executor task launch worker for task 2.0 in stage 6.0 (TID 25) DEBUG BlockManager: Getting local block rdd_25_2 21/01/17 17:43:07.795 Executor task launch worker for task 2.0 in stage 6.0 (TID 25) DEBUG BlockManager: Level for block rdd_25_2 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:07.793 Executor task launch worker for task 3.0 in stage 6.0 (TID 26) INFO Executor: Finished task 3.0 in stage 6.0 (TID 26). 2170 bytes result sent to driver 21/01/17 17:43:07.794 Executor task launch worker for task 0.0 in stage 6.0 (TID 23) DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 21/01/17 17:43:07.795 Executor task launch worker for task 3.0 in stage 6.0 (TID 26) DEBUG ExecutorMetricsPoller: stageTCMP: (6, 0) -> 3 21/01/17 17:43:07.796 Executor task launch worker for task 0.0 in stage 6.0 (TID 23) INFO ShuffleBlockFetcherIterator: Getting 1 (251.0 B) non-empty blocks including 1 (251.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 21/01/17 17:43:07.796 Executor task launch worker for task 0.0 in stage 6.0 (TID 23) INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 1 ms 21/01/17 17:43:07.796 Executor task launch worker for task 0.0 in stage 6.0 (TID 23) DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_2_20_0,1) 21/01/17 17:43:07.796 Executor task launch worker for task 2.0 in stage 6.0 (TID 25) INFO Executor: Finished task 2.0 in stage 6.0 (TID 25). 2170 bytes result sent to driver 21/01/17 17:43:07.796 Executor task launch worker for task 2.0 in stage 6.0 (TID 25) DEBUG ExecutorMetricsPoller: stageTCMP: (6, 0) -> 2 21/01/17 17:43:07.797 dispatcher-event-loop-2 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_6.0, runningTasks: 3 21/01/17 17:43:07.797 dispatcher-event-loop-2 DEBUG TaskSetManager: No tasks for locality level NODE_LOCAL, so moving to locality level NO_PREF 21/01/17 17:43:07.797 dispatcher-event-loop-2 DEBUG TaskSetManager: Moving to ANY after waiting for 0ms 21/01/17 17:43:07.797 dispatcher-event-loop-2 INFO TaskSetManager: Starting task 4.0 in stage 6.0 (TID 27) (192.168.1.225, executor driver, partition 4, PROCESS_LOCAL, 7143 bytes) taskResourceAssignments Map() 21/01/17 17:43:07.798 Executor task launch worker for task 4.0 in stage 6.0 (TID 27) INFO Executor: Running task 4.0 in stage 6.0 (TID 27) 21/01/17 17:43:07.798 Executor task launch worker for task 4.0 in stage 6.0 (TID 27) DEBUG ExecutorMetricsPoller: stageTCMP: (6, 0) -> 3 21/01/17 17:43:07.794 Executor task launch worker for task 1.0 in stage 6.0 (TID 24) DEBUG BlockManager: Getting local block rdd_24_1 21/01/17 17:43:07.798 task-result-getter-3 INFO TaskSetManager: Finished task 3.0 in stage 6.0 (TID 26) in 55 ms on 192.168.1.225 (executor driver) (1/10) 21/01/17 17:43:07.796 Executor task launch worker for task 0.0 in stage 6.0 (TID 23) DEBUG BlockManager: Getting local shuffle block shuffle_2_20_0 21/01/17 17:43:07.799 task-result-getter-1 INFO TaskSetManager: Finished task 2.0 in stage 6.0 (TID 25) in 56 ms on 192.168.1.225 (executor driver) (2/10) 21/01/17 17:43:07.799 Executor task launch worker for task 0.0 in stage 6.0 (TID 23) DEBUG ShuffleBlockFetcherIterator: Got local blocks in 4 ms 21/01/17 17:43:07.800 dispatcher-event-loop-2 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_6.0, runningTasks: 3 21/01/17 17:43:07.800 dispatcher-event-loop-2 DEBUG TaskSetManager: No tasks for locality level NODE_LOCAL, so moving to locality level NO_PREF 21/01/17 17:43:07.800 dispatcher-event-loop-2 DEBUG TaskSetManager: Moving to ANY after waiting for 0ms 21/01/17 17:43:07.800 dispatcher-event-loop-2 INFO TaskSetManager: Starting task 5.0 in stage 6.0 (TID 28) (192.168.1.225, executor driver, partition 5, PROCESS_LOCAL, 7143 bytes) taskResourceAssignments Map() 21/01/17 17:43:07.800 Executor task launch worker for task 0.0 in stage 6.0 (TID 23) DEBUG ALS: Start sorting an uncompressed in-block of size 1. (sortId = -1506420410) 21/01/17 17:43:07.801 Executor task launch worker for task 0.0 in stage 6.0 (TID 23) DEBUG ALS: Sorting took 1.504E-5 seconds. (sortId = -1506420410) 21/01/17 17:43:07.801 Executor task launch worker for task 0.0 in stage 6.0 (TID 23) DEBUG TaskMemoryManager: Task 23 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@640385de 21/01/17 17:43:07.801 Executor task launch worker for task 0.0 in stage 6.0 (TID 23) INFO MemoryStore: Block rdd_24_0 stored as values in memory (estimated size 192.0 B, free 366.0 MiB) 21/01/17 17:43:07.802 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_24_0 for BlockManagerId(driver, 192.168.1.225, 40985, None) 21/01/17 17:43:07.802 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_24_0 in memory on 192.168.1.225:40985 (size: 192.0 B, free: 366.2 MiB) 21/01/17 17:43:07.802 Executor task launch worker for task 0.0 in stage 6.0 (TID 23) DEBUG BlockManagerMaster: Updated info of block rdd_24_0 21/01/17 17:43:07.803 Executor task launch worker for task 0.0 in stage 6.0 (TID 23) DEBUG BlockManager: Told master about block rdd_24_0 21/01/17 17:43:07.803 Executor task launch worker for task 0.0 in stage 6.0 (TID 23) DEBUG BlockManager: Put block rdd_24_0 locally took 15 ms 21/01/17 17:43:07.803 Executor task launch worker for task 0.0 in stage 6.0 (TID 23) DEBUG BlockManager: Putting block rdd_24_0 without replication took 16 ms 21/01/17 17:43:07.803 Executor task launch worker for task 0.0 in stage 6.0 (TID 23) DEBUG BlockManager: Getting local block rdd_24_0 21/01/17 17:43:07.803 Executor task launch worker for task 0.0 in stage 6.0 (TID 23) DEBUG BlockManager: Level for block rdd_24_0 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:07.804 Executor task launch worker for task 0.0 in stage 6.0 (TID 23) INFO MemoryStore: Block rdd_25_0 stored as values in memory (estimated size 288.0 B, free 366.0 MiB) 21/01/17 17:43:07.804 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_25_0 for BlockManagerId(driver, 192.168.1.225, 40985, None) 21/01/17 17:43:07.804 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_25_0 in memory on 192.168.1.225:40985 (size: 288.0 B, free: 366.2 MiB) 21/01/17 17:43:07.805 Executor task launch worker for task 0.0 in stage 6.0 (TID 23) DEBUG BlockManagerMaster: Updated info of block rdd_25_0 21/01/17 17:43:07.805 Executor task launch worker for task 0.0 in stage 6.0 (TID 23) DEBUG BlockManager: Told master about block rdd_25_0 21/01/17 17:43:07.805 Executor task launch worker for task 0.0 in stage 6.0 (TID 23) DEBUG BlockManager: Put block rdd_25_0 locally took 25 ms 21/01/17 17:43:07.805 Executor task launch worker for task 0.0 in stage 6.0 (TID 23) DEBUG BlockManager: Putting block rdd_25_0 without replication took 26 ms 21/01/17 17:43:07.805 Executor task launch worker for task 0.0 in stage 6.0 (TID 23) DEBUG BlockManager: Getting local block rdd_25_0 21/01/17 17:43:07.805 Executor task launch worker for task 1.0 in stage 6.0 (TID 24) DEBUG BlockManager: Level for block rdd_24_1 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:07.805 Executor task launch worker for task 0.0 in stage 6.0 (TID 23) DEBUG BlockManager: Level for block rdd_25_0 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:07.806 Executor task launch worker for task 4.0 in stage 6.0 (TID 27) DEBUG BlockManager: Getting local block rdd_25_4 21/01/17 17:43:07.806 Executor task launch worker for task 4.0 in stage 6.0 (TID 27) DEBUG BlockManager: Block rdd_25_4 was not found 21/01/17 17:43:07.806 Executor task launch worker for task 4.0 in stage 6.0 (TID 27) DEBUG BlockManager: Getting remote block rdd_25_4 21/01/17 17:43:07.806 Executor task launch worker for task 4.0 in stage 6.0 (TID 27) DEBUG BlockManager: Block rdd_25_4 is unknown by block manager master 21/01/17 17:43:07.806 Executor task launch worker for task 1.0 in stage 6.0 (TID 24) INFO MemoryStore: Block rdd_25_1 stored as values in memory (estimated size 304.0 B, free 366.0 MiB) 21/01/17 17:43:07.806 Executor task launch worker for task 0.0 in stage 6.0 (TID 23) INFO Executor: Finished task 0.0 in stage 6.0 (TID 23). 2170 bytes result sent to driver 21/01/17 17:43:07.806 Executor task launch worker for task 0.0 in stage 6.0 (TID 23) DEBUG ExecutorMetricsPoller: stageTCMP: (6, 0) -> 2 21/01/17 17:43:07.807 dispatcher-event-loop-3 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_6.0, runningTasks: 3 21/01/17 17:43:07.807 dispatcher-event-loop-3 DEBUG TaskSetManager: No tasks for locality level NODE_LOCAL, so moving to locality level NO_PREF 21/01/17 17:43:07.807 dispatcher-event-loop-3 DEBUG TaskSetManager: Moving to ANY after waiting for 0ms 21/01/17 17:43:07.807 dispatcher-event-loop-3 INFO TaskSetManager: Starting task 6.0 in stage 6.0 (TID 29) (192.168.1.225, executor driver, partition 6, PROCESS_LOCAL, 7143 bytes) taskResourceAssignments Map() 21/01/17 17:43:07.808 Executor task launch worker for task 6.0 in stage 6.0 (TID 29) INFO Executor: Running task 6.0 in stage 6.0 (TID 29) 21/01/17 17:43:07.806 Executor task launch worker for task 4.0 in stage 6.0 (TID 27) DEBUG BlockManager: Getting local block rdd_24_4 21/01/17 17:43:07.808 Executor task launch worker for task 4.0 in stage 6.0 (TID 27) DEBUG BlockManager: Block rdd_24_4 was not found 21/01/17 17:43:07.808 Executor task launch worker for task 4.0 in stage 6.0 (TID 27) DEBUG BlockManager: Getting remote block rdd_24_4 21/01/17 17:43:07.808 task-result-getter-0 INFO TaskSetManager: Finished task 0.0 in stage 6.0 (TID 23) in 66 ms on 192.168.1.225 (executor driver) (3/10) 21/01/17 17:43:07.808 Executor task launch worker for task 6.0 in stage 6.0 (TID 29) DEBUG ExecutorMetricsPoller: stageTCMP: (6, 0) -> 3 21/01/17 17:43:07.808 Executor task launch worker for task 4.0 in stage 6.0 (TID 27) DEBUG BlockManager: Block rdd_24_4 is unknown by block manager master 21/01/17 17:43:07.809 Executor task launch worker for task 4.0 in stage 6.0 (TID 27) DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 2 21/01/17 17:43:07.809 Executor task launch worker for task 4.0 in stage 6.0 (TID 27) DEBUG MapOutputTrackerMaster: Convert map statuses for shuffle 2, mappers 0-4, partitions 4-5 21/01/17 17:43:07.809 Executor task launch worker for task 5.0 in stage 6.0 (TID 28) INFO Executor: Running task 5.0 in stage 6.0 (TID 28) 21/01/17 17:43:07.809 Executor task launch worker for task 4.0 in stage 6.0 (TID 27) DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 21/01/17 17:43:07.809 Executor task launch worker for task 5.0 in stage 6.0 (TID 28) DEBUG ExecutorMetricsPoller: stageTCMP: (6, 0) -> 4 21/01/17 17:43:07.810 Executor task launch worker for task 4.0 in stage 6.0 (TID 27) INFO ShuffleBlockFetcherIterator: Getting 0 (0.0 B) non-empty blocks including 0 (0.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 21/01/17 17:43:07.810 Executor task launch worker for task 4.0 in stage 6.0 (TID 27) INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 21/01/17 17:43:07.810 Executor task launch worker for task 4.0 in stage 6.0 (TID 27) DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: 21/01/17 17:43:07.810 Executor task launch worker for task 4.0 in stage 6.0 (TID 27) DEBUG ShuffleBlockFetcherIterator: Got local blocks in 0 ms 21/01/17 17:43:07.810 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_25_1 for BlockManagerId(driver, 192.168.1.225, 40985, None) 21/01/17 17:43:07.810 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_25_1 in memory on 192.168.1.225:40985 (size: 304.0 B, free: 366.2 MiB) 21/01/17 17:43:07.810 Executor task launch worker for task 4.0 in stage 6.0 (TID 27) DEBUG TaskMemoryManager: Task 27 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@15195dcb 21/01/17 17:43:07.811 Executor task launch worker for task 1.0 in stage 6.0 (TID 24) DEBUG BlockManagerMaster: Updated info of block rdd_25_1 21/01/17 17:43:07.811 Executor task launch worker for task 1.0 in stage 6.0 (TID 24) DEBUG BlockManager: Told master about block rdd_25_1 21/01/17 17:43:07.811 Executor task launch worker for task 1.0 in stage 6.0 (TID 24) DEBUG BlockManager: Put block rdd_25_1 locally took 50 ms 21/01/17 17:43:07.811 Executor task launch worker for task 4.0 in stage 6.0 (TID 27) INFO MemoryStore: Block rdd_24_4 stored as values in memory (estimated size 16.0 B, free 366.0 MiB) 21/01/17 17:43:07.811 Executor task launch worker for task 1.0 in stage 6.0 (TID 24) DEBUG BlockManager: Putting block rdd_25_1 without replication took 51 ms 21/01/17 17:43:07.811 Executor task launch worker for task 1.0 in stage 6.0 (TID 24) DEBUG BlockManager: Getting local block rdd_25_1 21/01/17 17:43:07.811 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_24_4 for BlockManagerId(driver, 192.168.1.225, 40985, None) 21/01/17 17:43:07.811 Executor task launch worker for task 1.0 in stage 6.0 (TID 24) DEBUG BlockManager: Level for block rdd_25_1 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:07.811 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_24_4 in memory on 192.168.1.225:40985 (size: 16.0 B, free: 366.2 MiB) 21/01/17 17:43:07.812 Executor task launch worker for task 4.0 in stage 6.0 (TID 27) DEBUG BlockManagerMaster: Updated info of block rdd_24_4 21/01/17 17:43:07.812 Executor task launch worker for task 4.0 in stage 6.0 (TID 27) DEBUG BlockManager: Told master about block rdd_24_4 21/01/17 17:43:07.812 Executor task launch worker for task 4.0 in stage 6.0 (TID 27) DEBUG BlockManager: Put block rdd_24_4 locally took 3 ms 21/01/17 17:43:07.812 Executor task launch worker for task 4.0 in stage 6.0 (TID 27) DEBUG BlockManager: Putting block rdd_24_4 without replication took 3 ms 21/01/17 17:43:07.812 Executor task launch worker for task 4.0 in stage 6.0 (TID 27) DEBUG BlockManager: Getting local block rdd_24_4 21/01/17 17:43:07.812 Executor task launch worker for task 4.0 in stage 6.0 (TID 27) DEBUG BlockManager: Level for block rdd_24_4 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:07.812 Executor task launch worker for task 1.0 in stage 6.0 (TID 24) INFO Executor: Finished task 1.0 in stage 6.0 (TID 24). 2170 bytes result sent to driver 21/01/17 17:43:07.813 Executor task launch worker for task 4.0 in stage 6.0 (TID 27) INFO MemoryStore: Block rdd_25_4 stored as values in memory (estimated size 16.0 B, free 366.0 MiB) 21/01/17 17:43:07.813 dispatcher-event-loop-2 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_6.0, runningTasks: 3 21/01/17 17:43:07.813 dispatcher-event-loop-2 DEBUG TaskSetManager: No tasks for locality level NODE_LOCAL, so moving to locality level NO_PREF 21/01/17 17:43:07.813 dispatcher-event-loop-2 DEBUG TaskSetManager: Moving to ANY after waiting for 0ms 21/01/17 17:43:07.814 dispatcher-event-loop-2 INFO TaskSetManager: Starting task 7.0 in stage 6.0 (TID 30) (192.168.1.225, executor driver, partition 7, PROCESS_LOCAL, 7143 bytes) taskResourceAssignments Map() 21/01/17 17:43:07.814 Executor task launch worker for task 7.0 in stage 6.0 (TID 30) INFO Executor: Running task 7.0 in stage 6.0 (TID 30) 21/01/17 17:43:07.814 task-result-getter-2 INFO TaskSetManager: Finished task 1.0 in stage 6.0 (TID 24) in 72 ms on 192.168.1.225 (executor driver) (4/10) 21/01/17 17:43:07.814 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_25_4 for BlockManagerId(driver, 192.168.1.225, 40985, None) 21/01/17 17:43:07.814 Executor task launch worker for task 7.0 in stage 6.0 (TID 30) DEBUG ExecutorMetricsPoller: stageTCMP: (6, 0) -> 5 21/01/17 17:43:07.815 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_25_4 in memory on 192.168.1.225:40985 (size: 16.0 B, free: 366.2 MiB) 21/01/17 17:43:07.815 Executor task launch worker for task 4.0 in stage 6.0 (TID 27) DEBUG BlockManagerMaster: Updated info of block rdd_25_4 21/01/17 17:43:07.815 Executor task launch worker for task 4.0 in stage 6.0 (TID 27) DEBUG BlockManager: Told master about block rdd_25_4 21/01/17 17:43:07.815 Executor task launch worker for task 4.0 in stage 6.0 (TID 27) DEBUG BlockManager: Put block rdd_25_4 locally took 8 ms 21/01/17 17:43:07.815 Executor task launch worker for task 4.0 in stage 6.0 (TID 27) DEBUG BlockManager: Putting block rdd_25_4 without replication took 8 ms 21/01/17 17:43:07.815 Executor task launch worker for task 4.0 in stage 6.0 (TID 27) DEBUG BlockManager: Getting local block rdd_25_4 21/01/17 17:43:07.815 Executor task launch worker for task 4.0 in stage 6.0 (TID 27) DEBUG BlockManager: Level for block rdd_25_4 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:07.816 Executor task launch worker for task 4.0 in stage 6.0 (TID 27) INFO Executor: Finished task 4.0 in stage 6.0 (TID 27). 2170 bytes result sent to driver 21/01/17 17:43:07.816 Executor task launch worker for task 4.0 in stage 6.0 (TID 27) DEBUG ExecutorMetricsPoller: stageTCMP: (6, 0) -> 4 21/01/17 17:43:07.817 Executor task launch worker for task 5.0 in stage 6.0 (TID 28) DEBUG BlockManager: Getting local block rdd_25_5 21/01/17 17:43:07.817 Executor task launch worker for task 5.0 in stage 6.0 (TID 28) DEBUG BlockManager: Block rdd_25_5 was not found 21/01/17 17:43:07.817 Executor task launch worker for task 5.0 in stage 6.0 (TID 28) DEBUG BlockManager: Getting remote block rdd_25_5 21/01/17 17:43:07.817 Executor task launch worker for task 5.0 in stage 6.0 (TID 28) DEBUG BlockManager: Block rdd_25_5 is unknown by block manager master 21/01/17 17:43:07.817 Executor task launch worker for task 5.0 in stage 6.0 (TID 28) DEBUG BlockManager: Getting local block rdd_24_5 21/01/17 17:43:07.817 Executor task launch worker for task 5.0 in stage 6.0 (TID 28) DEBUG BlockManager: Block rdd_24_5 was not found 21/01/17 17:43:07.817 Executor task launch worker for task 5.0 in stage 6.0 (TID 28) DEBUG BlockManager: Getting remote block rdd_24_5 21/01/17 17:43:07.817 Executor task launch worker for task 5.0 in stage 6.0 (TID 28) DEBUG BlockManager: Block rdd_24_5 is unknown by block manager master 21/01/17 17:43:07.818 Executor task launch worker for task 5.0 in stage 6.0 (TID 28) DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 2 21/01/17 17:43:07.818 Executor task launch worker for task 5.0 in stage 6.0 (TID 28) DEBUG MapOutputTrackerMaster: Convert map statuses for shuffle 2, mappers 0-4, partitions 5-6 21/01/17 17:43:07.818 Executor task launch worker for task 5.0 in stage 6.0 (TID 28) DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 21/01/17 17:43:07.818 Executor task launch worker for task 5.0 in stage 6.0 (TID 28) INFO ShuffleBlockFetcherIterator: Getting 0 (0.0 B) non-empty blocks including 0 (0.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 21/01/17 17:43:07.818 Executor task launch worker for task 5.0 in stage 6.0 (TID 28) INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 21/01/17 17:43:07.819 Executor task launch worker for task 5.0 in stage 6.0 (TID 28) DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: 21/01/17 17:43:07.819 Executor task launch worker for task 5.0 in stage 6.0 (TID 28) DEBUG ShuffleBlockFetcherIterator: Got local blocks in 0 ms 21/01/17 17:43:07.819 Executor task launch worker for task 5.0 in stage 6.0 (TID 28) DEBUG TaskMemoryManager: Task 28 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@d54c423 21/01/17 17:43:07.820 Executor task launch worker for task 5.0 in stage 6.0 (TID 28) INFO MemoryStore: Block rdd_24_5 stored as values in memory (estimated size 16.0 B, free 366.0 MiB) 21/01/17 17:43:07.820 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_24_5 for BlockManagerId(driver, 192.168.1.225, 40985, None) 21/01/17 17:43:07.820 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_24_5 in memory on 192.168.1.225:40985 (size: 16.0 B, free: 366.2 MiB) 21/01/17 17:43:07.792 Executor task launch worker for task 0.0 in stage 1.0 (TID 1) INFO CodeGenerator: Code generated in 62.646634 ms 21/01/17 17:43:07.821 dispatcher-event-loop-3 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_6.0, runningTasks: 3 21/01/17 17:43:07.822 dispatcher-event-loop-3 DEBUG TaskSetManager: No tasks for locality level NODE_LOCAL, so moving to locality level NO_PREF 21/01/17 17:43:07.822 dispatcher-event-loop-3 DEBUG TaskSetManager: Moving to ANY after waiting for 0ms 21/01/17 17:43:07.822 dispatcher-event-loop-3 INFO TaskSetManager: Starting task 8.0 in stage 6.0 (TID 31) (192.168.1.225, executor driver, partition 8, PROCESS_LOCAL, 7143 bytes) taskResourceAssignments Map() 21/01/17 17:43:07.823 Executor task launch worker for task 8.0 in stage 6.0 (TID 31) INFO Executor: Running task 8.0 in stage 6.0 (TID 31) 21/01/17 17:43:07.823 task-result-getter-3 INFO TaskSetManager: Finished task 4.0 in stage 6.0 (TID 27) in 26 ms on 192.168.1.225 (executor driver) (5/10) 21/01/17 17:43:07.823 Executor task launch worker for task 8.0 in stage 6.0 (TID 31) DEBUG ExecutorMetricsPoller: stageTCMP: (6, 0) -> 5 21/01/17 17:43:07.823 Executor task launch worker for task 5.0 in stage 6.0 (TID 28) DEBUG BlockManagerMaster: Updated info of block rdd_24_5 21/01/17 17:43:07.823 Executor task launch worker for task 5.0 in stage 6.0 (TID 28) DEBUG BlockManager: Told master about block rdd_24_5 21/01/17 17:43:07.823 Executor task launch worker for task 5.0 in stage 6.0 (TID 28) DEBUG BlockManager: Put block rdd_24_5 locally took 5 ms 21/01/17 17:43:07.823 Executor task launch worker for task 5.0 in stage 6.0 (TID 28) DEBUG BlockManager: Putting block rdd_24_5 without replication took 5 ms 21/01/17 17:43:07.823 Executor task launch worker for task 5.0 in stage 6.0 (TID 28) DEBUG BlockManager: Getting local block rdd_24_5 21/01/17 17:43:07.823 Executor task launch worker for task 5.0 in stage 6.0 (TID 28) DEBUG BlockManager: Level for block rdd_24_5 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:07.824 Executor task launch worker for task 7.0 in stage 6.0 (TID 30) DEBUG BlockManager: Getting local block rdd_25_7 21/01/17 17:43:07.824 Executor task launch worker for task 7.0 in stage 6.0 (TID 30) DEBUG BlockManager: Block rdd_25_7 was not found 21/01/17 17:43:07.824 Executor task launch worker for task 7.0 in stage 6.0 (TID 30) DEBUG BlockManager: Getting remote block rdd_25_7 21/01/17 17:43:07.825 Executor task launch worker for task 7.0 in stage 6.0 (TID 30) DEBUG BlockManager: Block rdd_25_7 is unknown by block manager master 21/01/17 17:43:07.825 Executor task launch worker for task 7.0 in stage 6.0 (TID 30) DEBUG BlockManager: Getting local block rdd_24_7 21/01/17 17:43:07.825 Executor task launch worker for task 7.0 in stage 6.0 (TID 30) DEBUG BlockManager: Block rdd_24_7 was not found 21/01/17 17:43:07.825 Executor task launch worker for task 7.0 in stage 6.0 (TID 30) DEBUG BlockManager: Getting remote block rdd_24_7 21/01/17 17:43:07.825 Executor task launch worker for task 7.0 in stage 6.0 (TID 30) DEBUG BlockManager: Block rdd_24_7 is unknown by block manager master 21/01/17 17:43:07.825 Executor task launch worker for task 7.0 in stage 6.0 (TID 30) DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 2 21/01/17 17:43:07.825 Executor task launch worker for task 7.0 in stage 6.0 (TID 30) DEBUG MapOutputTrackerMaster: Convert map statuses for shuffle 2, mappers 0-4, partitions 7-8 21/01/17 17:43:07.826 Executor task launch worker for task 7.0 in stage 6.0 (TID 30) DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 21/01/17 17:43:07.826 Executor task launch worker for task 7.0 in stage 6.0 (TID 30) INFO ShuffleBlockFetcherIterator: Getting 0 (0.0 B) non-empty blocks including 0 (0.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 21/01/17 17:43:07.826 Executor task launch worker for task 7.0 in stage 6.0 (TID 30) INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 21/01/17 17:43:07.826 Executor task launch worker for task 7.0 in stage 6.0 (TID 30) DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: 21/01/17 17:43:07.826 Executor task launch worker for task 7.0 in stage 6.0 (TID 30) DEBUG ShuffleBlockFetcherIterator: Got local blocks in 0 ms 21/01/17 17:43:07.827 Executor task launch worker for task 7.0 in stage 6.0 (TID 30) DEBUG TaskMemoryManager: Task 30 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@5513c625 21/01/17 17:43:07.827 Executor task launch worker for task 7.0 in stage 6.0 (TID 30) INFO MemoryStore: Block rdd_24_7 stored as values in memory (estimated size 16.0 B, free 366.0 MiB) 21/01/17 17:43:07.827 Executor task launch worker for task 1.0 in stage 6.0 (TID 24) DEBUG ExecutorMetricsPoller: stageTCMP: (6, 0) -> 4 21/01/17 17:43:07.828 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_24_7 for BlockManagerId(driver, 192.168.1.225, 40985, None) 21/01/17 17:43:07.828 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_24_7 in memory on 192.168.1.225:40985 (size: 16.0 B, free: 366.2 MiB) 21/01/17 17:43:07.829 Executor task launch worker for task 7.0 in stage 6.0 (TID 30) DEBUG BlockManagerMaster: Updated info of block rdd_24_7 21/01/17 17:43:07.829 Executor task launch worker for task 7.0 in stage 6.0 (TID 30) DEBUG BlockManager: Told master about block rdd_24_7 21/01/17 17:43:07.829 Executor task launch worker for task 7.0 in stage 6.0 (TID 30) DEBUG BlockManager: Put block rdd_24_7 locally took 3 ms 21/01/17 17:43:07.829 Executor task launch worker for task 7.0 in stage 6.0 (TID 30) DEBUG BlockManager: Putting block rdd_24_7 without replication took 3 ms 21/01/17 17:43:07.829 Executor task launch worker for task 7.0 in stage 6.0 (TID 30) DEBUG BlockManager: Getting local block rdd_24_7 21/01/17 17:43:07.829 Executor task launch worker for task 7.0 in stage 6.0 (TID 30) DEBUG BlockManager: Level for block rdd_24_7 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:07.830 Executor task launch worker for task 8.0 in stage 6.0 (TID 31) DEBUG BlockManager: Getting local block rdd_25_8 21/01/17 17:43:07.830 Executor task launch worker for task 8.0 in stage 6.0 (TID 31) DEBUG BlockManager: Block rdd_25_8 was not found 21/01/17 17:43:07.830 Executor task launch worker for task 8.0 in stage 6.0 (TID 31) DEBUG BlockManager: Getting remote block rdd_25_8 21/01/17 17:43:07.830 Executor task launch worker for task 7.0 in stage 6.0 (TID 30) INFO MemoryStore: Block rdd_25_7 stored as values in memory (estimated size 16.0 B, free 366.0 MiB) 21/01/17 17:43:07.830 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_25_7 for BlockManagerId(driver, 192.168.1.225, 40985, None) 21/01/17 17:43:07.831 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_25_7 in memory on 192.168.1.225:40985 (size: 16.0 B, free: 366.2 MiB) 21/01/17 17:43:07.831 Executor task launch worker for task 7.0 in stage 6.0 (TID 30) DEBUG BlockManagerMaster: Updated info of block rdd_25_7 21/01/17 17:43:07.831 Executor task launch worker for task 7.0 in stage 6.0 (TID 30) DEBUG BlockManager: Told master about block rdd_25_7 21/01/17 17:43:07.832 Executor task launch worker for task 7.0 in stage 6.0 (TID 30) DEBUG BlockManager: Put block rdd_25_7 locally took 6 ms 21/01/17 17:43:07.832 Executor task launch worker for task 7.0 in stage 6.0 (TID 30) DEBUG BlockManager: Putting block rdd_25_7 without replication took 6 ms 21/01/17 17:43:07.832 Executor task launch worker for task 7.0 in stage 6.0 (TID 30) DEBUG BlockManager: Getting local block rdd_25_7 21/01/17 17:43:07.831 Executor task launch worker for task 3.0 in stage 1.0 (TID 4) INFO PythonRunner: Times: total = 130, boot = 66, init = 64, finish = 0 21/01/17 17:43:07.832 Executor task launch worker for task 7.0 in stage 6.0 (TID 30) DEBUG BlockManager: Level for block rdd_25_7 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:07.824 Executor task launch worker for task 5.0 in stage 6.0 (TID 28) INFO MemoryStore: Block rdd_25_5 stored as values in memory (estimated size 16.0 B, free 366.0 MiB) 21/01/17 17:43:07.830 Executor task launch worker for task 6.0 in stage 6.0 (TID 29) DEBUG BlockManager: Getting local block rdd_25_6 21/01/17 17:43:07.833 Executor task launch worker for task 6.0 in stage 6.0 (TID 29) DEBUG BlockManager: Block rdd_25_6 was not found 21/01/17 17:43:07.833 Executor task launch worker for task 6.0 in stage 6.0 (TID 29) DEBUG BlockManager: Getting remote block rdd_25_6 21/01/17 17:43:07.833 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_25_5 for BlockManagerId(driver, 192.168.1.225, 40985, None) 21/01/17 17:43:07.833 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_25_5 in memory on 192.168.1.225:40985 (size: 16.0 B, free: 366.2 MiB) 21/01/17 17:43:07.834 Executor task launch worker for task 6.0 in stage 6.0 (TID 29) DEBUG BlockManager: Block rdd_25_6 is unknown by block manager master 21/01/17 17:43:07.834 Executor task launch worker for task 6.0 in stage 6.0 (TID 29) DEBUG BlockManager: Getting local block rdd_24_6 21/01/17 17:43:07.834 Executor task launch worker for task 6.0 in stage 6.0 (TID 29) DEBUG BlockManager: Block rdd_24_6 was not found 21/01/17 17:43:07.834 Executor task launch worker for task 6.0 in stage 6.0 (TID 29) DEBUG BlockManager: Getting remote block rdd_24_6 21/01/17 17:43:07.832 Executor task launch worker for task 2.0 in stage 1.0 (TID 3) INFO PythonRunner: Times: total = 202, boot = 92, init = 110, finish = 0 21/01/17 17:43:07.834 Executor task launch worker for task 6.0 in stage 6.0 (TID 29) DEBUG BlockManager: Block rdd_24_6 is unknown by block manager master 21/01/17 17:43:07.834 Executor task launch worker for task 6.0 in stage 6.0 (TID 29) DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 2 21/01/17 17:43:07.834 Executor task launch worker for task 6.0 in stage 6.0 (TID 29) DEBUG MapOutputTrackerMaster: Convert map statuses for shuffle 2, mappers 0-4, partitions 6-7 21/01/17 17:43:07.835 Executor task launch worker for task 6.0 in stage 6.0 (TID 29) DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 21/01/17 17:43:07.835 Executor task launch worker for task 6.0 in stage 6.0 (TID 29) INFO ShuffleBlockFetcherIterator: Getting 0 (0.0 B) non-empty blocks including 0 (0.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 21/01/17 17:43:07.835 Executor task launch worker for task 6.0 in stage 6.0 (TID 29) INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 21/01/17 17:43:07.835 Executor task launch worker for task 6.0 in stage 6.0 (TID 29) DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: 21/01/17 17:43:07.836 Executor task launch worker for task 6.0 in stage 6.0 (TID 29) DEBUG ShuffleBlockFetcherIterator: Got local blocks in 0 ms 21/01/17 17:43:07.836 Executor task launch worker for task 1.0 in stage 1.0 (TID 2) INFO PythonRunner: Times: total = 65, boot = 8, init = 57, finish = 0 21/01/17 17:43:07.836 Executor task launch worker for task 6.0 in stage 6.0 (TID 29) DEBUG TaskMemoryManager: Task 29 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@458abccb 21/01/17 17:43:07.837 Executor task launch worker for task 6.0 in stage 6.0 (TID 29) INFO MemoryStore: Block rdd_24_6 stored as values in memory (estimated size 16.0 B, free 366.0 MiB) 21/01/17 17:43:07.837 Executor task launch worker for task 7.0 in stage 6.0 (TID 30) INFO Executor: Finished task 7.0 in stage 6.0 (TID 30). 2170 bytes result sent to driver 21/01/17 17:43:07.837 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_24_6 for BlockManagerId(driver, 192.168.1.225, 40985, None) 21/01/17 17:43:07.837 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_24_6 in memory on 192.168.1.225:40985 (size: 16.0 B, free: 366.2 MiB) 21/01/17 17:43:07.837 dispatcher-event-loop-2 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_6.0, runningTasks: 3 21/01/17 17:43:07.838 Executor task launch worker for task 6.0 in stage 6.0 (TID 29) DEBUG BlockManagerMaster: Updated info of block rdd_24_6 21/01/17 17:43:07.838 Executor task launch worker for task 6.0 in stage 6.0 (TID 29) DEBUG BlockManager: Told master about block rdd_24_6 21/01/17 17:43:07.838 Executor task launch worker for task 6.0 in stage 6.0 (TID 29) DEBUG BlockManager: Put block rdd_24_6 locally took 3 ms 21/01/17 17:43:07.838 Executor task launch worker for task 6.0 in stage 6.0 (TID 29) DEBUG BlockManager: Putting block rdd_24_6 without replication took 3 ms 21/01/17 17:43:07.838 Executor task launch worker for task 6.0 in stage 6.0 (TID 29) DEBUG BlockManager: Getting local block rdd_24_6 21/01/17 17:43:07.838 Executor task launch worker for task 6.0 in stage 6.0 (TID 29) DEBUG BlockManager: Level for block rdd_24_6 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:07.838 dispatcher-event-loop-2 DEBUG TaskSetManager: No tasks for locality level NODE_LOCAL, so moving to locality level NO_PREF 21/01/17 17:43:07.838 Executor task launch worker for task 5.0 in stage 6.0 (TID 28) DEBUG BlockManagerMaster: Updated info of block rdd_25_5 21/01/17 17:43:07.838 dispatcher-event-loop-2 DEBUG TaskSetManager: Moving to ANY after waiting for 0ms 21/01/17 17:43:07.838 Executor task launch worker for task 5.0 in stage 6.0 (TID 28) DEBUG BlockManager: Told master about block rdd_25_5 21/01/17 17:43:07.839 Executor task launch worker for task 5.0 in stage 6.0 (TID 28) DEBUG BlockManager: Put block rdd_25_5 locally took 21 ms 21/01/17 17:43:07.839 Executor task launch worker for task 6.0 in stage 6.0 (TID 29) INFO MemoryStore: Block rdd_25_6 stored as values in memory (estimated size 16.0 B, free 366.0 MiB) 21/01/17 17:43:07.839 Executor task launch worker for task 5.0 in stage 6.0 (TID 28) DEBUG BlockManager: Putting block rdd_25_5 without replication took 21 ms 21/01/17 17:43:07.839 Executor task launch worker for task 5.0 in stage 6.0 (TID 28) DEBUG BlockManager: Getting local block rdd_25_5 21/01/17 17:43:07.839 Executor task launch worker for task 5.0 in stage 6.0 (TID 28) DEBUG BlockManager: Level for block rdd_25_5 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:07.839 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_25_6 for BlockManagerId(driver, 192.168.1.225, 40985, None) 21/01/17 17:43:07.839 dispatcher-event-loop-2 INFO TaskSetManager: Starting task 9.0 in stage 6.0 (TID 32) (192.168.1.225, executor driver, partition 9, PROCESS_LOCAL, 7143 bytes) taskResourceAssignments Map() 21/01/17 17:43:07.839 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_25_6 in memory on 192.168.1.225:40985 (size: 16.0 B, free: 366.2 MiB) 21/01/17 17:43:07.839 task-result-getter-1 INFO TaskSetManager: Finished task 7.0 in stage 6.0 (TID 30) in 26 ms on 192.168.1.225 (executor driver) (6/10) 21/01/17 17:43:07.840 Executor task launch worker for task 9.0 in stage 6.0 (TID 32) INFO Executor: Running task 9.0 in stage 6.0 (TID 32) 21/01/17 17:43:07.840 Executor task launch worker for task 5.0 in stage 6.0 (TID 28) INFO Executor: Finished task 5.0 in stage 6.0 (TID 28). 2170 bytes result sent to driver 21/01/17 17:43:07.840 Executor task launch worker for task 6.0 in stage 6.0 (TID 29) DEBUG BlockManagerMaster: Updated info of block rdd_25_6 21/01/17 17:43:07.840 Executor task launch worker for task 6.0 in stage 6.0 (TID 29) DEBUG BlockManager: Told master about block rdd_25_6 21/01/17 17:43:07.840 Executor task launch worker for task 6.0 in stage 6.0 (TID 29) DEBUG BlockManager: Put block rdd_25_6 locally took 6 ms 21/01/17 17:43:07.840 Executor task launch worker for task 6.0 in stage 6.0 (TID 29) DEBUG BlockManager: Putting block rdd_25_6 without replication took 6 ms 21/01/17 17:43:07.840 Executor task launch worker for task 6.0 in stage 6.0 (TID 29) DEBUG BlockManager: Getting local block rdd_25_6 21/01/17 17:43:07.840 dispatcher-event-loop-3 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_6.0, runningTasks: 3 21/01/17 17:43:07.840 Executor task launch worker for task 6.0 in stage 6.0 (TID 29) DEBUG BlockManager: Level for block rdd_25_6 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:07.841 Executor task launch worker for task 7.0 in stage 6.0 (TID 30) DEBUG ExecutorMetricsPoller: stageTCMP: (6, 0) -> 3 21/01/17 17:43:07.841 Executor task launch worker for task 5.0 in stage 6.0 (TID 28) DEBUG ExecutorMetricsPoller: stageTCMP: (6, 0) -> 2 21/01/17 17:43:07.841 Executor task launch worker for task 9.0 in stage 6.0 (TID 32) DEBUG ExecutorMetricsPoller: stageTCMP: (6, 0) -> 3 21/01/17 17:43:07.842 Executor task launch worker for task 6.0 in stage 6.0 (TID 29) INFO Executor: Finished task 6.0 in stage 6.0 (TID 29). 2170 bytes result sent to driver 21/01/17 17:43:07.842 dispatcher-event-loop-3 DEBUG TaskSetManager: No tasks for locality level NODE_LOCAL, so moving to locality level NO_PREF 21/01/17 17:43:07.842 dispatcher-event-loop-3 DEBUG TaskSetManager: No tasks for locality level NO_PREF, so moving to locality level ANY 21/01/17 17:43:07.842 Executor task launch worker for task 6.0 in stage 6.0 (TID 29) DEBUG ExecutorMetricsPoller: stageTCMP: (6, 0) -> 2 21/01/17 17:43:07.843 task-result-getter-0 INFO TaskSetManager: Finished task 5.0 in stage 6.0 (TID 28) in 43 ms on 192.168.1.225 (executor driver) (7/10) 21/01/17 17:43:07.846 Thread-3 DEBUG CountVectorizer: Input schema: {"type":"struct","fields":[{"name":"id","type":"long","nullable":true,"metadata":{}},{"name":"words","type":{"type":"array","elementType":"string","containsNull":true},"nullable":true,"metadata":{}},{"name":"expected","type":{"type":"udt","class":"org.apache.spark.ml.linalg.VectorUDT","pyClass":"pyspark.ml.linalg.VectorUDT","sqlType":{"type":"struct","fields":[{"name":"type","type":"byte","nullable":false,"metadata":{}},{"name":"size","type":"integer","nullable":true,"metadata":{}},{"name":"indices","type":{"type":"array","elementType":"integer","containsNull":false},"nullable":true,"metadata":{}},{"name":"values","type":{"type":"array","elementType":"double","containsNull":false},"nullable":true,"metadata":{}}]}},"nullable":true,"metadata":{}}]} 21/01/17 17:43:07.847 Executor task launch worker for task 8.0 in stage 6.0 (TID 31) DEBUG BlockManager: Block rdd_25_8 is unknown by block manager master 21/01/17 17:43:07.848 Executor task launch worker for task 8.0 in stage 6.0 (TID 31) DEBUG BlockManager: Getting local block rdd_24_8 21/01/17 17:43:07.848 Executor task launch worker for task 8.0 in stage 6.0 (TID 31) DEBUG BlockManager: Block rdd_24_8 was not found 21/01/17 17:43:07.848 Executor task launch worker for task 8.0 in stage 6.0 (TID 31) DEBUG BlockManager: Getting remote block rdd_24_8 21/01/17 17:43:07.848 Executor task launch worker for task 8.0 in stage 6.0 (TID 31) DEBUG BlockManager: Block rdd_24_8 is unknown by block manager master 21/01/17 17:43:07.848 Executor task launch worker for task 8.0 in stage 6.0 (TID 31) DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 2 21/01/17 17:43:07.848 Executor task launch worker for task 8.0 in stage 6.0 (TID 31) DEBUG MapOutputTrackerMaster: Convert map statuses for shuffle 2, mappers 0-4, partitions 8-9 21/01/17 17:43:07.848 Thread-3 DEBUG CountVectorizer: Expected output schema: {"type":"struct","fields":[{"name":"id","type":"long","nullable":true,"metadata":{}},{"name":"words","type":{"type":"array","elementType":"string","containsNull":true},"nullable":true,"metadata":{}},{"name":"expected","type":{"type":"udt","class":"org.apache.spark.ml.linalg.VectorUDT","pyClass":"pyspark.ml.linalg.VectorUDT","sqlType":{"type":"struct","fields":[{"name":"type","type":"byte","nullable":false,"metadata":{}},{"name":"size","type":"integer","nullable":true,"metadata":{}},{"name":"indices","type":{"type":"array","elementType":"integer","containsNull":false},"nullable":true,"metadata":{}},{"name":"values","type":{"type":"array","elementType":"double","containsNull":false},"nullable":true,"metadata":{}}]}},"nullable":true,"metadata":{}},{"name":"features","type":{"type":"udt","class":"org.apache.spark.ml.linalg.VectorUDT","pyClass":"pyspark.ml.linalg.VectorUDT","sqlType":{"type":"struct","fields":[{"name":"type","type":"byte","nullable":false,"metadata":{}},{"name":"size","type":"integer","nullable":true,"metadata":{}},{"name":"indices","type":{"type":"array","elementType":"integer","containsNull":false},"nullable":true,"metadata":{}},{"name":"values","type":{"type":"array","elementType":"double","containsNull":false},"nullable":true,"metadata":{}}]}},"nullable":false,"metadata":{}}]} 21/01/17 17:43:07.852 Executor task launch worker for task 2.0 in stage 1.0 (TID 3) DEBUG LocalDiskShuffleMapOutputWriter: Writing shuffle index file for mapId 3 with length 4 21/01/17 17:43:07.852 Thread-3 DEBUG Analyzer$ResolveReferences: Resolving 'words to words#38 21/01/17 17:43:07.857 task-result-getter-2 INFO TaskSetManager: Finished task 6.0 in stage 6.0 (TID 29) in 36 ms on 192.168.1.225 (executor driver) (8/10) 21/01/17 17:43:07.858 Executor task launch worker for task 8.0 in stage 6.0 (TID 31) DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 21/01/17 17:43:07.859 Executor task launch worker for task 8.0 in stage 6.0 (TID 31) INFO ShuffleBlockFetcherIterator: Getting 0 (0.0 B) non-empty blocks including 0 (0.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 21/01/17 17:43:07.859 Executor task launch worker for task 8.0 in stage 6.0 (TID 31) INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 1 ms 21/01/17 17:43:07.859 Executor task launch worker for task 8.0 in stage 6.0 (TID 31) DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: 21/01/17 17:43:07.859 Executor task launch worker for task 8.0 in stage 6.0 (TID 31) DEBUG ShuffleBlockFetcherIterator: Got local blocks in 1 ms 21/01/17 17:43:07.860 Executor task launch worker for task 8.0 in stage 6.0 (TID 31) DEBUG TaskMemoryManager: Task 31 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@7fa5d6a3 21/01/17 17:43:07.860 Executor task launch worker for task 8.0 in stage 6.0 (TID 31) INFO MemoryStore: Block rdd_24_8 stored as values in memory (estimated size 16.0 B, free 366.0 MiB) 21/01/17 17:43:07.860 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_24_8 for BlockManagerId(driver, 192.168.1.225, 40985, None) 21/01/17 17:43:07.861 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_24_8 in memory on 192.168.1.225:40985 (size: 16.0 B, free: 366.2 MiB) 21/01/17 17:43:07.861 Executor task launch worker for task 8.0 in stage 6.0 (TID 31) DEBUG BlockManagerMaster: Updated info of block rdd_24_8 21/01/17 17:43:07.861 Executor task launch worker for task 8.0 in stage 6.0 (TID 31) DEBUG BlockManager: Told master about block rdd_24_8 21/01/17 17:43:07.861 Executor task launch worker for task 8.0 in stage 6.0 (TID 31) DEBUG BlockManager: Put block rdd_24_8 locally took 12 ms 21/01/17 17:43:07.861 Executor task launch worker for task 8.0 in stage 6.0 (TID 31) DEBUG BlockManager: Putting block rdd_24_8 without replication took 13 ms 21/01/17 17:43:07.861 Executor task launch worker for task 8.0 in stage 6.0 (TID 31) DEBUG BlockManager: Getting local block rdd_24_8 21/01/17 17:43:07.861 Executor task launch worker for task 8.0 in stage 6.0 (TID 31) DEBUG BlockManager: Level for block rdd_24_8 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:07.862 Executor task launch worker for task 8.0 in stage 6.0 (TID 31) INFO MemoryStore: Block rdd_25_8 stored as values in memory (estimated size 16.0 B, free 366.0 MiB) 21/01/17 17:43:07.862 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_25_8 for BlockManagerId(driver, 192.168.1.225, 40985, None) 21/01/17 17:43:07.862 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_25_8 in memory on 192.168.1.225:40985 (size: 16.0 B, free: 366.2 MiB) 21/01/17 17:43:07.863 Executor task launch worker for task 8.0 in stage 6.0 (TID 31) DEBUG BlockManagerMaster: Updated info of block rdd_25_8 21/01/17 17:43:07.863 Executor task launch worker for task 8.0 in stage 6.0 (TID 31) DEBUG BlockManager: Told master about block rdd_25_8 21/01/17 17:43:07.863 Executor task launch worker for task 8.0 in stage 6.0 (TID 31) DEBUG BlockManager: Put block rdd_25_8 locally took 15 ms 21/01/17 17:43:07.863 Executor task launch worker for task 8.0 in stage 6.0 (TID 31) DEBUG BlockManager: Putting block rdd_25_8 without replication took 15 ms 21/01/17 17:43:07.863 Executor task launch worker for task 8.0 in stage 6.0 (TID 31) DEBUG BlockManager: Getting local block rdd_25_8 21/01/17 17:43:07.863 Executor task launch worker for task 9.0 in stage 6.0 (TID 32) DEBUG BlockManager: Getting local block rdd_25_9 21/01/17 17:43:07.863 Executor task launch worker for task 8.0 in stage 6.0 (TID 31) DEBUG BlockManager: Level for block rdd_25_8 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:07.863 Executor task launch worker for task 9.0 in stage 6.0 (TID 32) DEBUG BlockManager: Block rdd_25_9 was not found 21/01/17 17:43:07.863 Executor task launch worker for task 9.0 in stage 6.0 (TID 32) DEBUG BlockManager: Getting remote block rdd_25_9 21/01/17 17:43:07.863 Executor task launch worker for task 9.0 in stage 6.0 (TID 32) DEBUG BlockManager: Block rdd_25_9 is unknown by block manager master 21/01/17 17:43:07.864 Executor task launch worker for task 9.0 in stage 6.0 (TID 32) DEBUG BlockManager: Getting local block rdd_24_9 21/01/17 17:43:07.864 Executor task launch worker for task 9.0 in stage 6.0 (TID 32) DEBUG BlockManager: Block rdd_24_9 was not found 21/01/17 17:43:07.864 Executor task launch worker for task 9.0 in stage 6.0 (TID 32) DEBUG BlockManager: Getting remote block rdd_24_9 21/01/17 17:43:07.864 Executor task launch worker for task 9.0 in stage 6.0 (TID 32) DEBUG BlockManager: Block rdd_24_9 is unknown by block manager master 21/01/17 17:43:07.864 Executor task launch worker for task 9.0 in stage 6.0 (TID 32) DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 2 21/01/17 17:43:07.864 Executor task launch worker for task 9.0 in stage 6.0 (TID 32) DEBUG MapOutputTrackerMaster: Convert map statuses for shuffle 2, mappers 0-4, partitions 9-10 21/01/17 17:43:07.865 Executor task launch worker for task 9.0 in stage 6.0 (TID 32) DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 21/01/17 17:43:07.865 Executor task launch worker for task 9.0 in stage 6.0 (TID 32) INFO ShuffleBlockFetcherIterator: Getting 0 (0.0 B) non-empty blocks including 0 (0.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 21/01/17 17:43:07.865 Executor task launch worker for task 9.0 in stage 6.0 (TID 32) INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 21/01/17 17:43:07.865 Executor task launch worker for task 9.0 in stage 6.0 (TID 32) DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: 21/01/17 17:43:07.865 Executor task launch worker for task 9.0 in stage 6.0 (TID 32) DEBUG ShuffleBlockFetcherIterator: Got local blocks in 0 ms 21/01/17 17:43:07.865 Executor task launch worker for task 8.0 in stage 6.0 (TID 31) INFO Executor: Finished task 8.0 in stage 6.0 (TID 31). 2170 bytes result sent to driver 21/01/17 17:43:07.866 Executor task launch worker for task 8.0 in stage 6.0 (TID 31) DEBUG ExecutorMetricsPoller: stageTCMP: (6, 0) -> 1 21/01/17 17:43:07.866 task-result-getter-3 INFO TaskSetManager: Finished task 8.0 in stage 6.0 (TID 31) in 44 ms on 192.168.1.225 (executor driver) (9/10) 21/01/17 17:43:07.867 Executor task launch worker for task 9.0 in stage 6.0 (TID 32) DEBUG TaskMemoryManager: Task 32 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@118bbb06 21/01/17 17:43:07.867 Executor task launch worker for task 9.0 in stage 6.0 (TID 32) INFO MemoryStore: Block rdd_24_9 stored as values in memory (estimated size 16.0 B, free 366.0 MiB) 21/01/17 17:43:07.867 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_24_9 for BlockManagerId(driver, 192.168.1.225, 40985, None) 21/01/17 17:43:07.868 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_24_9 in memory on 192.168.1.225:40985 (size: 16.0 B, free: 366.2 MiB) 21/01/17 17:43:07.870 Executor task launch worker for task 9.0 in stage 6.0 (TID 32) DEBUG BlockManagerMaster: Updated info of block rdd_24_9 21/01/17 17:43:07.870 Executor task launch worker for task 9.0 in stage 6.0 (TID 32) DEBUG BlockManager: Told master about block rdd_24_9 21/01/17 17:43:07.870 Executor task launch worker for task 9.0 in stage 6.0 (TID 32) DEBUG BlockManager: Put block rdd_24_9 locally took 6 ms 21/01/17 17:43:07.870 Executor task launch worker for task 9.0 in stage 6.0 (TID 32) DEBUG BlockManager: Putting block rdd_24_9 without replication took 6 ms 21/01/17 17:43:07.870 Executor task launch worker for task 9.0 in stage 6.0 (TID 32) DEBUG BlockManager: Getting local block rdd_24_9 21/01/17 17:43:07.870 Executor task launch worker for task 9.0 in stage 6.0 (TID 32) DEBUG BlockManager: Level for block rdd_24_9 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:07.871 Executor task launch worker for task 9.0 in stage 6.0 (TID 32) INFO MemoryStore: Block rdd_25_9 stored as values in memory (estimated size 16.0 B, free 366.0 MiB) 21/01/17 17:43:07.871 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_25_9 for BlockManagerId(driver, 192.168.1.225, 40985, None) 21/01/17 17:43:07.854 Executor task launch worker for task 1.0 in stage 1.0 (TID 2) DEBUG LocalDiskShuffleMapOutputWriter: Writing shuffle index file for mapId 2 with length 4 21/01/17 17:43:07.874 Executor task launch worker for task 1.0 in stage 1.0 (TID 2) DEBUG IndexShuffleBlockResolver: Shuffle index for mapId 2: [334,0,0,0] 21/01/17 17:43:07.885 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_25_9 in memory on 192.168.1.225:40985 (size: 16.0 B, free: 366.2 MiB) 21/01/17 17:43:07.885 Executor task launch worker for task 9.0 in stage 6.0 (TID 32) DEBUG BlockManagerMaster: Updated info of block rdd_25_9 21/01/17 17:43:07.885 Executor task launch worker for task 9.0 in stage 6.0 (TID 32) DEBUG BlockManager: Told master about block rdd_25_9 21/01/17 17:43:07.885 Executor task launch worker for task 9.0 in stage 6.0 (TID 32) DEBUG BlockManager: Put block rdd_25_9 locally took 21 ms 21/01/17 17:43:07.885 Executor task launch worker for task 1.0 in stage 1.0 (TID 2) DEBUG TaskMemoryManager: Task 2 release 0.0 B from org.apache.spark.util.collection.ExternalSorter@2783dcef 21/01/17 17:43:07.886 Executor task launch worker for task 9.0 in stage 6.0 (TID 32) DEBUG BlockManager: Putting block rdd_25_9 without replication took 21 ms 21/01/17 17:43:07.886 Executor task launch worker for task 9.0 in stage 6.0 (TID 32) DEBUG BlockManager: Getting local block rdd_25_9 21/01/17 17:43:07.886 Executor task launch worker for task 9.0 in stage 6.0 (TID 32) DEBUG BlockManager: Level for block rdd_25_9 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:07.887 Executor task launch worker for task 3.0 in stage 1.0 (TID 4) DEBUG LocalDiskShuffleMapOutputWriter: Writing shuffle index file for mapId 4 with length 4 21/01/17 17:43:07.887 Executor task launch worker for task 9.0 in stage 6.0 (TID 32) INFO Executor: Finished task 9.0 in stage 6.0 (TID 32). 2170 bytes result sent to driver 21/01/17 17:43:07.887 Executor task launch worker for task 9.0 in stage 6.0 (TID 32) DEBUG ExecutorMetricsPoller: removing (6, 0) from stageTCMP 21/01/17 17:43:07.887 Executor task launch worker for task 3.0 in stage 1.0 (TID 4) DEBUG IndexShuffleBlockResolver: Shuffle index for mapId 4: [298,0,0,0] 21/01/17 17:43:07.876 Executor task launch worker for task 2.0 in stage 1.0 (TID 3) DEBUG IndexShuffleBlockResolver: Shuffle index for mapId 3: [297,0,0,0] 21/01/17 17:43:07.888 task-result-getter-1 INFO TaskSetManager: Finished task 9.0 in stage 6.0 (TID 32) in 49 ms on 192.168.1.225 (executor driver) (10/10) 21/01/17 17:43:07.888 task-result-getter-1 INFO TaskSchedulerImpl: Removed TaskSet 6.0, whose tasks have all completed, from pool 21/01/17 17:43:07.889 dag-scheduler-event-loop INFO DAGScheduler: ResultStage 6 (count at ALS.scala:980) finished in 0.156 s 21/01/17 17:43:07.889 dag-scheduler-event-loop DEBUG DAGScheduler: After removal of stage 5, remaining stages = 2 21/01/17 17:43:07.889 dag-scheduler-event-loop DEBUG DAGScheduler: After removal of stage 4, remaining stages = 1 21/01/17 17:43:07.890 dag-scheduler-event-loop DEBUG DAGScheduler: After removal of stage 6, remaining stages = 0 21/01/17 17:43:07.890 dag-scheduler-event-loop INFO DAGScheduler: Job 2 is finished. Cancelling potential speculative or zombie tasks for this job 21/01/17 17:43:07.890 dag-scheduler-event-loop INFO TaskSchedulerImpl: Killing all running tasks in stage 6: Stage finished 21/01/17 17:43:07.890 Thread-3 INFO DAGScheduler: Job 2 finished: count at ALS.scala:980, took 0.270690 s 21/01/17 17:43:07.892 Executor task launch worker for task 1.0 in stage 1.0 (TID 2) INFO Executor: Finished task 1.0 in stage 1.0 (TID 2). 2073 bytes result sent to driver 21/01/17 17:43:07.892 Thread-3 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$initialize$1 21/01/17 17:43:07.893 Executor task launch worker for task 1.0 in stage 1.0 (TID 2) DEBUG ExecutorMetricsPoller: stageTCMP: (1, 0) -> 3 21/01/17 17:43:07.893 Executor task launch worker for task 0.0 in stage 1.0 (TID 1) INFO PythonRunner: Times: total = 121, boot = 28, init = 93, finish = 0 21/01/17 17:43:07.894 Executor task launch worker for task 0.0 in stage 1.0 (TID 1) DEBUG LocalDiskShuffleMapOutputWriter: Writing shuffle index file for mapId 1 with length 4 21/01/17 17:43:07.894 Thread-3 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$initialize$1) is now cleaned +++ 21/01/17 17:43:07.894 Executor task launch worker for task 0.0 in stage 1.0 (TID 1) DEBUG IndexShuffleBlockResolver: Shuffle index for mapId 1: [292,0,0,0] 21/01/17 17:43:07.896 Thread-3 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$initialize$1 21/01/17 17:43:07.898 Thread-3 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$initialize$1) is now cleaned +++ 21/01/17 17:43:07.900 dispatcher-event-loop-3 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_1.0, runningTasks: 3 21/01/17 17:43:07.900 dispatcher-event-loop-3 DEBUG TaskSetManager: No tasks for locality level NO_PREF, so moving to locality level ANY 21/01/17 17:43:07.907 Thread-3 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$cogroup$4 21/01/17 17:43:07.888 Executor task launch worker for task 2.0 in stage 1.0 (TID 3) DEBUG TaskMemoryManager: Task 3 release 0.0 B from org.apache.spark.util.collection.ExternalSorter@67727e94 21/01/17 17:43:07.887 Executor task launch worker for task 3.0 in stage 1.0 (TID 4) DEBUG TaskMemoryManager: Task 4 release 0.0 B from org.apache.spark.util.collection.ExternalSorter@40f21b33 21/01/17 17:43:07.895 Executor task launch worker for task 0.0 in stage 1.0 (TID 1) DEBUG TaskMemoryManager: Task 1 release 0.0 B from org.apache.spark.util.collection.ExternalSorter@2ac91cd5 21/01/17 17:43:07.910 Thread-3 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$cogroup$4) is now cleaned +++ 21/01/17 17:43:07.910 task-result-getter-1 INFO TaskSetManager: Finished task 1.0 in stage 1.0 (TID 2) in 725 ms on 192.168.1.225 (executor driver) (1/4) 21/01/17 17:43:07.912 Executor task launch worker for task 0.0 in stage 1.0 (TID 1) INFO Executor: Finished task 0.0 in stage 1.0 (TID 1). 2030 bytes result sent to driver 21/01/17 17:43:07.912 Executor task launch worker for task 0.0 in stage 1.0 (TID 1) DEBUG ExecutorMetricsPoller: stageTCMP: (1, 0) -> 2 21/01/17 17:43:07.912 dag-scheduler-event-loop DEBUG DAGScheduler: ShuffleMapTask finished on driver 21/01/17 17:43:07.913 Executor task launch worker for task 2.0 in stage 1.0 (TID 3) INFO Executor: Finished task 2.0 in stage 1.0 (TID 3). 2030 bytes result sent to driver 21/01/17 17:43:07.914 Executor task launch worker for task 2.0 in stage 1.0 (TID 3) DEBUG ExecutorMetricsPoller: stageTCMP: (1, 0) -> 1 21/01/17 17:43:07.915 Executor task launch worker for task 3.0 in stage 1.0 (TID 4) INFO Executor: Finished task 3.0 in stage 1.0 (TID 4). 2030 bytes result sent to driver 21/01/17 17:43:07.915 Executor task launch worker for task 3.0 in stage 1.0 (TID 4) DEBUG ExecutorMetricsPoller: removing (1, 0) from stageTCMP 21/01/17 17:43:07.923 Thread-3 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$join$2 21/01/17 17:43:07.926 Thread-3 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$join$2) is now cleaned +++ 21/01/17 17:43:07.928 Thread-3 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$computeFactors$1 21/01/17 17:43:07.930 Thread-3 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$computeFactors$1) is now cleaned +++ 21/01/17 17:43:07.932 Thread-3 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$groupByKey$2 21/01/17 17:43:07.935 Thread-3 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$groupByKey$2) is now cleaned +++ 21/01/17 17:43:07.941 Thread-3 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$groupByKey$3 21/01/17 17:43:07.941 task-result-getter-2 INFO TaskSetManager: Finished task 0.0 in stage 1.0 (TID 1) in 757 ms on 192.168.1.225 (executor driver) (2/4) 21/01/17 17:43:07.943 dag-scheduler-event-loop DEBUG DAGScheduler: ShuffleMapTask finished on driver 21/01/17 17:43:07.945 Thread-3 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$groupByKey$3) is now cleaned +++ 21/01/17 17:43:07.945 Thread-3 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$groupByKey$4 21/01/17 17:43:07.946 task-result-getter-0 INFO TaskSetManager: Finished task 3.0 in stage 1.0 (TID 4) in 760 ms on 192.168.1.225 (executor driver) (3/4) 21/01/17 17:43:07.947 dag-scheduler-event-loop DEBUG DAGScheduler: ShuffleMapTask finished on driver 21/01/17 17:43:07.949 Thread-3 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$groupByKey$4) is now cleaned +++ 21/01/17 17:43:07.951 Thread-3 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$cogroup$4 21/01/17 17:43:07.954 task-result-getter-3 INFO TaskSetManager: Finished task 2.0 in stage 1.0 (TID 3) in 768 ms on 192.168.1.225 (executor driver) (4/4) 21/01/17 17:43:07.954 Thread-3 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$cogroup$4) is now cleaned +++ 21/01/17 17:43:07.954 task-result-getter-3 INFO TaskSchedulerImpl: Removed TaskSet 1.0, whose tasks have all completed, from pool 21/01/17 17:43:07.955 dag-scheduler-event-loop DEBUG DAGScheduler: ShuffleMapTask finished on driver 21/01/17 17:43:07.955 dag-scheduler-event-loop INFO DAGScheduler: ShuffleMapStage 1 (map at ClusteringMetrics.scala:500) finished in 0.829 s 21/01/17 17:43:07.956 Thread-3 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$join$2 21/01/17 17:43:07.956 dag-scheduler-event-loop INFO DAGScheduler: looking for newly runnable stages 21/01/17 17:43:07.957 dag-scheduler-event-loop INFO DAGScheduler: running: Set() 21/01/17 17:43:07.957 dag-scheduler-event-loop INFO DAGScheduler: waiting: Set(ResultStage 2) 21/01/17 17:43:07.957 dag-scheduler-event-loop INFO DAGScheduler: failed: Set() 21/01/17 17:43:07.958 dag-scheduler-event-loop DEBUG MapOutputTrackerMaster: Increasing epoch to 1 21/01/17 17:43:07.959 Thread-3 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$join$2) is now cleaned +++ 21/01/17 17:43:07.960 dag-scheduler-event-loop DEBUG DAGScheduler: submitStage(ResultStage 2 (name=collectAsMap at ClusteringMetrics.scala:515;jobs=1)) 21/01/17 17:43:07.961 dag-scheduler-event-loop DEBUG DAGScheduler: missing: List() 21/01/17 17:43:07.961 dag-scheduler-event-loop INFO DAGScheduler: Submitting ResultStage 2 (ShuffledRDD[12] at aggregateByKey at ClusteringMetrics.scala:501), which has no missing parents 21/01/17 17:43:07.961 dag-scheduler-event-loop DEBUG DAGScheduler: submitMissingTasks(ResultStage 2) 21/01/17 17:43:07.961 Thread-3 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$computeFactors$4 21/01/17 17:43:07.963 Thread-3 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$computeFactors$4) is now cleaned +++ 21/01/17 17:43:07.968 Thread-3 DEBUG CoGroupedRDD: Adding one-to-one dependency with itemInBlocks MapPartitionsRDD[24] at mapValues at ALS.scala:1640 21/01/17 17:43:07.968 Thread-3 DEBUG CoGroupedRDD: Adding one-to-one dependency with ShuffledRDD[32] at groupByKey at ALS.scala:1707 21/01/17 17:43:07.970 Thread-3 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$cogroup$4 21/01/17 17:43:07.973 Thread-3 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$cogroup$4) is now cleaned +++ 21/01/17 17:43:07.974 Thread-3 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$join$2 21/01/17 17:43:07.978 Thread-3 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$join$2) is now cleaned +++ 21/01/17 17:43:07.978 Thread-3 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$computeFactors$1 21/01/17 17:43:07.980 Thread-3 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$computeFactors$1) is now cleaned +++ 21/01/17 17:43:07.981 Thread-3 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$groupByKey$2 21/01/17 17:43:07.985 Thread-3 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$groupByKey$2) is now cleaned +++ 21/01/17 17:43:07.985 Thread-3 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$groupByKey$3 21/01/17 17:43:07.987 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_2 stored as values in memory (estimated size 4.5 KiB, free 366.3 MiB) 21/01/17 17:43:07.987 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_2 locally took 1 ms 21/01/17 17:43:07.987 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_2 without replication took 1 ms 21/01/17 17:43:07.987 Thread-3 DEBUG WholeStageCodegenExec: /* 001 */ public Object generate(Object[] references) { /* 002 */ return new GeneratedIteratorForCodegenStage1(references); /* 003 */ } /* 004 */ /* 005 */ // codegenStageId=1 /* 006 */ final class GeneratedIteratorForCodegenStage1 extends org.apache.spark.sql.execution.BufferedRowIterator { /* 007 */ private Object[] references; /* 008 */ private scala.collection.Iterator[] inputs; /* 009 */ private scala.collection.Iterator rdd_input_0; /* 010 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[] rdd_mutableStateArray_0 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[3]; /* 011 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[] rdd_mutableStateArray_1 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[4]; /* 012 */ /* 013 */ public GeneratedIteratorForCodegenStage1(Object[] references) { /* 014 */ this.references = references; /* 015 */ } /* 016 */ /* 017 */ public void init(int index, scala.collection.Iterator[] inputs) { /* 018 */ partitionIndex = index; /* 019 */ this.inputs = inputs; /* 020 */ rdd_input_0 = inputs[0]; /* 021 */ rdd_mutableStateArray_0[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(3, 64); /* 022 */ rdd_mutableStateArray_1[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[0], 8); /* 023 */ rdd_mutableStateArray_0[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(rdd_mutableStateArray_0[0], 4); /* 024 */ rdd_mutableStateArray_1[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[1], 4); /* 025 */ rdd_mutableStateArray_1[2] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[1], 8); /* 026 */ rdd_mutableStateArray_0[2] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(1, 32); /* 027 */ rdd_mutableStateArray_1[3] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[2], 8); /* 028 */ /* 029 */ } /* 030 */ /* 031 */ protected void processNext() throws java.io.IOException { /* 032 */ while ( rdd_input_0.hasNext()) { /* 033 */ InternalRow rdd_row_0 = (InternalRow) rdd_input_0.next(); /* 034 */ ((org.apache.spark.sql.execution.metric.SQLMetric) references[0] /* numOutputRows */).add(1); /* 035 */ // common sub-expressions /* 036 */ /* 037 */ boolean rdd_isNull_1 = rdd_row_0.isNullAt(1); /* 038 */ ArrayData rdd_value_1 = rdd_isNull_1 ? /* 039 */ null : (rdd_row_0.getArray(1)); /* 040 */ rdd_mutableStateArray_0[2].reset(); /* 041 */ /* 042 */ rdd_mutableStateArray_0[2].zeroOutNullBytes(); /* 043 */ /* 044 */ if (rdd_isNull_1) { /* 045 */ rdd_mutableStateArray_0[2].setNullAt(0); /* 046 */ } else { /* 047 */ // Remember the current cursor so that we can calculate how many bytes are /* 048 */ // written later. /* 049 */ final int project_previousCursor_0 = rdd_mutableStateArray_0[2].cursor(); /* 050 */ /* 051 */ final ArrayData project_tmpInput_0 = rdd_value_1; /* 052 */ if (project_tmpInput_0 instanceof UnsafeArrayData) { /* 053 */ rdd_mutableStateArray_0[2].write((UnsafeArrayData) project_tmpInput_0); /* 054 */ } else { /* 055 */ final int project_numElements_0 = project_tmpInput_0.numElements(); /* 056 */ rdd_mutableStateArray_1[3].initialize(project_numElements_0); /* 057 */ /* 058 */ for (int project_index_0 = 0; project_index_0 < project_numElements_0; project_index_0++) { /* 059 */ if (project_tmpInput_0.isNullAt(project_index_0)) { /* 060 */ rdd_mutableStateArray_1[3].setNull8Bytes(project_index_0); /* 061 */ } else { /* 062 */ rdd_mutableStateArray_1[3].write(project_index_0, project_tmpInput_0.getUTF8String(project_index_0)); /* 063 */ } /* 064 */ /* 065 */ } /* 066 */ } /* 067 */ /* 068 */ rdd_mutableStateArray_0[2].setOffsetAndSizeFromPreviousCursor(0, project_previousCursor_0); /* 069 */ } /* 070 */ append((rdd_mutableStateArray_0[2].getRow())); /* 071 */ if (shouldStop()) return; /* 072 */ } /* 073 */ } /* 074 */ /* 075 */ } 21/01/17 17:43:07.988 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_2_piece0 stored as bytes in memory (estimated size 2.5 KiB, free 366.3 MiB) 21/01/17 17:43:07.988 Thread-3 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$groupByKey$3) is now cleaned +++ 21/01/17 17:43:07.989 Thread-3 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$groupByKey$4 21/01/17 17:43:07.989 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master broadcast_2_piece0 for BlockManagerId(driver, 192.168.1.225, 33787, None) 21/01/17 17:43:07.989 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added broadcast_2_piece0 in memory on 192.168.1.225:33787 (size: 2.5 KiB, free: 366.3 MiB) 21/01/17 17:43:07.989 Thread-3 DEBUG CodeGenerator: /* 001 */ public Object generate(Object[] references) { /* 002 */ return new GeneratedIteratorForCodegenStage1(references); /* 003 */ } /* 004 */ /* 005 */ // codegenStageId=1 /* 006 */ final class GeneratedIteratorForCodegenStage1 extends org.apache.spark.sql.execution.BufferedRowIterator { /* 007 */ private Object[] references; /* 008 */ private scala.collection.Iterator[] inputs; /* 009 */ private scala.collection.Iterator rdd_input_0; /* 010 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[] rdd_mutableStateArray_0 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[3]; /* 011 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[] rdd_mutableStateArray_1 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[4]; /* 012 */ /* 013 */ public GeneratedIteratorForCodegenStage1(Object[] references) { /* 014 */ this.references = references; /* 015 */ } /* 016 */ /* 017 */ public void init(int index, scala.collection.Iterator[] inputs) { /* 018 */ partitionIndex = index; /* 019 */ this.inputs = inputs; /* 020 */ rdd_input_0 = inputs[0]; /* 021 */ rdd_mutableStateArray_0[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(3, 64); /* 022 */ rdd_mutableStateArray_1[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[0], 8); /* 023 */ rdd_mutableStateArray_0[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(rdd_mutableStateArray_0[0], 4); /* 024 */ rdd_mutableStateArray_1[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[1], 4); /* 025 */ rdd_mutableStateArray_1[2] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[1], 8); /* 026 */ rdd_mutableStateArray_0[2] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(1, 32); /* 027 */ rdd_mutableStateArray_1[3] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[2], 8); /* 028 */ /* 029 */ } /* 030 */ /* 031 */ protected void processNext() throws java.io.IOException { /* 032 */ while ( rdd_input_0.hasNext()) { /* 033 */ InternalRow rdd_row_0 = (InternalRow) rdd_input_0.next(); /* 034 */ ((org.apache.spark.sql.execution.metric.SQLMetric) references[0] /* numOutputRows */).add(1); /* 035 */ // common sub-expressions /* 036 */ /* 037 */ boolean rdd_isNull_1 = rdd_row_0.isNullAt(1); /* 038 */ ArrayData rdd_value_1 = rdd_isNull_1 ? /* 039 */ null : (rdd_row_0.getArray(1)); /* 040 */ rdd_mutableStateArray_0[2].reset(); /* 041 */ /* 042 */ rdd_mutableStateArray_0[2].zeroOutNullBytes(); /* 043 */ /* 044 */ if (rdd_isNull_1) { /* 045 */ rdd_mutableStateArray_0[2].setNullAt(0); /* 046 */ } else { /* 047 */ // Remember the current cursor so that we can calculate how many bytes are /* 048 */ // written later. /* 049 */ final int project_previousCursor_0 = rdd_mutableStateArray_0[2].cursor(); /* 050 */ /* 051 */ final ArrayData project_tmpInput_0 = rdd_value_1; /* 052 */ if (project_tmpInput_0 instanceof UnsafeArrayData) { /* 053 */ rdd_mutableStateArray_0[2].write((UnsafeArrayData) project_tmpInput_0); /* 054 */ } else { /* 055 */ final int project_numElements_0 = project_tmpInput_0.numElements(); /* 056 */ rdd_mutableStateArray_1[3].initialize(project_numElements_0); /* 057 */ /* 058 */ for (int project_index_0 = 0; project_index_0 < project_numElements_0; project_index_0++) { /* 059 */ if (project_tmpInput_0.isNullAt(project_index_0)) { /* 060 */ rdd_mutableStateArray_1[3].setNull8Bytes(project_index_0); /* 061 */ } else { /* 062 */ rdd_mutableStateArray_1[3].write(project_index_0, project_tmpInput_0.getUTF8String(project_index_0)); /* 063 */ } /* 064 */ /* 065 */ } /* 066 */ } /* 067 */ /* 068 */ rdd_mutableStateArray_0[2].setOffsetAndSizeFromPreviousCursor(0, project_previousCursor_0); /* 069 */ } /* 070 */ append((rdd_mutableStateArray_0[2].getRow())); /* 071 */ if (shouldStop()) return; /* 072 */ } /* 073 */ } /* 074 */ /* 075 */ } 21/01/17 17:43:07.990 dag-scheduler-event-loop DEBUG BlockManagerMaster: Updated info of block broadcast_2_piece0 21/01/17 17:43:07.990 dag-scheduler-event-loop DEBUG BlockManager: Told master about block broadcast_2_piece0 21/01/17 17:43:07.990 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_2_piece0 locally took 2 ms 21/01/17 17:43:07.990 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_2_piece0 without replication took 2 ms 21/01/17 17:43:07.990 dag-scheduler-event-loop INFO SparkContext: Created broadcast 2 from broadcast at DAGScheduler.scala:1383 21/01/17 17:43:07.992 Thread-3 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$groupByKey$4) is now cleaned +++ 21/01/17 17:43:07.993 dag-scheduler-event-loop INFO DAGScheduler: Submitting 4 missing tasks from ResultStage 2 (ShuffledRDD[12] at aggregateByKey at ClusteringMetrics.scala:501) (first 15 tasks are for partitions Vector(0, 1, 2, 3)) 21/01/17 17:43:07.993 dag-scheduler-event-loop INFO TaskSchedulerImpl: Adding task set 2.0 with 4 tasks resource profile 0 21/01/17 17:43:07.994 Thread-3 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$cogroup$4 21/01/17 17:43:07.997 Thread-3 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$cogroup$4) is now cleaned +++ 21/01/17 17:43:07.998 Thread-3 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$join$2 21/01/17 17:43:08.001 Thread-3 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$join$2) is now cleaned +++ 21/01/17 17:43:08.002 Thread-3 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$computeFactors$4 21/01/17 17:43:08.004 Thread-3 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$computeFactors$4) is now cleaned +++ 21/01/17 17:43:08.005 Thread-3 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$train$11 21/01/17 17:43:08.007 Thread-3 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$train$11) is now cleaned +++ 21/01/17 17:43:08.008 Thread-3 DEBUG CoGroupedRDD: Adding one-to-one dependency with userInBlocks MapPartitionsRDD[19] at mapValues at ALS.scala:1640 21/01/17 17:43:08.008 Thread-3 DEBUG CoGroupedRDD: Adding one-to-one dependency with ShuffledRDD[41] at groupByKey at ALS.scala:1707 21/01/17 17:43:07.994 dag-scheduler-event-loop DEBUG TaskSetManager: Epoch for TaskSet 2.0: 1 21/01/17 17:43:08.010 dag-scheduler-event-loop DEBUG TaskSetManager: Adding pending tasks took 1 ms 21/01/17 17:43:08.012 dag-scheduler-event-loop DEBUG TaskSetManager: Valid locality levels for TaskSet 2.0: NODE_LOCAL, NO_PREF, ANY 21/01/17 17:43:08.013 dispatcher-event-loop-3 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_2.0, runningTasks: 0 21/01/17 17:43:08.015 dispatcher-event-loop-3 INFO TaskSetManager: Starting task 0.0 in stage 2.0 (TID 5) (192.168.1.225, executor driver, partition 0, NODE_LOCAL, 7143 bytes) taskResourceAssignments Map() 21/01/17 17:43:08.016 dispatcher-event-loop-3 DEBUG TaskSetManager: No tasks for locality level NODE_LOCAL, so moving to locality level NO_PREF 21/01/17 17:43:08.016 dispatcher-event-loop-3 DEBUG TaskSetManager: Moving to ANY after waiting for 0ms 21/01/17 17:43:08.017 dispatcher-event-loop-3 INFO TaskSetManager: Starting task 1.0 in stage 2.0 (TID 6) (192.168.1.225, executor driver, partition 1, PROCESS_LOCAL, 7143 bytes) taskResourceAssignments Map() 21/01/17 17:43:08.017 dispatcher-event-loop-3 INFO TaskSetManager: Starting task 2.0 in stage 2.0 (TID 7) (192.168.1.225, executor driver, partition 2, PROCESS_LOCAL, 7143 bytes) taskResourceAssignments Map() 21/01/17 17:43:08.018 dispatcher-event-loop-3 INFO TaskSetManager: Starting task 3.0 in stage 2.0 (TID 8) (192.168.1.225, executor driver, partition 3, PROCESS_LOCAL, 7143 bytes) taskResourceAssignments Map() 21/01/17 17:43:08.018 Thread-3 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$cogroup$4 21/01/17 17:43:08.018 Executor task launch worker for task 0.0 in stage 2.0 (TID 5) INFO Executor: Running task 0.0 in stage 2.0 (TID 5) 21/01/17 17:43:08.018 Executor task launch worker for task 2.0 in stage 2.0 (TID 7) INFO Executor: Running task 2.0 in stage 2.0 (TID 7) 21/01/17 17:43:08.019 Executor task launch worker for task 2.0 in stage 2.0 (TID 7) DEBUG ExecutorMetricsPoller: stageTCMP: (2, 0) -> 1 21/01/17 17:43:08.019 Executor task launch worker for task 3.0 in stage 2.0 (TID 8) INFO Executor: Running task 3.0 in stage 2.0 (TID 8) 21/01/17 17:43:08.020 Executor task launch worker for task 3.0 in stage 2.0 (TID 8) DEBUG ExecutorMetricsPoller: stageTCMP: (2, 0) -> 3 21/01/17 17:43:08.021 Executor task launch worker for task 2.0 in stage 2.0 (TID 7) DEBUG BlockManager: Getting local block broadcast_2 21/01/17 17:43:08.019 Executor task launch worker for task 1.0 in stage 2.0 (TID 6) INFO Executor: Running task 1.0 in stage 2.0 (TID 6) 21/01/17 17:43:08.021 Executor task launch worker for task 2.0 in stage 2.0 (TID 7) DEBUG BlockManager: Level for block broadcast_2 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:08.019 Executor task launch worker for task 0.0 in stage 2.0 (TID 5) DEBUG ExecutorMetricsPoller: stageTCMP: (2, 0) -> 2 21/01/17 17:43:08.026 Thread-3 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$cogroup$4) is now cleaned +++ 21/01/17 17:43:08.027 Thread-3 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$join$2 21/01/17 17:43:08.030 Executor task launch worker for task 1.0 in stage 2.0 (TID 6) DEBUG ExecutorMetricsPoller: stageTCMP: (2, 0) -> 4 21/01/17 17:43:08.030 Thread-3 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$join$2) is now cleaned +++ 21/01/17 17:43:08.032 Thread-3 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$train$12 21/01/17 17:43:08.034 Thread-3 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$train$12) is now cleaned +++ 21/01/17 17:43:08.036 Thread-3 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$train$14 21/01/17 17:43:08.024 Executor task launch worker for task 2.0 in stage 2.0 (TID 7) DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 0 21/01/17 17:43:08.038 Thread-3 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$train$14) is now cleaned +++ 21/01/17 17:43:08.040 Executor task launch worker for task 2.0 in stage 2.0 (TID 7) DEBUG MapOutputTrackerMaster: Convert map statuses for shuffle 0, mappers 0-4, partitions 2-3 21/01/17 17:43:08.042 Thread-3 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$cogroup$4 21/01/17 17:43:08.045 Executor task launch worker for task 0.0 in stage 2.0 (TID 5) DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 0 21/01/17 17:43:08.045 Executor task launch worker for task 0.0 in stage 2.0 (TID 5) DEBUG MapOutputTrackerMaster: Convert map statuses for shuffle 0, mappers 0-4, partitions 0-1 21/01/17 17:43:08.046 Thread-3 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$cogroup$4) is now cleaned +++ 21/01/17 17:43:08.047 Thread-3 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$join$2 21/01/17 17:43:08.050 Thread-3 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$join$2) is now cleaned +++ 21/01/17 17:43:08.052 Thread-3 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$train$15 21/01/17 17:43:08.053 Executor task launch worker for task 3.0 in stage 2.0 (TID 8) DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 0 21/01/17 17:43:08.053 Executor task launch worker for task 3.0 in stage 2.0 (TID 8) DEBUG MapOutputTrackerMaster: Convert map statuses for shuffle 0, mappers 0-4, partitions 3-4 21/01/17 17:43:08.054 Thread-3 INFO CodeGenerator: Code generated in 66.047776 ms 21/01/17 17:43:08.054 Executor task launch worker for task 1.0 in stage 2.0 (TID 6) DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 0 21/01/17 17:43:08.054 Executor task launch worker for task 1.0 in stage 2.0 (TID 6) DEBUG MapOutputTrackerMaster: Convert map statuses for shuffle 0, mappers 0-4, partitions 1-2 21/01/17 17:43:08.054 Thread-3 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$doExecute$4$adapted 21/01/17 17:43:08.054 Thread-3 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$train$15) is now cleaned +++ 21/01/17 17:43:08.055 Thread-3 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$doExecute$4$adapted) is now cleaned +++ 21/01/17 17:43:08.056 Thread-3 DEBUG CoGroupedRDD: Adding one-to-one dependency with MapPartitionsRDD[46] at mapValues at ALS.scala:1053 21/01/17 17:43:08.056 Thread-3 DEBUG CoGroupedRDD: Adding one-to-one dependency with MapPartitionsRDD[45] at mapValues at ALS.scala:1715 21/01/17 17:43:08.056 Thread-3 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$count$1$adapted 21/01/17 17:43:08.061 Executor task launch worker for task 1.0 in stage 2.0 (TID 6) DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 21/01/17 17:43:08.061 Thread-3 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$count$1$adapted) is now cleaned +++ 21/01/17 17:43:08.062 Thread-3 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$runJob$5 21/01/17 17:43:08.065 Executor task launch worker for task 3.0 in stage 2.0 (TID 8) DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 21/01/17 17:43:08.066 Executor task launch worker for task 3.0 in stage 2.0 (TID 8) INFO ShuffleBlockFetcherIterator: Getting 0 (0.0 B) non-empty blocks including 0 (0.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 21/01/17 17:43:08.068 Executor task launch worker for task 3.0 in stage 2.0 (TID 8) INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 7 ms 21/01/17 17:43:08.068 Executor task launch worker for task 3.0 in stage 2.0 (TID 8) DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: 21/01/17 17:43:08.068 Thread-3 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$runJob$5) is now cleaned +++ 21/01/17 17:43:08.068 Thread-3 INFO SparkContext: Starting job: count at ALS.scala:1075 21/01/17 17:43:08.068 Executor task launch worker for task 3.0 in stage 2.0 (TID 8) DEBUG ShuffleBlockFetcherIterator: Got local blocks in 8 ms 21/01/17 17:43:08.069 Executor task launch worker for task 0.0 in stage 2.0 (TID 5) DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 21/01/17 17:43:08.069 dag-scheduler-event-loop DEBUG DAGScheduler: Merging stage rdd profiles: Set() 21/01/17 17:43:08.070 dag-scheduler-event-loop DEBUG CoGroupedRDD: Adding one-to-one dependency with itemOutBlocks MapPartitionsRDD[25] at mapValues at ALS.scala:1649 21/01/17 17:43:08.070 dag-scheduler-event-loop DEBUG CoGroupedRDD: Adding one-to-one dependency with MapPartitionsRDD[36] at mapValues at ALS.scala:1715 21/01/17 17:43:08.071 dag-scheduler-event-loop DEBUG CoGroupedRDD: Adding one-to-one dependency with userOutBlocks MapPartitionsRDD[20] at mapValues at ALS.scala:1649 21/01/17 17:43:08.071 dag-scheduler-event-loop DEBUG CoGroupedRDD: Adding one-to-one dependency with MapPartitionsRDD[26] at mapPartitions at ALS.scala:1276 21/01/17 17:43:08.071 dag-scheduler-event-loop DEBUG DAGScheduler: Merging stage rdd profiles: Set() 21/01/17 17:43:08.071 Executor task launch worker for task 2.0 in stage 2.0 (TID 7) DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 21/01/17 17:43:08.072 Executor task launch worker for task 2.0 in stage 2.0 (TID 7) INFO ShuffleBlockFetcherIterator: Getting 0 (0.0 B) non-empty blocks including 0 (0.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 21/01/17 17:43:08.072 Executor task launch worker for task 2.0 in stage 2.0 (TID 7) INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 11 ms 21/01/17 17:43:08.072 Executor task launch worker for task 2.0 in stage 2.0 (TID 7) DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: 21/01/17 17:43:08.072 dag-scheduler-event-loop DEBUG DAGScheduler: Merging stage rdd profiles: Set() 21/01/17 17:43:08.072 Executor task launch worker for task 2.0 in stage 2.0 (TID 7) DEBUG ShuffleBlockFetcherIterator: Got local blocks in 11 ms 21/01/17 17:43:08.072 Executor task launch worker for task 1.0 in stage 2.0 (TID 6) INFO ShuffleBlockFetcherIterator: Getting 0 (0.0 B) non-empty blocks including 0 (0.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 21/01/17 17:43:08.072 dag-scheduler-event-loop DEBUG DAGScheduler: Merging stage rdd profiles: Set() 21/01/17 17:43:08.072 Executor task launch worker for task 1.0 in stage 2.0 (TID 6) INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 13 ms 21/01/17 17:43:08.072 Executor task launch worker for task 1.0 in stage 2.0 (TID 6) DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: 21/01/17 17:43:08.072 Executor task launch worker for task 1.0 in stage 2.0 (TID 6) DEBUG ShuffleBlockFetcherIterator: Got local blocks in 13 ms 21/01/17 17:43:08.072 dag-scheduler-event-loop DEBUG DAGScheduler: Merging stage rdd profiles: Set() 21/01/17 17:43:08.073 dag-scheduler-event-loop INFO DAGScheduler: Registering RDD 31 (flatMap at ALS.scala:1701) as input to shuffle 4 21/01/17 17:43:08.074 dag-scheduler-event-loop DEBUG DAGScheduler: Merging stage rdd profiles: Set() 21/01/17 17:43:08.074 dag-scheduler-event-loop INFO DAGScheduler: Registering RDD 40 (flatMap at ALS.scala:1701) as input to shuffle 3 21/01/17 17:43:08.075 dag-scheduler-event-loop INFO DAGScheduler: Got job 3 (count at ALS.scala:1075) with 10 output partitions 21/01/17 17:43:08.075 dag-scheduler-event-loop INFO DAGScheduler: Final stage: ResultStage 12 (count at ALS.scala:1075) 21/01/17 17:43:08.075 dag-scheduler-event-loop INFO DAGScheduler: Parents of final stage: List(ShuffleMapStage 11, ShuffleMapStage 8) 21/01/17 17:43:08.076 Executor task launch worker for task 0.0 in stage 2.0 (TID 5) INFO ShuffleBlockFetcherIterator: Getting 4 (1246.0 B) non-empty blocks including 4 (1246.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 21/01/17 17:43:08.076 Executor task launch worker for task 0.0 in stage 2.0 (TID 5) INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 7 ms 21/01/17 17:43:08.076 dag-scheduler-event-loop INFO DAGScheduler: Missing parents: List(ShuffleMapStage 11) 21/01/17 17:43:08.077 Executor task launch worker for task 0.0 in stage 2.0 (TID 5) DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_0_1_0,0), (shuffle_0_2_0,1), (shuffle_0_3_0,2), (shuffle_0_4_0,3) 21/01/17 17:43:08.077 dag-scheduler-event-loop DEBUG DAGScheduler: submitStage(ResultStage 12 (name=count at ALS.scala:1075;jobs=3)) 21/01/17 17:43:08.077 Executor task launch worker for task 0.0 in stage 2.0 (TID 5) DEBUG BlockManager: Getting local shuffle block shuffle_0_1_0 21/01/17 17:43:08.077 dag-scheduler-event-loop DEBUG DAGScheduler: missing: List(ShuffleMapStage 11) 21/01/17 17:43:08.077 dag-scheduler-event-loop DEBUG DAGScheduler: submitStage(ShuffleMapStage 11 (name=flatMap at ALS.scala:1701;jobs=3)) 21/01/17 17:43:08.079 dag-scheduler-event-loop DEBUG DAGScheduler: missing: List(ShuffleMapStage 10) 21/01/17 17:43:08.079 dag-scheduler-event-loop DEBUG DAGScheduler: submitStage(ShuffleMapStage 10 (name=flatMap at ALS.scala:1701;jobs=3)) 21/01/17 17:43:08.080 dag-scheduler-event-loop DEBUG DAGScheduler: missing: List() 21/01/17 17:43:08.080 dag-scheduler-event-loop INFO DAGScheduler: Submitting ShuffleMapStage 10 (MapPartitionsRDD[31] at flatMap at ALS.scala:1701), which has no missing parents 21/01/17 17:43:08.080 dag-scheduler-event-loop DEBUG DAGScheduler: submitMissingTasks(ShuffleMapStage 10) 21/01/17 17:43:08.080 Executor task launch worker for task 0.0 in stage 2.0 (TID 5) DEBUG BlockManager: Getting local shuffle block shuffle_0_2_0 21/01/17 17:43:08.080 Executor task launch worker for task 0.0 in stage 2.0 (TID 5) DEBUG BlockManager: Getting local shuffle block shuffle_0_3_0 21/01/17 17:43:08.081 Executor task launch worker for task 0.0 in stage 2.0 (TID 5) DEBUG BlockManager: Getting local shuffle block shuffle_0_4_0 21/01/17 17:43:08.081 Executor task launch worker for task 0.0 in stage 2.0 (TID 5) DEBUG ShuffleBlockFetcherIterator: Got local blocks in 11 ms 21/01/17 17:43:08.081 Executor task launch worker for task 2.0 in stage 2.0 (TID 7) DEBUG TaskMemoryManager: Task 7 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@632cd0e0 21/01/17 17:43:08.083 Executor task launch worker for task 2.0 in stage 2.0 (TID 7) INFO Executor: Finished task 2.0 in stage 2.0 (TID 7). 1263 bytes result sent to driver 21/01/17 17:43:08.083 dispatcher-event-loop-3 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_2.0, runningTasks: 3 21/01/17 17:43:08.084 dispatcher-event-loop-3 DEBUG TaskSetManager: No tasks for locality level NODE_LOCAL, so moving to locality level NO_PREF 21/01/17 17:43:08.084 dispatcher-event-loop-3 DEBUG TaskSetManager: No tasks for locality level NO_PREF, so moving to locality level ANY 21/01/17 17:43:08.084 Executor task launch worker for task 2.0 in stage 2.0 (TID 7) DEBUG ExecutorMetricsPoller: stageTCMP: (2, 0) -> 3 21/01/17 17:43:08.084 task-result-getter-1 INFO TaskSetManager: Finished task 2.0 in stage 2.0 (TID 7) in 67 ms on 192.168.1.225 (executor driver) (1/4) 21/01/17 17:43:08.085 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_6 stored as values in memory (estimated size 35.5 KiB, free 366.0 MiB) 21/01/17 17:43:08.086 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_6 locally took 1 ms 21/01/17 17:43:08.086 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_6 without replication took 1 ms 21/01/17 17:43:08.087 Executor task launch worker for task 3.0 in stage 2.0 (TID 8) DEBUG TaskMemoryManager: Task 8 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@66dc7633 21/01/17 17:43:08.088 Thread-3 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$rdd$1 21/01/17 17:43:08.089 Executor task launch worker for task 1.0 in stage 2.0 (TID 6) DEBUG TaskMemoryManager: Task 6 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@59c90066 21/01/17 17:43:08.094 Executor task launch worker for task 1.0 in stage 2.0 (TID 6) INFO Executor: Finished task 1.0 in stage 2.0 (TID 6). 1263 bytes result sent to driver 21/01/17 17:43:08.095 Executor task launch worker for task 3.0 in stage 2.0 (TID 8) INFO Executor: Finished task 3.0 in stage 2.0 (TID 8). 1263 bytes result sent to driver 21/01/17 17:43:08.095 Executor task launch worker for task 3.0 in stage 2.0 (TID 8) DEBUG ExecutorMetricsPoller: stageTCMP: (2, 0) -> 2 21/01/17 17:43:08.095 Executor task launch worker for task 1.0 in stage 2.0 (TID 6) DEBUG ExecutorMetricsPoller: stageTCMP: (2, 0) -> 1 21/01/17 17:43:08.095 task-result-getter-2 INFO TaskSetManager: Finished task 1.0 in stage 2.0 (TID 6) in 78 ms on 192.168.1.225 (executor driver) (2/4) 21/01/17 17:43:08.096 task-result-getter-0 INFO TaskSetManager: Finished task 3.0 in stage 2.0 (TID 8) in 78 ms on 192.168.1.225 (executor driver) (3/4) 21/01/17 17:43:08.097 Thread-3 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$rdd$1) is now cleaned +++ 21/01/17 17:43:08.098 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_6_piece0 stored as bytes in memory (estimated size 16.5 KiB, free 366.0 MiB) 21/01/17 17:43:08.099 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master broadcast_6_piece0 for BlockManagerId(driver, 192.168.1.225, 40985, None) 21/01/17 17:43:08.099 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added broadcast_6_piece0 in memory on 192.168.1.225:40985 (size: 16.5 KiB, free: 366.2 MiB) 21/01/17 17:43:08.100 dag-scheduler-event-loop DEBUG BlockManagerMaster: Updated info of block broadcast_6_piece0 21/01/17 17:43:08.100 dag-scheduler-event-loop DEBUG BlockManager: Told master about block broadcast_6_piece0 21/01/17 17:43:08.100 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_6_piece0 locally took 13 ms 21/01/17 17:43:08.100 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_6_piece0 without replication took 13 ms 21/01/17 17:43:08.100 dag-scheduler-event-loop INFO SparkContext: Created broadcast 6 from broadcast at DAGScheduler.scala:1383 21/01/17 17:43:08.101 dag-scheduler-event-loop INFO DAGScheduler: Submitting 10 missing tasks from ShuffleMapStage 10 (MapPartitionsRDD[31] at flatMap at ALS.scala:1701) (first 15 tasks are for partitions Vector(0, 1, 2, 3, 4, 5, 6, 7, 8, 9)) 21/01/17 17:43:08.102 dag-scheduler-event-loop INFO TaskSchedulerImpl: Adding task set 10.0 with 10 tasks resource profile 0 21/01/17 17:43:08.102 dag-scheduler-event-loop DEBUG TaskSetManager: Epoch for TaskSet 10.0: 3 21/01/17 17:43:08.102 dag-scheduler-event-loop DEBUG TaskSetManager: Adding pending tasks took 0 ms 21/01/17 17:43:08.102 dag-scheduler-event-loop DEBUG TaskSetManager: Valid locality levels for TaskSet 10.0: PROCESS_LOCAL, NODE_LOCAL, ANY 21/01/17 17:43:08.104 dispatcher-event-loop-1 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_10.0, runningTasks: 0 21/01/17 17:43:08.106 Thread-3 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$fit$2 21/01/17 17:43:08.106 dispatcher-event-loop-1 INFO TaskSetManager: Starting task 0.0 in stage 10.0 (TID 33) (192.168.1.225, executor driver, partition 0, PROCESS_LOCAL, 7362 bytes) taskResourceAssignments Map() 21/01/17 17:43:08.106 Thread-3 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$fit$2) is now cleaned +++ 21/01/17 17:43:08.107 dispatcher-event-loop-1 INFO TaskSetManager: Starting task 1.0 in stage 10.0 (TID 34) (192.168.1.225, executor driver, partition 1, PROCESS_LOCAL, 7362 bytes) taskResourceAssignments Map() 21/01/17 17:43:08.107 dispatcher-event-loop-1 INFO TaskSetManager: Starting task 2.0 in stage 10.0 (TID 35) (192.168.1.225, executor driver, partition 2, PROCESS_LOCAL, 7362 bytes) taskResourceAssignments Map() 21/01/17 17:43:08.107 dispatcher-event-loop-1 INFO TaskSetManager: Starting task 3.0 in stage 10.0 (TID 36) (192.168.1.225, executor driver, partition 3, PROCESS_LOCAL, 7362 bytes) taskResourceAssignments Map() 21/01/17 17:43:08.108 Executor task launch worker for task 0.0 in stage 10.0 (TID 33) INFO Executor: Running task 0.0 in stage 10.0 (TID 33) 21/01/17 17:43:08.108 Executor task launch worker for task 1.0 in stage 10.0 (TID 34) INFO Executor: Running task 1.0 in stage 10.0 (TID 34) 21/01/17 17:43:08.109 Executor task launch worker for task 2.0 in stage 10.0 (TID 35) INFO Executor: Running task 2.0 in stage 10.0 (TID 35) 21/01/17 17:43:08.109 Executor task launch worker for task 2.0 in stage 10.0 (TID 35) DEBUG ExecutorMetricsPoller: stageTCMP: (10, 0) -> 1 21/01/17 17:43:08.109 Executor task launch worker for task 1.0 in stage 10.0 (TID 34) DEBUG ExecutorMetricsPoller: stageTCMP: (10, 0) -> 2 21/01/17 17:43:08.110 Executor task launch worker for task 2.0 in stage 10.0 (TID 35) DEBUG BlockManager: Getting local block broadcast_6 21/01/17 17:43:08.110 Executor task launch worker for task 0.0 in stage 10.0 (TID 33) DEBUG ExecutorMetricsPoller: stageTCMP: (10, 0) -> 3 21/01/17 17:43:08.110 Executor task launch worker for task 2.0 in stage 10.0 (TID 35) DEBUG BlockManager: Level for block broadcast_6 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:08.111 Executor task launch worker for task 3.0 in stage 10.0 (TID 36) INFO Executor: Running task 3.0 in stage 10.0 (TID 36) 21/01/17 17:43:08.112 Executor task launch worker for task 3.0 in stage 10.0 (TID 36) DEBUG ExecutorMetricsPoller: stageTCMP: (10, 0) -> 4 21/01/17 17:43:08.115 Thread-3 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$fit$4 21/01/17 17:43:08.116 Thread-3 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$fit$4) is now cleaned +++ 21/01/17 17:43:08.132 Executor task launch worker for task 0.0 in stage 10.0 (TID 33) DEBUG BlockManager: Getting local block rdd_20_0 21/01/17 17:43:08.132 Executor task launch worker for task 3.0 in stage 10.0 (TID 36) DEBUG BlockManager: Getting local block rdd_20_3 21/01/17 17:43:08.132 Executor task launch worker for task 0.0 in stage 10.0 (TID 33) DEBUG BlockManager: Level for block rdd_20_0 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:08.132 Executor task launch worker for task 3.0 in stage 10.0 (TID 36) DEBUG BlockManager: Level for block rdd_20_3 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:08.132 Executor task launch worker for task 0.0 in stage 10.0 (TID 33) INFO BlockManager: Found block rdd_20_0 locally 21/01/17 17:43:08.132 Executor task launch worker for task 3.0 in stage 10.0 (TID 36) INFO BlockManager: Found block rdd_20_3 locally 21/01/17 17:43:08.132 Executor task launch worker for task 0.0 in stage 10.0 (TID 33) DEBUG BlockManager: Getting local block rdd_19_0 21/01/17 17:43:08.132 Executor task launch worker for task 3.0 in stage 10.0 (TID 36) DEBUG BlockManager: Getting local block rdd_19_3 21/01/17 17:43:08.132 Executor task launch worker for task 0.0 in stage 10.0 (TID 33) DEBUG BlockManager: Level for block rdd_19_0 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:08.132 Executor task launch worker for task 3.0 in stage 10.0 (TID 36) DEBUG BlockManager: Level for block rdd_19_3 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:08.132 Executor task launch worker for task 0.0 in stage 10.0 (TID 33) INFO BlockManager: Found block rdd_19_0 locally 21/01/17 17:43:08.132 Executor task launch worker for task 3.0 in stage 10.0 (TID 36) INFO BlockManager: Found block rdd_19_3 locally 21/01/17 17:43:08.133 Executor task launch worker for task 1.0 in stage 10.0 (TID 34) DEBUG BlockManager: Getting local block rdd_20_1 21/01/17 17:43:08.134 Executor task launch worker for task 1.0 in stage 10.0 (TID 34) DEBUG BlockManager: Level for block rdd_20_1 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:08.134 Executor task launch worker for task 1.0 in stage 10.0 (TID 34) INFO BlockManager: Found block rdd_20_1 locally 21/01/17 17:43:08.134 Executor task launch worker for task 1.0 in stage 10.0 (TID 34) DEBUG BlockManager: Getting local block rdd_19_1 21/01/17 17:43:08.134 Executor task launch worker for task 1.0 in stage 10.0 (TID 34) DEBUG BlockManager: Level for block rdd_19_1 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:08.134 Executor task launch worker for task 1.0 in stage 10.0 (TID 34) INFO BlockManager: Found block rdd_19_1 locally 21/01/17 17:43:08.133 Thread-3 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$reduceByKey$2 21/01/17 17:43:08.136 Thread-3 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$reduceByKey$2) is now cleaned +++ 21/01/17 17:43:08.136 Executor task launch worker for task 2.0 in stage 10.0 (TID 35) DEBUG BlockManager: Getting local block rdd_20_2 21/01/17 17:43:08.136 Executor task launch worker for task 2.0 in stage 10.0 (TID 35) DEBUG BlockManager: Level for block rdd_20_2 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:08.136 Executor task launch worker for task 2.0 in stage 10.0 (TID 35) INFO BlockManager: Found block rdd_20_2 locally 21/01/17 17:43:08.136 Executor task launch worker for task 2.0 in stage 10.0 (TID 35) DEBUG BlockManager: Getting local block rdd_19_2 21/01/17 17:43:08.136 Executor task launch worker for task 2.0 in stage 10.0 (TID 35) DEBUG BlockManager: Level for block rdd_19_2 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:08.137 Executor task launch worker for task 2.0 in stage 10.0 (TID 35) INFO BlockManager: Found block rdd_19_2 locally 21/01/17 17:43:08.138 Executor task launch worker for task 0.0 in stage 2.0 (TID 5) DEBUG TaskMemoryManager: Task 5 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@6726e202 21/01/17 17:43:08.139 Executor task launch worker for task 3.0 in stage 10.0 (TID 36) DEBUG TaskMemoryManager: Task 36 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@fb4d834 21/01/17 17:43:08.139 Executor task launch worker for task 3.0 in stage 10.0 (TID 36) DEBUG LocalDiskShuffleMapOutputWriter: Writing shuffle index file for mapId 36 with length 10 21/01/17 17:43:08.139 Executor task launch worker for task 3.0 in stage 10.0 (TID 36) DEBUG IndexShuffleBlockResolver: Shuffle index for mapId 36: [0,0,0,0,0,0,0,0,0,0] 21/01/17 17:43:08.140 Executor task launch worker for task 3.0 in stage 10.0 (TID 36) INFO Executor: Finished task 3.0 in stage 10.0 (TID 36). 1950 bytes result sent to driver 21/01/17 17:43:08.141 dispatcher-event-loop-2 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_10.0, runningTasks: 3 21/01/17 17:43:08.141 dispatcher-event-loop-2 INFO TaskSetManager: Starting task 4.0 in stage 10.0 (TID 37) (192.168.1.225, executor driver, partition 4, PROCESS_LOCAL, 7362 bytes) taskResourceAssignments Map() 21/01/17 17:43:08.141 Executor task launch worker for task 1.0 in stage 10.0 (TID 34) WARN BLAS: Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS 21/01/17 17:43:08.142 Executor task launch worker for task 3.0 in stage 10.0 (TID 36) DEBUG ExecutorMetricsPoller: stageTCMP: (10, 0) -> 3 21/01/17 17:43:08.142 Executor task launch worker for task 4.0 in stage 10.0 (TID 37) INFO Executor: Running task 4.0 in stage 10.0 (TID 37) 21/01/17 17:43:08.142 Executor task launch worker for task 1.0 in stage 10.0 (TID 34) WARN BLAS: Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS 21/01/17 17:43:08.142 Executor task launch worker for task 4.0 in stage 10.0 (TID 37) DEBUG ExecutorMetricsPoller: stageTCMP: (10, 0) -> 4 21/01/17 17:43:08.145 Executor task launch worker for task 0.0 in stage 2.0 (TID 5) INFO Executor: Finished task 0.0 in stage 2.0 (TID 5). 1683 bytes result sent to driver 21/01/17 17:43:08.146 Executor task launch worker for task 0.0 in stage 2.0 (TID 5) DEBUG ExecutorMetricsPoller: removing (2, 0) from stageTCMP 21/01/17 17:43:08.147 task-result-getter-3 INFO TaskSetManager: Finished task 0.0 in stage 2.0 (TID 5) in 134 ms on 192.168.1.225 (executor driver) (4/4) 21/01/17 17:43:08.147 task-result-getter-3 INFO TaskSchedulerImpl: Removed TaskSet 2.0, whose tasks have all completed, from pool 21/01/17 17:43:08.147 task-result-getter-0 INFO TaskSetManager: Finished task 3.0 in stage 10.0 (TID 36) in 40 ms on 192.168.1.225 (executor driver) (1/10) 21/01/17 17:43:08.148 dag-scheduler-event-loop INFO DAGScheduler: ResultStage 2 (collectAsMap at ClusteringMetrics.scala:515) finished in 0.164 s 21/01/17 17:43:08.148 Executor task launch worker for task 4.0 in stage 10.0 (TID 37) DEBUG BlockManager: Getting local block rdd_20_4 21/01/17 17:43:08.148 dag-scheduler-event-loop DEBUG DAGScheduler: After removal of stage 2, remaining stages = 1 21/01/17 17:43:08.148 Executor task launch worker for task 4.0 in stage 10.0 (TID 37) DEBUG BlockManager: Level for block rdd_20_4 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:08.148 dag-scheduler-event-loop DEBUG DAGScheduler: After removal of stage 1, remaining stages = 0 21/01/17 17:43:08.148 Executor task launch worker for task 4.0 in stage 10.0 (TID 37) INFO BlockManager: Found block rdd_20_4 locally 21/01/17 17:43:08.148 Executor task launch worker for task 4.0 in stage 10.0 (TID 37) DEBUG BlockManager: Getting local block rdd_19_4 21/01/17 17:43:08.148 dag-scheduler-event-loop INFO DAGScheduler: Job 1 is finished. Cancelling potential speculative or zombie tasks for this job 21/01/17 17:43:08.148 dag-scheduler-event-loop INFO TaskSchedulerImpl: Killing all running tasks in stage 2: Stage finished 21/01/17 17:43:08.148 Executor task launch worker for task 4.0 in stage 10.0 (TID 37) DEBUG BlockManager: Level for block rdd_19_4 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:08.148 Executor task launch worker for task 4.0 in stage 10.0 (TID 37) INFO BlockManager: Found block rdd_19_4 locally 21/01/17 17:43:08.149 Executor task launch worker for task 4.0 in stage 10.0 (TID 37) DEBUG TaskMemoryManager: Task 37 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@9dcdef1 21/01/17 17:43:08.149 Executor task launch worker for task 4.0 in stage 10.0 (TID 37) DEBUG LocalDiskShuffleMapOutputWriter: Writing shuffle index file for mapId 37 with length 10 21/01/17 17:43:08.149 Thread-3 INFO DAGScheduler: Job 1 finished: collectAsMap at ClusteringMetrics.scala:515, took 1.046066 s 21/01/17 17:43:08.150 Executor task launch worker for task 4.0 in stage 10.0 (TID 37) DEBUG IndexShuffleBlockResolver: Shuffle index for mapId 37: [0,0,0,0,0,0,0,0,0,0] 21/01/17 17:43:08.152 dag-scheduler-event-loop DEBUG DAGScheduler: ShuffleMapTask finished on driver 21/01/17 17:43:08.152 Executor task launch worker for task 4.0 in stage 10.0 (TID 37) INFO Executor: Finished task 4.0 in stage 10.0 (TID 37). 1950 bytes result sent to driver 21/01/17 17:43:08.153 dispatcher-event-loop-1 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_10.0, runningTasks: 3 21/01/17 17:43:08.153 dispatcher-event-loop-1 INFO TaskSetManager: Starting task 5.0 in stage 10.0 (TID 38) (192.168.1.225, executor driver, partition 5, PROCESS_LOCAL, 7362 bytes) taskResourceAssignments Map() 21/01/17 17:43:08.154 task-result-getter-2 INFO TaskSetManager: Finished task 4.0 in stage 10.0 (TID 37) in 12 ms on 192.168.1.225 (executor driver) (2/10) 21/01/17 17:43:08.154 Thread-3 INFO MemoryStore: Block broadcast_3 stored as values in memory (estimated size 424.0 B, free 366.3 MiB) 21/01/17 17:43:08.154 Executor task launch worker for task 4.0 in stage 10.0 (TID 37) DEBUG ExecutorMetricsPoller: stageTCMP: (10, 0) -> 3 21/01/17 17:43:08.154 Thread-3 DEBUG BlockManager: Put block broadcast_3 locally took 1 ms 21/01/17 17:43:08.154 Thread-3 DEBUG BlockManager: Putting block broadcast_3 without replication took 1 ms 21/01/17 17:43:08.156 dag-scheduler-event-loop DEBUG DAGScheduler: ShuffleMapTask finished on driver 21/01/17 17:43:08.156 Thread-3 INFO MemoryStore: Block broadcast_3_piece0 stored as bytes in memory (estimated size 443.0 B, free 366.3 MiB) 21/01/17 17:43:08.156 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master broadcast_3_piece0 for BlockManagerId(driver, 192.168.1.225, 33787, None) 21/01/17 17:43:08.157 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added broadcast_3_piece0 in memory on 192.168.1.225:33787 (size: 443.0 B, free: 366.3 MiB) 21/01/17 17:43:08.161 Executor task launch worker for task 5.0 in stage 10.0 (TID 38) INFO Executor: Running task 5.0 in stage 10.0 (TID 38) 21/01/17 17:43:08.162 Executor task launch worker for task 5.0 in stage 10.0 (TID 38) DEBUG ExecutorMetricsPoller: stageTCMP: (10, 0) -> 4 21/01/17 17:43:08.163 Executor task launch worker for task 0.0 in stage 10.0 (TID 33) DEBUG TaskMemoryManager: Task 33 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@7259998e 21/01/17 17:43:08.164 Executor task launch worker for task 2.0 in stage 10.0 (TID 35) DEBUG TaskMemoryManager: Task 35 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@388f4785 21/01/17 17:43:08.164 Thread-3 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$fit$9 21/01/17 17:43:08.165 Thread-3 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$fit$9) is now cleaned +++ 21/01/17 17:43:08.166 Executor task launch worker for task 0.0 in stage 10.0 (TID 33) DEBUG LocalDiskShuffleMapOutputWriter: Writing shuffle index file for mapId 33 with length 10 21/01/17 17:43:08.166 Executor task launch worker for task 0.0 in stage 10.0 (TID 33) DEBUG IndexShuffleBlockResolver: Shuffle index for mapId 33: [228,229,213,213,213,212,213,213,213,213] 21/01/17 17:43:08.166 Thread-3 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$fit$9 21/01/17 17:43:08.167 Thread-3 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$fit$9) is now cleaned +++ 21/01/17 17:43:08.167 Thread-3 DEBUG BlockManagerMaster: Updated info of block broadcast_3_piece0 21/01/17 17:43:08.167 Thread-3 DEBUG BlockManager: Told master about block broadcast_3_piece0 21/01/17 17:43:08.167 Thread-3 DEBUG BlockManager: Put block broadcast_3_piece0 locally took 11 ms 21/01/17 17:43:08.167 Thread-3 DEBUG BlockManager: Putting block broadcast_3_piece0 without replication took 11 ms 21/01/17 17:43:08.168 Thread-3 INFO SparkContext: Created broadcast 3 from broadcast at ClusteringMetrics.scala:579 21/01/17 17:43:08.169 Thread-3 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$fit$11 21/01/17 17:43:08.170 Thread-3 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$fit$11) is now cleaned +++ 21/01/17 17:43:08.180 Executor task launch worker for task 5.0 in stage 10.0 (TID 38) DEBUG BlockManager: Getting local block rdd_20_5 21/01/17 17:43:08.180 Executor task launch worker for task 5.0 in stage 10.0 (TID 38) DEBUG BlockManager: Level for block rdd_20_5 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:08.180 Executor task launch worker for task 5.0 in stage 10.0 (TID 38) INFO BlockManager: Found block rdd_20_5 locally 21/01/17 17:43:08.180 Executor task launch worker for task 5.0 in stage 10.0 (TID 38) DEBUG BlockManager: Getting local block rdd_19_5 21/01/17 17:43:08.180 Executor task launch worker for task 5.0 in stage 10.0 (TID 38) DEBUG BlockManager: Level for block rdd_19_5 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:08.180 Executor task launch worker for task 5.0 in stage 10.0 (TID 38) INFO BlockManager: Found block rdd_19_5 locally 21/01/17 17:43:08.181 Executor task launch worker for task 5.0 in stage 10.0 (TID 38) DEBUG TaskMemoryManager: Task 38 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@7cdc26d1 21/01/17 17:43:08.181 Executor task launch worker for task 5.0 in stage 10.0 (TID 38) DEBUG LocalDiskShuffleMapOutputWriter: Writing shuffle index file for mapId 38 with length 10 21/01/17 17:43:08.181 Executor task launch worker for task 1.0 in stage 10.0 (TID 34) DEBUG TaskMemoryManager: Task 34 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@e1132d8 21/01/17 17:43:08.181 Executor task launch worker for task 5.0 in stage 10.0 (TID 38) DEBUG IndexShuffleBlockResolver: Shuffle index for mapId 38: [0,0,0,0,0,0,0,0,0,0] 21/01/17 17:43:08.183 Executor task launch worker for task 0.0 in stage 10.0 (TID 33) INFO Executor: Finished task 0.0 in stage 10.0 (TID 33). 2122 bytes result sent to driver 21/01/17 17:43:08.183 Executor task launch worker for task 0.0 in stage 10.0 (TID 33) DEBUG ExecutorMetricsPoller: stageTCMP: (10, 0) -> 3 21/01/17 17:43:08.184 dispatcher-event-loop-2 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_10.0, runningTasks: 3 21/01/17 17:43:08.184 Executor task launch worker for task 2.0 in stage 10.0 (TID 35) DEBUG LocalDiskShuffleMapOutputWriter: Writing shuffle index file for mapId 35 with length 10 21/01/17 17:43:08.184 Executor task launch worker for task 2.0 in stage 10.0 (TID 35) DEBUG IndexShuffleBlockResolver: Shuffle index for mapId 35: [214,231,228,215,215,214,215,215,215,215] 21/01/17 17:43:08.184 dispatcher-event-loop-2 INFO TaskSetManager: Starting task 6.0 in stage 10.0 (TID 39) (192.168.1.225, executor driver, partition 6, PROCESS_LOCAL, 7362 bytes) taskResourceAssignments Map() 21/01/17 17:43:08.184 Executor task launch worker for task 5.0 in stage 10.0 (TID 38) INFO Executor: Finished task 5.0 in stage 10.0 (TID 38). 1950 bytes result sent to driver 21/01/17 17:43:08.185 Thread-3 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$count$1$adapted 21/01/17 17:43:08.185 Executor task launch worker for task 2.0 in stage 10.0 (TID 35) INFO Executor: Finished task 2.0 in stage 10.0 (TID 35). 2122 bytes result sent to driver 21/01/17 17:43:08.184 Executor task launch worker for task 1.0 in stage 10.0 (TID 34) DEBUG LocalDiskShuffleMapOutputWriter: Writing shuffle index file for mapId 34 with length 10 21/01/17 17:43:08.185 task-result-getter-3 INFO TaskSetManager: Finished task 0.0 in stage 10.0 (TID 33) in 81 ms on 192.168.1.225 (executor driver) (3/10) 21/01/17 17:43:08.185 Executor task launch worker for task 6.0 in stage 10.0 (TID 39) INFO Executor: Running task 6.0 in stage 10.0 (TID 39) 21/01/17 17:43:08.186 Executor task launch worker for task 1.0 in stage 10.0 (TID 34) DEBUG IndexShuffleBlockResolver: Shuffle index for mapId 34: [214,226,231,215,215,214,215,215,215,215] 21/01/17 17:43:08.186 dispatcher-event-loop-0 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_10.0, runningTasks: 3 21/01/17 17:43:08.192 Thread-3 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$count$1$adapted) is now cleaned +++ 21/01/17 17:43:08.194 Thread-3 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$runJob$5 21/01/17 17:43:08.196 dispatcher-event-loop-0 INFO TaskSetManager: Starting task 7.0 in stage 10.0 (TID 40) (192.168.1.225, executor driver, partition 7, PROCESS_LOCAL, 7362 bytes) taskResourceAssignments Map() 21/01/17 17:43:08.196 Executor task launch worker for task 1.0 in stage 10.0 (TID 34) INFO Executor: Finished task 1.0 in stage 10.0 (TID 34). 2122 bytes result sent to driver 21/01/17 17:43:08.196 Thread-3 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$runJob$5) is now cleaned +++ 21/01/17 17:43:08.197 Executor task launch worker for task 7.0 in stage 10.0 (TID 40) INFO Executor: Running task 7.0 in stage 10.0 (TID 40) 21/01/17 17:43:08.197 Thread-3 INFO SparkContext: Starting job: count at CountVectorizer.scala:233 21/01/17 17:43:08.197 Executor task launch worker for task 1.0 in stage 10.0 (TID 34) DEBUG ExecutorMetricsPoller: stageTCMP: (10, 0) -> 2 21/01/17 17:43:08.198 Executor task launch worker for task 2.0 in stage 10.0 (TID 35) DEBUG ExecutorMetricsPoller: stageTCMP: (10, 0) -> 1 21/01/17 17:43:08.198 Executor task launch worker for task 7.0 in stage 10.0 (TID 40) DEBUG ExecutorMetricsPoller: stageTCMP: (10, 0) -> 2 21/01/17 17:43:08.198 Executor task launch worker for task 6.0 in stage 10.0 (TID 39) DEBUG ExecutorMetricsPoller: stageTCMP: (10, 0) -> 3 21/01/17 17:43:08.199 dag-scheduler-event-loop DEBUG DAGScheduler: ShuffleMapTask finished on driver 21/01/17 17:43:08.199 dispatcher-event-loop-1 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_10.0, runningTasks: 3 21/01/17 17:43:08.200 dispatcher-event-loop-1 INFO TaskSetManager: Starting task 8.0 in stage 10.0 (TID 41) (192.168.1.225, executor driver, partition 8, PROCESS_LOCAL, 7362 bytes) taskResourceAssignments Map() 21/01/17 17:43:08.200 task-result-getter-1 INFO TaskSetManager: Finished task 5.0 in stage 10.0 (TID 38) in 47 ms on 192.168.1.225 (executor driver) (4/10) 21/01/17 17:43:08.200 task-result-getter-1 INFO TaskSetManager: Finished task 1.0 in stage 10.0 (TID 34) in 93 ms on 192.168.1.225 (executor driver) (5/10) 21/01/17 17:43:08.201 dispatcher-event-loop-1 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_10.0, runningTasks: 3 21/01/17 17:43:08.201 Executor task launch worker for task 5.0 in stage 10.0 (TID 38) DEBUG ExecutorMetricsPoller: stageTCMP: (10, 0) -> 2 21/01/17 17:43:08.201 dispatcher-event-loop-1 INFO TaskSetManager: Starting task 9.0 in stage 10.0 (TID 42) (192.168.1.225, executor driver, partition 9, PROCESS_LOCAL, 7362 bytes) taskResourceAssignments Map() 21/01/17 17:43:08.202 dag-scheduler-event-loop DEBUG DAGScheduler: ShuffleMapTask finished on driver 21/01/17 17:43:08.203 Executor task launch worker for task 8.0 in stage 10.0 (TID 41) INFO Executor: Running task 8.0 in stage 10.0 (TID 41) 21/01/17 17:43:08.203 dag-scheduler-event-loop DEBUG DAGScheduler: ShuffleMapTask finished on driver 21/01/17 17:43:08.203 Executor task launch worker for task 8.0 in stage 10.0 (TID 41) DEBUG ExecutorMetricsPoller: stageTCMP: (10, 0) -> 3 21/01/17 17:43:08.203 Executor task launch worker for task 6.0 in stage 10.0 (TID 39) DEBUG BlockManager: Getting local block rdd_20_6 21/01/17 17:43:08.203 Executor task launch worker for task 6.0 in stage 10.0 (TID 39) DEBUG BlockManager: Level for block rdd_20_6 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:08.203 Executor task launch worker for task 6.0 in stage 10.0 (TID 39) INFO BlockManager: Found block rdd_20_6 locally 21/01/17 17:43:08.203 Executor task launch worker for task 6.0 in stage 10.0 (TID 39) DEBUG BlockManager: Getting local block rdd_19_6 21/01/17 17:43:08.204 Executor task launch worker for task 6.0 in stage 10.0 (TID 39) DEBUG BlockManager: Level for block rdd_19_6 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:08.204 Executor task launch worker for task 6.0 in stage 10.0 (TID 39) INFO BlockManager: Found block rdd_19_6 locally 21/01/17 17:43:08.204 Executor task launch worker for task 6.0 in stage 10.0 (TID 39) DEBUG TaskMemoryManager: Task 39 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@5e8e4575 21/01/17 17:43:08.204 Executor task launch worker for task 6.0 in stage 10.0 (TID 39) DEBUG LocalDiskShuffleMapOutputWriter: Writing shuffle index file for mapId 39 with length 10 21/01/17 17:43:08.205 Executor task launch worker for task 6.0 in stage 10.0 (TID 39) DEBUG IndexShuffleBlockResolver: Shuffle index for mapId 39: [0,0,0,0,0,0,0,0,0,0] 21/01/17 17:43:08.202 dag-scheduler-event-loop DEBUG SortShuffleManager: Can't use serialized shuffle for shuffle 0 because the serializer, org.apache.spark.serializer.JavaSerializer, does not support object relocation 21/01/17 17:43:08.203 Executor task launch worker for task 9.0 in stage 10.0 (TID 42) INFO Executor: Running task 9.0 in stage 10.0 (TID 42) 21/01/17 17:43:08.206 Executor task launch worker for task 9.0 in stage 10.0 (TID 42) DEBUG ExecutorMetricsPoller: stageTCMP: (10, 0) -> 4 21/01/17 17:43:08.206 Executor task launch worker for task 6.0 in stage 10.0 (TID 39) INFO Executor: Finished task 6.0 in stage 10.0 (TID 39). 1950 bytes result sent to driver 21/01/17 17:43:08.207 Executor task launch worker for task 6.0 in stage 10.0 (TID 39) DEBUG ExecutorMetricsPoller: stageTCMP: (10, 0) -> 3 21/01/17 17:43:08.207 dag-scheduler-event-loop DEBUG DAGScheduler: Merging stage rdd profiles: Set() 21/01/17 17:43:08.213 dag-scheduler-event-loop DEBUG DAGScheduler: Merging stage rdd profiles: Set() 21/01/17 17:43:08.215 dag-scheduler-event-loop INFO DAGScheduler: Registering RDD 19 (flatMap at CountVectorizer.scala:212) as input to shuffle 0 21/01/17 17:43:08.218 dag-scheduler-event-loop INFO DAGScheduler: Got job 2 (count at CountVectorizer.scala:233) with 4 output partitions 21/01/17 17:43:08.218 dag-scheduler-event-loop INFO DAGScheduler: Final stage: ResultStage 3 (count at CountVectorizer.scala:233) 21/01/17 17:43:08.219 dag-scheduler-event-loop INFO DAGScheduler: Parents of final stage: List(ShuffleMapStage 2) 21/01/17 17:43:08.217 dispatcher-event-loop-3 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_10.0, runningTasks: 3 21/01/17 17:43:08.222 dispatcher-event-loop-3 DEBUG TaskSetManager: No tasks for locality level PROCESS_LOCAL, so moving to locality level NODE_LOCAL 21/01/17 17:43:08.222 dispatcher-event-loop-3 DEBUG TaskSetManager: No tasks for locality level NODE_LOCAL, so moving to locality level ANY 21/01/17 17:43:08.222 task-result-getter-3 INFO TaskSetManager: Finished task 6.0 in stage 10.0 (TID 39) in 38 ms on 192.168.1.225 (executor driver) (6/10) 21/01/17 17:43:08.223 Executor task launch worker for task 7.0 in stage 10.0 (TID 40) DEBUG BlockManager: Getting local block rdd_20_7 21/01/17 17:43:08.223 Executor task launch worker for task 7.0 in stage 10.0 (TID 40) DEBUG BlockManager: Level for block rdd_20_7 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:08.223 Executor task launch worker for task 7.0 in stage 10.0 (TID 40) INFO BlockManager: Found block rdd_20_7 locally 21/01/17 17:43:08.223 task-result-getter-0 INFO TaskSetManager: Finished task 2.0 in stage 10.0 (TID 35) in 116 ms on 192.168.1.225 (executor driver) (7/10) 21/01/17 17:43:08.223 Executor task launch worker for task 9.0 in stage 10.0 (TID 42) DEBUG BlockManager: Getting local block rdd_20_9 21/01/17 17:43:08.223 Executor task launch worker for task 9.0 in stage 10.0 (TID 42) DEBUG BlockManager: Level for block rdd_20_9 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:08.224 Executor task launch worker for task 9.0 in stage 10.0 (TID 42) INFO BlockManager: Found block rdd_20_9 locally 21/01/17 17:43:08.224 Executor task launch worker for task 9.0 in stage 10.0 (TID 42) DEBUG BlockManager: Getting local block rdd_19_9 21/01/17 17:43:08.224 Executor task launch worker for task 9.0 in stage 10.0 (TID 42) DEBUG BlockManager: Level for block rdd_19_9 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:08.224 Executor task launch worker for task 9.0 in stage 10.0 (TID 42) INFO BlockManager: Found block rdd_19_9 locally 21/01/17 17:43:08.224 Executor task launch worker for task 9.0 in stage 10.0 (TID 42) DEBUG TaskMemoryManager: Task 42 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@24e58a52 21/01/17 17:43:08.225 Executor task launch worker for task 9.0 in stage 10.0 (TID 42) DEBUG LocalDiskShuffleMapOutputWriter: Writing shuffle index file for mapId 42 with length 10 21/01/17 17:43:08.225 Executor task launch worker for task 9.0 in stage 10.0 (TID 42) DEBUG IndexShuffleBlockResolver: Shuffle index for mapId 42: [0,0,0,0,0,0,0,0,0,0] 21/01/17 17:43:08.225 dag-scheduler-event-loop DEBUG DAGScheduler: ShuffleMapTask finished on driver 21/01/17 17:43:08.226 dag-scheduler-event-loop DEBUG DAGScheduler: ShuffleMapTask finished on driver 21/01/17 17:43:08.229 Executor task launch worker for task 9.0 in stage 10.0 (TID 42) INFO Executor: Finished task 9.0 in stage 10.0 (TID 42). 1950 bytes result sent to driver 21/01/17 17:43:08.229 Executor task launch worker for task 9.0 in stage 10.0 (TID 42) DEBUG ExecutorMetricsPoller: stageTCMP: (10, 0) -> 2 21/01/17 17:43:08.221 Executor task launch worker for task 8.0 in stage 10.0 (TID 41) DEBUG BlockManager: Getting local block rdd_20_8 21/01/17 17:43:08.223 Executor task launch worker for task 7.0 in stage 10.0 (TID 40) DEBUG BlockManager: Getting local block rdd_19_7 21/01/17 17:43:08.230 Executor task launch worker for task 8.0 in stage 10.0 (TID 41) DEBUG BlockManager: Level for block rdd_20_8 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:08.230 Executor task launch worker for task 8.0 in stage 10.0 (TID 41) INFO BlockManager: Found block rdd_20_8 locally 21/01/17 17:43:08.230 Executor task launch worker for task 8.0 in stage 10.0 (TID 41) DEBUG BlockManager: Getting local block rdd_19_8 21/01/17 17:43:08.230 task-result-getter-1 INFO TaskSetManager: Finished task 9.0 in stage 10.0 (TID 42) in 29 ms on 192.168.1.225 (executor driver) (8/10) 21/01/17 17:43:08.230 Executor task launch worker for task 7.0 in stage 10.0 (TID 40) DEBUG BlockManager: Level for block rdd_19_7 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:08.230 Executor task launch worker for task 8.0 in stage 10.0 (TID 41) DEBUG BlockManager: Level for block rdd_19_8 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:08.230 Executor task launch worker for task 7.0 in stage 10.0 (TID 40) INFO BlockManager: Found block rdd_19_7 locally 21/01/17 17:43:08.230 Executor task launch worker for task 8.0 in stage 10.0 (TID 41) INFO BlockManager: Found block rdd_19_8 locally 21/01/17 17:43:08.231 dag-scheduler-event-loop DEBUG DAGScheduler: ShuffleMapTask finished on driver 21/01/17 17:43:08.231 Executor task launch worker for task 8.0 in stage 10.0 (TID 41) DEBUG TaskMemoryManager: Task 41 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@3c4c1c4e 21/01/17 17:43:08.231 Executor task launch worker for task 8.0 in stage 10.0 (TID 41) DEBUG LocalDiskShuffleMapOutputWriter: Writing shuffle index file for mapId 41 with length 10 21/01/17 17:43:08.231 Executor task launch worker for task 7.0 in stage 10.0 (TID 40) DEBUG TaskMemoryManager: Task 40 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@969fdc5 21/01/17 17:43:08.231 Executor task launch worker for task 7.0 in stage 10.0 (TID 40) DEBUG LocalDiskShuffleMapOutputWriter: Writing shuffle index file for mapId 40 with length 10 21/01/17 17:43:08.231 Executor task launch worker for task 8.0 in stage 10.0 (TID 41) DEBUG IndexShuffleBlockResolver: Shuffle index for mapId 41: [0,0,0,0,0,0,0,0,0,0] 21/01/17 17:43:08.231 Executor task launch worker for task 7.0 in stage 10.0 (TID 40) DEBUG IndexShuffleBlockResolver: Shuffle index for mapId 40: [0,0,0,0,0,0,0,0,0,0] 21/01/17 17:43:08.232 Executor task launch worker for task 8.0 in stage 10.0 (TID 41) INFO Executor: Finished task 8.0 in stage 10.0 (TID 41). 1950 bytes result sent to driver 21/01/17 17:43:08.232 Executor task launch worker for task 8.0 in stage 10.0 (TID 41) DEBUG ExecutorMetricsPoller: stageTCMP: (10, 0) -> 1 21/01/17 17:43:08.232 Executor task launch worker for task 7.0 in stage 10.0 (TID 40) INFO Executor: Finished task 7.0 in stage 10.0 (TID 40). 1950 bytes result sent to driver 21/01/17 17:43:08.232 Executor task launch worker for task 7.0 in stage 10.0 (TID 40) DEBUG ExecutorMetricsPoller: removing (10, 0) from stageTCMP 21/01/17 17:43:08.233 dag-scheduler-event-loop INFO DAGScheduler: Missing parents: List(ShuffleMapStage 2) 21/01/17 17:43:08.233 task-result-getter-3 INFO TaskSetManager: Finished task 7.0 in stage 10.0 (TID 40) in 37 ms on 192.168.1.225 (executor driver) (9/10) 21/01/17 17:43:08.233 task-result-getter-2 INFO TaskSetManager: Finished task 8.0 in stage 10.0 (TID 41) in 34 ms on 192.168.1.225 (executor driver) (10/10) 21/01/17 17:43:08.233 task-result-getter-2 INFO TaskSchedulerImpl: Removed TaskSet 10.0, whose tasks have all completed, from pool 21/01/17 17:43:08.234 dag-scheduler-event-loop DEBUG DAGScheduler: ShuffleMapTask finished on driver 21/01/17 17:43:08.235 dag-scheduler-event-loop DEBUG DAGScheduler: ShuffleMapTask finished on driver 21/01/17 17:43:08.235 dag-scheduler-event-loop INFO DAGScheduler: ShuffleMapStage 10 (flatMap at ALS.scala:1701) finished in 0.153 s 21/01/17 17:43:08.235 dag-scheduler-event-loop INFO DAGScheduler: looking for newly runnable stages 21/01/17 17:43:08.235 dag-scheduler-event-loop INFO DAGScheduler: running: Set() 21/01/17 17:43:08.235 dag-scheduler-event-loop INFO DAGScheduler: waiting: Set(ResultStage 12, ShuffleMapStage 11) 21/01/17 17:43:08.235 dag-scheduler-event-loop INFO DAGScheduler: failed: Set() 21/01/17 17:43:08.235 dag-scheduler-event-loop DEBUG MapOutputTrackerMaster: Increasing epoch to 4 21/01/17 17:43:08.236 dag-scheduler-event-loop DEBUG DAGScheduler: submitStage(ShuffleMapStage 11 (name=flatMap at ALS.scala:1701;jobs=3)) 21/01/17 17:43:08.237 dag-scheduler-event-loop DEBUG DAGScheduler: missing: List() 21/01/17 17:43:08.237 dag-scheduler-event-loop INFO DAGScheduler: Submitting ShuffleMapStage 11 (MapPartitionsRDD[40] at flatMap at ALS.scala:1701), which has no missing parents 21/01/17 17:43:08.237 dag-scheduler-event-loop DEBUG DAGScheduler: submitMissingTasks(ShuffleMapStage 11) 21/01/17 17:43:08.237 dag-scheduler-event-loop DEBUG DAGScheduler: submitStage(ResultStage 3 (name=count at CountVectorizer.scala:233;jobs=2)) 21/01/17 17:43:08.237 dag-scheduler-event-loop DEBUG DAGScheduler: missing: List(ShuffleMapStage 2) 21/01/17 17:43:08.238 dag-scheduler-event-loop DEBUG DAGScheduler: submitStage(ShuffleMapStage 2 (name=flatMap at CountVectorizer.scala:212;jobs=2)) 21/01/17 17:43:08.240 dag-scheduler-event-loop DEBUG DAGScheduler: missing: List() 21/01/17 17:43:08.240 dag-scheduler-event-loop INFO DAGScheduler: Submitting ShuffleMapStage 2 (MapPartitionsRDD[19] at flatMap at CountVectorizer.scala:212), which has no missing parents 21/01/17 17:43:08.240 dag-scheduler-event-loop DEBUG DAGScheduler: submitMissingTasks(ShuffleMapStage 2) 21/01/17 17:43:08.249 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_7 stored as values in memory (estimated size 38.5 KiB, free 365.9 MiB) 21/01/17 17:43:08.249 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_7 locally took 0 ms 21/01/17 17:43:08.249 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_7 without replication took 1 ms 21/01/17 17:43:08.250 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_7_piece0 stored as bytes in memory (estimated size 17.4 KiB, free 365.9 MiB) 21/01/17 17:43:08.250 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master broadcast_7_piece0 for BlockManagerId(driver, 192.168.1.225, 40985, None) 21/01/17 17:43:08.251 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added broadcast_7_piece0 in memory on 192.168.1.225:40985 (size: 17.4 KiB, free: 366.2 MiB) 21/01/17 17:43:08.251 dag-scheduler-event-loop DEBUG BlockManagerMaster: Updated info of block broadcast_7_piece0 21/01/17 17:43:08.251 dag-scheduler-event-loop DEBUG BlockManager: Told master about block broadcast_7_piece0 21/01/17 17:43:08.251 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_7_piece0 locally took 1 ms 21/01/17 17:43:08.251 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_7_piece0 without replication took 1 ms 21/01/17 17:43:08.251 dag-scheduler-event-loop INFO SparkContext: Created broadcast 7 from broadcast at DAGScheduler.scala:1383 21/01/17 17:43:08.252 dag-scheduler-event-loop INFO DAGScheduler: Submitting 10 missing tasks from ShuffleMapStage 11 (MapPartitionsRDD[40] at flatMap at ALS.scala:1701) (first 15 tasks are for partitions Vector(0, 1, 2, 3, 4, 5, 6, 7, 8, 9)) 21/01/17 17:43:08.252 dag-scheduler-event-loop INFO TaskSchedulerImpl: Adding task set 11.0 with 10 tasks resource profile 0 21/01/17 17:43:08.253 dag-scheduler-event-loop DEBUG TaskSetManager: Epoch for TaskSet 11.0: 4 21/01/17 17:43:08.253 dag-scheduler-event-loop DEBUG TaskSetManager: Adding pending tasks took 0 ms 21/01/17 17:43:08.253 dag-scheduler-event-loop DEBUG TaskSetManager: Valid locality levels for TaskSet 11.0: PROCESS_LOCAL, NODE_LOCAL, ANY 21/01/17 17:43:08.253 dispatcher-event-loop-3 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_11.0, runningTasks: 0 21/01/17 17:43:08.254 dispatcher-event-loop-3 INFO TaskSetManager: Starting task 0.0 in stage 11.0 (TID 43) (192.168.1.225, executor driver, partition 0, PROCESS_LOCAL, 7422 bytes) taskResourceAssignments Map() 21/01/17 17:43:08.254 dispatcher-event-loop-3 INFO TaskSetManager: Starting task 1.0 in stage 11.0 (TID 44) (192.168.1.225, executor driver, partition 1, PROCESS_LOCAL, 7422 bytes) taskResourceAssignments Map() 21/01/17 17:43:08.254 dispatcher-event-loop-3 INFO TaskSetManager: Starting task 2.0 in stage 11.0 (TID 45) (192.168.1.225, executor driver, partition 2, PROCESS_LOCAL, 7422 bytes) taskResourceAssignments Map() 21/01/17 17:43:08.255 dispatcher-event-loop-3 INFO TaskSetManager: Starting task 3.0 in stage 11.0 (TID 46) (192.168.1.225, executor driver, partition 3, PROCESS_LOCAL, 7422 bytes) taskResourceAssignments Map() 21/01/17 17:43:08.255 Executor task launch worker for task 0.0 in stage 11.0 (TID 43) INFO Executor: Running task 0.0 in stage 11.0 (TID 43) 21/01/17 17:43:08.255 Executor task launch worker for task 1.0 in stage 11.0 (TID 44) INFO Executor: Running task 1.0 in stage 11.0 (TID 44) 21/01/17 17:43:08.255 Executor task launch worker for task 2.0 in stage 11.0 (TID 45) INFO Executor: Running task 2.0 in stage 11.0 (TID 45) 21/01/17 17:43:08.255 Executor task launch worker for task 3.0 in stage 11.0 (TID 46) INFO Executor: Running task 3.0 in stage 11.0 (TID 46) 21/01/17 17:43:08.256 Executor task launch worker for task 0.0 in stage 11.0 (TID 43) DEBUG ExecutorMetricsPoller: stageTCMP: (11, 0) -> 1 21/01/17 17:43:08.256 Executor task launch worker for task 3.0 in stage 11.0 (TID 46) DEBUG ExecutorMetricsPoller: stageTCMP: (11, 0) -> 3 21/01/17 17:43:08.256 Executor task launch worker for task 0.0 in stage 11.0 (TID 43) DEBUG BlockManager: Getting local block broadcast_7 21/01/17 17:43:08.256 Executor task launch worker for task 1.0 in stage 11.0 (TID 44) DEBUG ExecutorMetricsPoller: stageTCMP: (11, 0) -> 4 21/01/17 17:43:08.256 Executor task launch worker for task 2.0 in stage 11.0 (TID 45) DEBUG ExecutorMetricsPoller: stageTCMP: (11, 0) -> 2 21/01/17 17:43:08.256 Executor task launch worker for task 0.0 in stage 11.0 (TID 43) DEBUG BlockManager: Level for block broadcast_7 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:08.259 Thread-3 DEBUG Analyzer$ResolveReferences: Resolving 'normalizedFeatures to normalizedFeatures#11 21/01/17 17:43:08.259 Thread-3 DEBUG Analyzer$ResolveReferences: Resolving 'prediction to prediction#1 21/01/17 17:43:08.259 Thread-3 DEBUG Analyzer$ResolveReferences: Resolving 'weightCol to weightCol#5 21/01/17 17:43:08.260 Thread-3 DEBUG Analyzer$ResolveReferences: Resolving 'weightCol to weightCol#5 21/01/17 17:43:08.260 Thread-3 DEBUG Analyzer$ResolveReferences: Resolving 'weightCol to weightCol#5 21/01/17 17:43:08.266 Executor task launch worker for task 0.0 in stage 11.0 (TID 43) DEBUG BlockManager: Getting local block rdd_25_0 21/01/17 17:43:08.266 Executor task launch worker for task 0.0 in stage 11.0 (TID 43) DEBUG BlockManager: Level for block rdd_25_0 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:08.266 Executor task launch worker for task 0.0 in stage 11.0 (TID 43) INFO BlockManager: Found block rdd_25_0 locally 21/01/17 17:43:08.266 Executor task launch worker for task 0.0 in stage 11.0 (TID 43) DEBUG BlockManager: Getting local block rdd_24_0 21/01/17 17:43:08.266 Executor task launch worker for task 0.0 in stage 11.0 (TID 43) DEBUG BlockManager: Level for block rdd_24_0 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:08.266 Executor task launch worker for task 0.0 in stage 11.0 (TID 43) INFO BlockManager: Found block rdd_24_0 locally 21/01/17 17:43:08.266 Executor task launch worker for task 0.0 in stage 11.0 (TID 43) DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 4 21/01/17 17:43:08.267 Executor task launch worker for task 0.0 in stage 11.0 (TID 43) DEBUG MapOutputTrackerMaster: Convert map statuses for shuffle 4, mappers 0-10, partitions 0-1 21/01/17 17:43:08.268 Executor task launch worker for task 0.0 in stage 11.0 (TID 43) DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 21/01/17 17:43:08.268 Executor task launch worker for task 0.0 in stage 11.0 (TID 43) INFO ShuffleBlockFetcherIterator: Getting 3 (684.0 B) non-empty blocks including 3 (684.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 21/01/17 17:43:08.268 Executor task launch worker for task 0.0 in stage 11.0 (TID 43) INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 21/01/17 17:43:08.268 Executor task launch worker for task 0.0 in stage 11.0 (TID 43) DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_4_33_0,0), (shuffle_4_34_0,1), (shuffle_4_35_0,2) 21/01/17 17:43:08.268 Executor task launch worker for task 0.0 in stage 11.0 (TID 43) DEBUG BlockManager: Getting local shuffle block shuffle_4_33_0 21/01/17 17:43:08.269 Executor task launch worker for task 0.0 in stage 11.0 (TID 43) DEBUG BlockManager: Getting local shuffle block shuffle_4_34_0 21/01/17 17:43:08.269 Executor task launch worker for task 0.0 in stage 11.0 (TID 43) DEBUG BlockManager: Getting local shuffle block shuffle_4_35_0 21/01/17 17:43:08.269 Executor task launch worker for task 0.0 in stage 11.0 (TID 43) DEBUG ShuffleBlockFetcherIterator: Got local blocks in 1 ms 21/01/17 17:43:08.269 Executor task launch worker for task 2.0 in stage 11.0 (TID 45) DEBUG BlockManager: Getting local block rdd_25_2 21/01/17 17:43:08.269 Executor task launch worker for task 2.0 in stage 11.0 (TID 45) DEBUG BlockManager: Level for block rdd_25_2 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:08.269 Executor task launch worker for task 2.0 in stage 11.0 (TID 45) INFO BlockManager: Found block rdd_25_2 locally 21/01/17 17:43:08.269 Executor task launch worker for task 2.0 in stage 11.0 (TID 45) DEBUG BlockManager: Getting local block rdd_24_2 21/01/17 17:43:08.269 Executor task launch worker for task 2.0 in stage 11.0 (TID 45) DEBUG BlockManager: Level for block rdd_24_2 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:08.269 Executor task launch worker for task 2.0 in stage 11.0 (TID 45) INFO BlockManager: Found block rdd_24_2 locally 21/01/17 17:43:08.270 Executor task launch worker for task 2.0 in stage 11.0 (TID 45) DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 4 21/01/17 17:43:08.270 Executor task launch worker for task 2.0 in stage 11.0 (TID 45) DEBUG MapOutputTrackerMaster: Convert map statuses for shuffle 4, mappers 0-10, partitions 2-3 21/01/17 17:43:08.270 Executor task launch worker for task 2.0 in stage 11.0 (TID 45) DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 21/01/17 17:43:08.271 Executor task launch worker for task 2.0 in stage 11.0 (TID 45) INFO ShuffleBlockFetcherIterator: Getting 3 (707.0 B) non-empty blocks including 3 (707.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 21/01/17 17:43:08.271 Executor task launch worker for task 2.0 in stage 11.0 (TID 45) INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 21/01/17 17:43:08.271 Executor task launch worker for task 2.0 in stage 11.0 (TID 45) DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_4_33_2,0), (shuffle_4_34_2,1), (shuffle_4_35_2,2) 21/01/17 17:43:08.271 Executor task launch worker for task 2.0 in stage 11.0 (TID 45) DEBUG BlockManager: Getting local shuffle block shuffle_4_33_2 21/01/17 17:43:08.271 Executor task launch worker for task 2.0 in stage 11.0 (TID 45) DEBUG BlockManager: Getting local shuffle block shuffle_4_34_2 21/01/17 17:43:08.271 Executor task launch worker for task 2.0 in stage 11.0 (TID 45) DEBUG BlockManager: Getting local shuffle block shuffle_4_35_2 21/01/17 17:43:08.271 Executor task launch worker for task 0.0 in stage 11.0 (TID 43) DEBUG TaskMemoryManager: Task 43 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@48de5b21 21/01/17 17:43:08.271 Executor task launch worker for task 2.0 in stage 11.0 (TID 45) DEBUG ShuffleBlockFetcherIterator: Got local blocks in 1 ms 21/01/17 17:43:08.272 Executor task launch worker for task 3.0 in stage 11.0 (TID 46) DEBUG BlockManager: Getting local block rdd_25_3 21/01/17 17:43:08.272 Executor task launch worker for task 3.0 in stage 11.0 (TID 46) DEBUG BlockManager: Level for block rdd_25_3 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:08.272 Executor task launch worker for task 3.0 in stage 11.0 (TID 46) INFO BlockManager: Found block rdd_25_3 locally 21/01/17 17:43:08.272 Executor task launch worker for task 3.0 in stage 11.0 (TID 46) DEBUG BlockManager: Getting local block rdd_24_3 21/01/17 17:43:08.272 Executor task launch worker for task 3.0 in stage 11.0 (TID 46) DEBUG BlockManager: Level for block rdd_24_3 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:08.272 Executor task launch worker for task 3.0 in stage 11.0 (TID 46) INFO BlockManager: Found block rdd_24_3 locally 21/01/17 17:43:08.272 Executor task launch worker for task 3.0 in stage 11.0 (TID 46) DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 4 21/01/17 17:43:08.272 Executor task launch worker for task 3.0 in stage 11.0 (TID 46) DEBUG MapOutputTrackerMaster: Convert map statuses for shuffle 4, mappers 0-10, partitions 3-4 21/01/17 17:43:08.273 Executor task launch worker for task 3.0 in stage 11.0 (TID 46) DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 21/01/17 17:43:08.273 Executor task launch worker for task 3.0 in stage 11.0 (TID 46) INFO ShuffleBlockFetcherIterator: Getting 3 (684.0 B) non-empty blocks including 3 (684.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 21/01/17 17:43:08.273 Executor task launch worker for task 3.0 in stage 11.0 (TID 46) INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 21/01/17 17:43:08.273 Executor task launch worker for task 3.0 in stage 11.0 (TID 46) DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_4_33_3,0), (shuffle_4_34_3,1), (shuffle_4_35_3,2) 21/01/17 17:43:08.273 Executor task launch worker for task 3.0 in stage 11.0 (TID 46) DEBUG BlockManager: Getting local shuffle block shuffle_4_33_3 21/01/17 17:43:08.274 Executor task launch worker for task 3.0 in stage 11.0 (TID 46) DEBUG BlockManager: Getting local shuffle block shuffle_4_34_3 21/01/17 17:43:08.274 Executor task launch worker for task 3.0 in stage 11.0 (TID 46) DEBUG BlockManager: Getting local shuffle block shuffle_4_35_3 21/01/17 17:43:08.274 Executor task launch worker for task 3.0 in stage 11.0 (TID 46) DEBUG ShuffleBlockFetcherIterator: Got local blocks in 1 ms 21/01/17 17:43:08.274 Executor task launch worker for task 2.0 in stage 11.0 (TID 45) DEBUG TaskMemoryManager: Task 45 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@61294639 21/01/17 17:43:08.276 Executor task launch worker for task 3.0 in stage 11.0 (TID 46) DEBUG TaskMemoryManager: Task 46 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@66f3e7d7 21/01/17 17:43:08.276 Executor task launch worker for task 3.0 in stage 11.0 (TID 46) DEBUG TaskMemoryManager: Task 46 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@5363ca69 21/01/17 17:43:08.277 Executor task launch worker for task 3.0 in stage 11.0 (TID 46) DEBUG TaskMemoryManager: Task 46 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@6bb99ea4 21/01/17 17:43:08.277 Executor task launch worker for task 3.0 in stage 11.0 (TID 46) DEBUG LocalDiskShuffleMapOutputWriter: Writing shuffle index file for mapId 46 with length 10 21/01/17 17:43:08.277 Executor task launch worker for task 3.0 in stage 11.0 (TID 46) DEBUG IndexShuffleBlockResolver: Shuffle index for mapId 46: [0,0,0,0,0,0,0,0,0,0] 21/01/17 17:43:08.278 Executor task launch worker for task 3.0 in stage 11.0 (TID 46) INFO Executor: Finished task 3.0 in stage 11.0 (TID 46). 2251 bytes result sent to driver 21/01/17 17:43:08.278 Executor task launch worker for task 3.0 in stage 11.0 (TID 46) DEBUG ExecutorMetricsPoller: stageTCMP: (11, 0) -> 3 21/01/17 17:43:08.279 dispatcher-event-loop-1 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_11.0, runningTasks: 3 21/01/17 17:43:08.279 dispatcher-event-loop-1 INFO TaskSetManager: Starting task 4.0 in stage 11.0 (TID 47) (192.168.1.225, executor driver, partition 4, PROCESS_LOCAL, 7422 bytes) taskResourceAssignments Map() 21/01/17 17:43:08.279 Executor task launch worker for task 1.0 in stage 11.0 (TID 44) DEBUG BlockManager: Getting local block rdd_25_1 21/01/17 17:43:08.279 Executor task launch worker for task 1.0 in stage 11.0 (TID 44) DEBUG BlockManager: Level for block rdd_25_1 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:08.279 Executor task launch worker for task 1.0 in stage 11.0 (TID 44) INFO BlockManager: Found block rdd_25_1 locally 21/01/17 17:43:08.279 Executor task launch worker for task 1.0 in stage 11.0 (TID 44) DEBUG BlockManager: Getting local block rdd_24_1 21/01/17 17:43:08.279 Executor task launch worker for task 1.0 in stage 11.0 (TID 44) DEBUG BlockManager: Level for block rdd_24_1 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:08.280 Executor task launch worker for task 1.0 in stage 11.0 (TID 44) INFO BlockManager: Found block rdd_24_1 locally 21/01/17 17:43:08.280 Executor task launch worker for task 1.0 in stage 11.0 (TID 44) DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 4 21/01/17 17:43:08.280 Executor task launch worker for task 1.0 in stage 11.0 (TID 44) DEBUG MapOutputTrackerMaster: Convert map statuses for shuffle 4, mappers 0-10, partitions 1-2 21/01/17 17:43:08.280 task-result-getter-0 INFO TaskSetManager: Finished task 3.0 in stage 11.0 (TID 46) in 25 ms on 192.168.1.225 (executor driver) (1/10) 21/01/17 17:43:08.280 Executor task launch worker for task 1.0 in stage 11.0 (TID 44) DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 21/01/17 17:43:08.281 Executor task launch worker for task 1.0 in stage 11.0 (TID 44) INFO ShuffleBlockFetcherIterator: Getting 3 (730.0 B) non-empty blocks including 3 (730.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 21/01/17 17:43:08.281 Executor task launch worker for task 1.0 in stage 11.0 (TID 44) INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 21/01/17 17:43:08.281 Executor task launch worker for task 1.0 in stage 11.0 (TID 44) DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_4_33_1,0), (shuffle_4_34_1,1), (shuffle_4_35_1,2) 21/01/17 17:43:08.281 Executor task launch worker for task 1.0 in stage 11.0 (TID 44) DEBUG BlockManager: Getting local shuffle block shuffle_4_33_1 21/01/17 17:43:08.281 dag-scheduler-event-loop DEBUG DAGScheduler: ShuffleMapTask finished on driver 21/01/17 17:43:08.281 Executor task launch worker for task 1.0 in stage 11.0 (TID 44) DEBUG BlockManager: Getting local shuffle block shuffle_4_34_1 21/01/17 17:43:08.281 Executor task launch worker for task 1.0 in stage 11.0 (TID 44) DEBUG BlockManager: Getting local shuffle block shuffle_4_35_1 21/01/17 17:43:08.281 Executor task launch worker for task 1.0 in stage 11.0 (TID 44) DEBUG ShuffleBlockFetcherIterator: Got local blocks in 1 ms 21/01/17 17:43:08.283 Executor task launch worker for task 1.0 in stage 11.0 (TID 44) DEBUG TaskMemoryManager: Task 44 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@6d1e1109 21/01/17 17:43:08.284 Executor task launch worker for task 4.0 in stage 11.0 (TID 47) INFO Executor: Running task 4.0 in stage 11.0 (TID 47) 21/01/17 17:43:08.285 Executor task launch worker for task 4.0 in stage 11.0 (TID 47) DEBUG ExecutorMetricsPoller: stageTCMP: (11, 0) -> 4 21/01/17 17:43:08.291 Executor task launch worker for task 4.0 in stage 11.0 (TID 47) DEBUG BlockManager: Getting local block rdd_25_4 21/01/17 17:43:08.291 Executor task launch worker for task 4.0 in stage 11.0 (TID 47) DEBUG BlockManager: Level for block rdd_25_4 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:08.291 Executor task launch worker for task 4.0 in stage 11.0 (TID 47) INFO BlockManager: Found block rdd_25_4 locally 21/01/17 17:43:08.291 Executor task launch worker for task 4.0 in stage 11.0 (TID 47) DEBUG BlockManager: Getting local block rdd_24_4 21/01/17 17:43:08.291 Executor task launch worker for task 4.0 in stage 11.0 (TID 47) DEBUG BlockManager: Level for block rdd_24_4 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:08.291 Executor task launch worker for task 4.0 in stage 11.0 (TID 47) INFO BlockManager: Found block rdd_24_4 locally 21/01/17 17:43:08.291 Executor task launch worker for task 4.0 in stage 11.0 (TID 47) DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 4 21/01/17 17:43:08.291 Executor task launch worker for task 4.0 in stage 11.0 (TID 47) DEBUG MapOutputTrackerMaster: Convert map statuses for shuffle 4, mappers 0-10, partitions 4-5 21/01/17 17:43:08.292 Executor task launch worker for task 4.0 in stage 11.0 (TID 47) DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 21/01/17 17:43:08.292 Executor task launch worker for task 4.0 in stage 11.0 (TID 47) INFO ShuffleBlockFetcherIterator: Getting 3 (684.0 B) non-empty blocks including 3 (684.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 21/01/17 17:43:08.292 Executor task launch worker for task 4.0 in stage 11.0 (TID 47) INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 21/01/17 17:43:08.292 Executor task launch worker for task 4.0 in stage 11.0 (TID 47) DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_4_33_4,0), (shuffle_4_34_4,1), (shuffle_4_35_4,2) 21/01/17 17:43:08.292 Executor task launch worker for task 4.0 in stage 11.0 (TID 47) DEBUG BlockManager: Getting local shuffle block shuffle_4_33_4 21/01/17 17:43:08.293 Executor task launch worker for task 4.0 in stage 11.0 (TID 47) DEBUG BlockManager: Getting local shuffle block shuffle_4_34_4 21/01/17 17:43:08.293 Executor task launch worker for task 4.0 in stage 11.0 (TID 47) DEBUG BlockManager: Getting local shuffle block shuffle_4_35_4 21/01/17 17:43:08.293 Executor task launch worker for task 4.0 in stage 11.0 (TID 47) DEBUG ShuffleBlockFetcherIterator: Got local blocks in 1 ms 21/01/17 17:43:08.295 Executor task launch worker for task 4.0 in stage 11.0 (TID 47) DEBUG TaskMemoryManager: Task 47 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@111978bf 21/01/17 17:43:08.295 Executor task launch worker for task 4.0 in stage 11.0 (TID 47) DEBUG TaskMemoryManager: Task 47 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@1f427b72 21/01/17 17:43:08.295 Executor task launch worker for task 4.0 in stage 11.0 (TID 47) DEBUG TaskMemoryManager: Task 47 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@7d141b8f 21/01/17 17:43:08.296 Executor task launch worker for task 4.0 in stage 11.0 (TID 47) DEBUG LocalDiskShuffleMapOutputWriter: Writing shuffle index file for mapId 47 with length 10 21/01/17 17:43:08.296 Executor task launch worker for task 4.0 in stage 11.0 (TID 47) DEBUG IndexShuffleBlockResolver: Shuffle index for mapId 47: [0,0,0,0,0,0,0,0,0,0] 21/01/17 17:43:08.298 Executor task launch worker for task 4.0 in stage 11.0 (TID 47) INFO Executor: Finished task 4.0 in stage 11.0 (TID 47). 2251 bytes result sent to driver 21/01/17 17:43:08.298 Executor task launch worker for task 4.0 in stage 11.0 (TID 47) DEBUG ExecutorMetricsPoller: stageTCMP: (11, 0) -> 3 21/01/17 17:43:08.298 dispatcher-event-loop-3 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_11.0, runningTasks: 3 21/01/17 17:43:08.299 dispatcher-event-loop-3 INFO TaskSetManager: Starting task 5.0 in stage 11.0 (TID 48) (192.168.1.225, executor driver, partition 5, PROCESS_LOCAL, 7422 bytes) taskResourceAssignments Map() 21/01/17 17:43:08.299 Executor task launch worker for task 5.0 in stage 11.0 (TID 48) INFO Executor: Running task 5.0 in stage 11.0 (TID 48) 21/01/17 17:43:08.299 task-result-getter-1 INFO TaskSetManager: Finished task 4.0 in stage 11.0 (TID 47) in 20 ms on 192.168.1.225 (executor driver) (2/10) 21/01/17 17:43:08.299 Executor task launch worker for task 5.0 in stage 11.0 (TID 48) DEBUG ExecutorMetricsPoller: stageTCMP: (11, 0) -> 4 21/01/17 17:43:08.310 dag-scheduler-event-loop DEBUG DAGScheduler: ShuffleMapTask finished on driver 21/01/17 17:43:08.311 Executor task launch worker for task 5.0 in stage 11.0 (TID 48) DEBUG BlockManager: Getting local block rdd_25_5 21/01/17 17:43:08.311 Executor task launch worker for task 5.0 in stage 11.0 (TID 48) DEBUG BlockManager: Level for block rdd_25_5 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:08.311 Executor task launch worker for task 5.0 in stage 11.0 (TID 48) INFO BlockManager: Found block rdd_25_5 locally 21/01/17 17:43:08.312 Executor task launch worker for task 5.0 in stage 11.0 (TID 48) DEBUG BlockManager: Getting local block rdd_24_5 21/01/17 17:43:08.312 Executor task launch worker for task 5.0 in stage 11.0 (TID 48) DEBUG BlockManager: Level for block rdd_24_5 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:08.312 Executor task launch worker for task 5.0 in stage 11.0 (TID 48) INFO BlockManager: Found block rdd_24_5 locally 21/01/17 17:43:08.312 Executor task launch worker for task 5.0 in stage 11.0 (TID 48) DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 4 21/01/17 17:43:08.312 Executor task launch worker for task 5.0 in stage 11.0 (TID 48) DEBUG MapOutputTrackerMaster: Convert map statuses for shuffle 4, mappers 0-10, partitions 5-6 21/01/17 17:43:08.312 Executor task launch worker for task 5.0 in stage 11.0 (TID 48) DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 21/01/17 17:43:08.313 Executor task launch worker for task 5.0 in stage 11.0 (TID 48) INFO ShuffleBlockFetcherIterator: Getting 3 (684.0 B) non-empty blocks including 3 (684.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 21/01/17 17:43:08.313 Executor task launch worker for task 5.0 in stage 11.0 (TID 48) INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 21/01/17 17:43:08.313 Executor task launch worker for task 5.0 in stage 11.0 (TID 48) DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_4_33_5,0), (shuffle_4_34_5,1), (shuffle_4_35_5,2) 21/01/17 17:43:08.313 Executor task launch worker for task 5.0 in stage 11.0 (TID 48) DEBUG BlockManager: Getting local shuffle block shuffle_4_33_5 21/01/17 17:43:08.313 Executor task launch worker for task 5.0 in stage 11.0 (TID 48) DEBUG BlockManager: Getting local shuffle block shuffle_4_34_5 21/01/17 17:43:08.313 Executor task launch worker for task 5.0 in stage 11.0 (TID 48) DEBUG BlockManager: Getting local shuffle block shuffle_4_35_5 21/01/17 17:43:08.313 Executor task launch worker for task 5.0 in stage 11.0 (TID 48) DEBUG ShuffleBlockFetcherIterator: Got local blocks in 1 ms 21/01/17 17:43:08.315 Executor task launch worker for task 5.0 in stage 11.0 (TID 48) DEBUG TaskMemoryManager: Task 48 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@62e629a3 21/01/17 17:43:08.316 Executor task launch worker for task 5.0 in stage 11.0 (TID 48) DEBUG TaskMemoryManager: Task 48 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@1bd1b5a3 21/01/17 17:43:08.316 Executor task launch worker for task 5.0 in stage 11.0 (TID 48) DEBUG TaskMemoryManager: Task 48 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@6d5695f6 21/01/17 17:43:08.316 Executor task launch worker for task 5.0 in stage 11.0 (TID 48) DEBUG LocalDiskShuffleMapOutputWriter: Writing shuffle index file for mapId 48 with length 10 21/01/17 17:43:08.316 Executor task launch worker for task 5.0 in stage 11.0 (TID 48) DEBUG IndexShuffleBlockResolver: Shuffle index for mapId 48: [0,0,0,0,0,0,0,0,0,0] 21/01/17 17:43:08.317 Executor task launch worker for task 5.0 in stage 11.0 (TID 48) INFO Executor: Finished task 5.0 in stage 11.0 (TID 48). 2251 bytes result sent to driver 21/01/17 17:43:08.318 dispatcher-event-loop-1 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_11.0, runningTasks: 3 21/01/17 17:43:08.318 dispatcher-event-loop-1 INFO TaskSetManager: Starting task 6.0 in stage 11.0 (TID 49) (192.168.1.225, executor driver, partition 6, PROCESS_LOCAL, 7422 bytes) taskResourceAssignments Map() 21/01/17 17:43:08.318 Executor task launch worker for task 5.0 in stage 11.0 (TID 48) DEBUG ExecutorMetricsPoller: stageTCMP: (11, 0) -> 3 21/01/17 17:43:08.318 Executor task launch worker for task 6.0 in stage 11.0 (TID 49) INFO Executor: Running task 6.0 in stage 11.0 (TID 49) 21/01/17 17:43:08.318 task-result-getter-3 INFO TaskSetManager: Finished task 5.0 in stage 11.0 (TID 48) in 20 ms on 192.168.1.225 (executor driver) (3/10) 21/01/17 17:43:08.319 Executor task launch worker for task 6.0 in stage 11.0 (TID 49) DEBUG ExecutorMetricsPoller: stageTCMP: (11, 0) -> 4 21/01/17 17:43:08.320 dag-scheduler-event-loop DEBUG DAGScheduler: ShuffleMapTask finished on driver 21/01/17 17:43:08.325 Executor task launch worker for task 6.0 in stage 11.0 (TID 49) DEBUG BlockManager: Getting local block rdd_25_6 21/01/17 17:43:08.325 Executor task launch worker for task 6.0 in stage 11.0 (TID 49) DEBUG BlockManager: Level for block rdd_25_6 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:08.325 Executor task launch worker for task 6.0 in stage 11.0 (TID 49) INFO BlockManager: Found block rdd_25_6 locally 21/01/17 17:43:08.325 Executor task launch worker for task 6.0 in stage 11.0 (TID 49) DEBUG BlockManager: Getting local block rdd_24_6 21/01/17 17:43:08.325 Executor task launch worker for task 6.0 in stage 11.0 (TID 49) DEBUG BlockManager: Level for block rdd_24_6 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:08.325 Executor task launch worker for task 6.0 in stage 11.0 (TID 49) INFO BlockManager: Found block rdd_24_6 locally 21/01/17 17:43:08.325 Executor task launch worker for task 6.0 in stage 11.0 (TID 49) DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 4 21/01/17 17:43:08.325 Executor task launch worker for task 6.0 in stage 11.0 (TID 49) DEBUG MapOutputTrackerMaster: Convert map statuses for shuffle 4, mappers 0-10, partitions 6-7 21/01/17 17:43:08.326 Executor task launch worker for task 6.0 in stage 11.0 (TID 49) DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 21/01/17 17:43:08.327 Executor task launch worker for task 6.0 in stage 11.0 (TID 49) INFO ShuffleBlockFetcherIterator: Getting 3 (684.0 B) non-empty blocks including 3 (684.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 21/01/17 17:43:08.327 Executor task launch worker for task 6.0 in stage 11.0 (TID 49) INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 21/01/17 17:43:08.327 Executor task launch worker for task 6.0 in stage 11.0 (TID 49) DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_4_33_6,0), (shuffle_4_34_6,1), (shuffle_4_35_6,2) 21/01/17 17:43:08.327 Executor task launch worker for task 6.0 in stage 11.0 (TID 49) DEBUG BlockManager: Getting local shuffle block shuffle_4_33_6 21/01/17 17:43:08.327 Executor task launch worker for task 6.0 in stage 11.0 (TID 49) DEBUG BlockManager: Getting local shuffle block shuffle_4_34_6 21/01/17 17:43:08.327 Executor task launch worker for task 6.0 in stage 11.0 (TID 49) DEBUG BlockManager: Getting local shuffle block shuffle_4_35_6 21/01/17 17:43:08.327 Executor task launch worker for task 6.0 in stage 11.0 (TID 49) DEBUG ShuffleBlockFetcherIterator: Got local blocks in 1 ms 21/01/17 17:43:08.328 Executor task launch worker for task 0.0 in stage 11.0 (TID 43) WARN LAPACK: Failed to load implementation from: com.github.fommil.netlib.NativeSystemLAPACK 21/01/17 17:43:08.329 Executor task launch worker for task 0.0 in stage 11.0 (TID 43) WARN LAPACK: Failed to load implementation from: com.github.fommil.netlib.NativeRefLAPACK 21/01/17 17:43:08.338 Executor task launch worker for task 6.0 in stage 11.0 (TID 49) DEBUG TaskMemoryManager: Task 49 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@6efe1f30 21/01/17 17:43:08.339 Executor task launch worker for task 6.0 in stage 11.0 (TID 49) DEBUG TaskMemoryManager: Task 49 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@d147172 21/01/17 17:43:08.339 Executor task launch worker for task 6.0 in stage 11.0 (TID 49) DEBUG TaskMemoryManager: Task 49 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@3c0e30d9 21/01/17 17:43:08.339 Executor task launch worker for task 6.0 in stage 11.0 (TID 49) DEBUG LocalDiskShuffleMapOutputWriter: Writing shuffle index file for mapId 49 with length 10 21/01/17 17:43:08.339 Executor task launch worker for task 6.0 in stage 11.0 (TID 49) DEBUG IndexShuffleBlockResolver: Shuffle index for mapId 49: [0,0,0,0,0,0,0,0,0,0] 21/01/17 17:43:08.340 Executor task launch worker for task 6.0 in stage 11.0 (TID 49) INFO Executor: Finished task 6.0 in stage 11.0 (TID 49). 2251 bytes result sent to driver 21/01/17 17:43:08.341 Executor task launch worker for task 6.0 in stage 11.0 (TID 49) DEBUG ExecutorMetricsPoller: stageTCMP: (11, 0) -> 3 21/01/17 17:43:08.342 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_4 stored as values in memory (estimated size 22.1 KiB, free 366.2 MiB) 21/01/17 17:43:08.343 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_4 locally took 6 ms 21/01/17 17:43:08.343 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_4 without replication took 6 ms 21/01/17 17:43:08.344 dispatcher-event-loop-3 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_11.0, runningTasks: 3 21/01/17 17:43:08.344 dispatcher-event-loop-3 INFO TaskSetManager: Starting task 7.0 in stage 11.0 (TID 50) (192.168.1.225, executor driver, partition 7, PROCESS_LOCAL, 7422 bytes) taskResourceAssignments Map() 21/01/17 17:43:08.345 task-result-getter-2 INFO TaskSetManager: Finished task 6.0 in stage 11.0 (TID 49) in 27 ms on 192.168.1.225 (executor driver) (4/10) 21/01/17 17:43:08.345 Executor task launch worker for task 7.0 in stage 11.0 (TID 50) INFO Executor: Running task 7.0 in stage 11.0 (TID 50) 21/01/17 17:43:08.346 Executor task launch worker for task 7.0 in stage 11.0 (TID 50) DEBUG ExecutorMetricsPoller: stageTCMP: (11, 0) -> 4 21/01/17 17:43:08.346 dag-scheduler-event-loop DEBUG DAGScheduler: ShuffleMapTask finished on driver 21/01/17 17:43:08.347 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_4_piece0 stored as bytes in memory (estimated size 10.8 KiB, free 366.2 MiB) 21/01/17 17:43:08.347 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master broadcast_4_piece0 for BlockManagerId(driver, 192.168.1.225, 46295, None) 21/01/17 17:43:08.348 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added broadcast_4_piece0 in memory on 192.168.1.225:46295 (size: 10.8 KiB, free: 366.3 MiB) 21/01/17 17:43:08.352 Executor task launch worker for task 7.0 in stage 11.0 (TID 50) DEBUG BlockManager: Getting local block rdd_25_7 21/01/17 17:43:08.352 Executor task launch worker for task 7.0 in stage 11.0 (TID 50) DEBUG BlockManager: Level for block rdd_25_7 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:08.352 Executor task launch worker for task 7.0 in stage 11.0 (TID 50) INFO BlockManager: Found block rdd_25_7 locally 21/01/17 17:43:08.352 Executor task launch worker for task 7.0 in stage 11.0 (TID 50) DEBUG BlockManager: Getting local block rdd_24_7 21/01/17 17:43:08.352 Executor task launch worker for task 7.0 in stage 11.0 (TID 50) DEBUG BlockManager: Level for block rdd_24_7 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:08.352 Executor task launch worker for task 7.0 in stage 11.0 (TID 50) INFO BlockManager: Found block rdd_24_7 locally 21/01/17 17:43:08.352 Executor task launch worker for task 7.0 in stage 11.0 (TID 50) DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 4 21/01/17 17:43:08.352 Executor task launch worker for task 7.0 in stage 11.0 (TID 50) DEBUG MapOutputTrackerMaster: Convert map statuses for shuffle 4, mappers 0-10, partitions 7-8 21/01/17 17:43:08.353 Executor task launch worker for task 7.0 in stage 11.0 (TID 50) DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 21/01/17 17:43:08.353 dag-scheduler-event-loop DEBUG BlockManagerMaster: Updated info of block broadcast_4_piece0 21/01/17 17:43:08.353 dag-scheduler-event-loop DEBUG BlockManager: Told master about block broadcast_4_piece0 21/01/17 17:43:08.353 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_4_piece0 locally took 9 ms 21/01/17 17:43:08.353 Executor task launch worker for task 7.0 in stage 11.0 (TID 50) INFO ShuffleBlockFetcherIterator: Getting 3 (684.0 B) non-empty blocks including 3 (684.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 21/01/17 17:43:08.353 Executor task launch worker for task 7.0 in stage 11.0 (TID 50) INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 21/01/17 17:43:08.354 Executor task launch worker for task 7.0 in stage 11.0 (TID 50) DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_4_33_7,0), (shuffle_4_34_7,1), (shuffle_4_35_7,2) 21/01/17 17:43:08.354 Executor task launch worker for task 7.0 in stage 11.0 (TID 50) DEBUG BlockManager: Getting local shuffle block shuffle_4_33_7 21/01/17 17:43:08.354 Executor task launch worker for task 7.0 in stage 11.0 (TID 50) DEBUG BlockManager: Getting local shuffle block shuffle_4_34_7 21/01/17 17:43:08.354 Executor task launch worker for task 7.0 in stage 11.0 (TID 50) DEBUG BlockManager: Getting local shuffle block shuffle_4_35_7 21/01/17 17:43:08.354 Executor task launch worker for task 7.0 in stage 11.0 (TID 50) DEBUG ShuffleBlockFetcherIterator: Got local blocks in 1 ms 21/01/17 17:43:08.354 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_4_piece0 without replication took 9 ms 21/01/17 17:43:08.355 dag-scheduler-event-loop INFO SparkContext: Created broadcast 4 from broadcast at DAGScheduler.scala:1383 21/01/17 17:43:08.356 Executor task launch worker for task 7.0 in stage 11.0 (TID 50) DEBUG TaskMemoryManager: Task 50 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@1b056e85 21/01/17 17:43:08.356 Executor task launch worker for task 7.0 in stage 11.0 (TID 50) DEBUG TaskMemoryManager: Task 50 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@b0a2464 21/01/17 17:43:08.357 Executor task launch worker for task 7.0 in stage 11.0 (TID 50) DEBUG TaskMemoryManager: Task 50 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@74bb2530 21/01/17 17:43:08.357 Executor task launch worker for task 7.0 in stage 11.0 (TID 50) DEBUG LocalDiskShuffleMapOutputWriter: Writing shuffle index file for mapId 50 with length 10 21/01/17 17:43:08.361 dag-scheduler-event-loop INFO DAGScheduler: Submitting 4 missing tasks from ShuffleMapStage 2 (MapPartitionsRDD[19] at flatMap at CountVectorizer.scala:212) (first 15 tasks are for partitions Vector(0, 1, 2, 3)) 21/01/17 17:43:08.361 dag-scheduler-event-loop INFO TaskSchedulerImpl: Adding task set 2.0 with 4 tasks resource profile 0 21/01/17 17:43:08.365 Executor task launch worker for task 7.0 in stage 11.0 (TID 50) DEBUG IndexShuffleBlockResolver: Shuffle index for mapId 50: [0,0,0,0,0,0,0,0,0,0] 21/01/17 17:43:08.366 Executor task launch worker for task 7.0 in stage 11.0 (TID 50) INFO Executor: Finished task 7.0 in stage 11.0 (TID 50). 2251 bytes result sent to driver 21/01/17 17:43:08.366 Executor task launch worker for task 7.0 in stage 11.0 (TID 50) DEBUG ExecutorMetricsPoller: stageTCMP: (11, 0) -> 3 21/01/17 17:43:08.367 dispatcher-event-loop-1 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_11.0, runningTasks: 3 21/01/17 17:43:08.367 dispatcher-event-loop-1 INFO TaskSetManager: Starting task 8.0 in stage 11.0 (TID 51) (192.168.1.225, executor driver, partition 8, PROCESS_LOCAL, 7422 bytes) taskResourceAssignments Map() 21/01/17 17:43:08.368 task-result-getter-0 INFO TaskSetManager: Finished task 7.0 in stage 11.0 (TID 50) in 24 ms on 192.168.1.225 (executor driver) (5/10) 21/01/17 17:43:08.369 dag-scheduler-event-loop DEBUG DAGScheduler: ShuffleMapTask finished on driver 21/01/17 17:43:08.362 dag-scheduler-event-loop DEBUG TaskSetManager: Epoch for TaskSet 2.0: 0 21/01/17 17:43:08.376 dag-scheduler-event-loop DEBUG TaskSetManager: Adding pending tasks took 0 ms 21/01/17 17:43:08.377 dag-scheduler-event-loop DEBUG TaskSetManager: Valid locality levels for TaskSet 2.0: NO_PREF, ANY 21/01/17 17:43:08.377 dispatcher-event-loop-1 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_2.0, runningTasks: 0 21/01/17 17:43:08.379 dispatcher-event-loop-1 INFO TaskSetManager: Starting task 0.0 in stage 2.0 (TID 8) (192.168.1.225, executor driver, partition 0, PROCESS_LOCAL, 7421 bytes) taskResourceAssignments Map() 21/01/17 17:43:08.379 dispatcher-event-loop-1 INFO TaskSetManager: Starting task 1.0 in stage 2.0 (TID 9) (192.168.1.225, executor driver, partition 1, PROCESS_LOCAL, 7385 bytes) taskResourceAssignments Map() 21/01/17 17:43:08.380 dispatcher-event-loop-1 INFO TaskSetManager: Starting task 2.0 in stage 2.0 (TID 10) (192.168.1.225, executor driver, partition 2, PROCESS_LOCAL, 7400 bytes) taskResourceAssignments Map() 21/01/17 17:43:08.381 Executor task launch worker for task 8.0 in stage 11.0 (TID 51) INFO Executor: Running task 8.0 in stage 11.0 (TID 51) 21/01/17 17:43:08.381 Executor task launch worker for task 8.0 in stage 11.0 (TID 51) DEBUG ExecutorMetricsPoller: stageTCMP: (11, 0) -> 4 21/01/17 17:43:08.383 dispatcher-event-loop-1 INFO TaskSetManager: Starting task 3.0 in stage 2.0 (TID 11) (192.168.1.225, executor driver, partition 3, PROCESS_LOCAL, 7382 bytes) taskResourceAssignments Map() 21/01/17 17:43:08.383 Executor task launch worker for task 0.0 in stage 2.0 (TID 8) INFO Executor: Running task 0.0 in stage 2.0 (TID 8) 21/01/17 17:43:08.383 Executor task launch worker for task 2.0 in stage 2.0 (TID 10) INFO Executor: Running task 2.0 in stage 2.0 (TID 10) 21/01/17 17:43:08.383 Executor task launch worker for task 1.0 in stage 2.0 (TID 9) INFO Executor: Running task 1.0 in stage 2.0 (TID 9) 21/01/17 17:43:08.384 Executor task launch worker for task 3.0 in stage 2.0 (TID 11) INFO Executor: Running task 3.0 in stage 2.0 (TID 11) 21/01/17 17:43:08.384 Executor task launch worker for task 0.0 in stage 2.0 (TID 8) DEBUG ExecutorMetricsPoller: stageTCMP: (2, 0) -> 1 21/01/17 17:43:08.385 Executor task launch worker for task 0.0 in stage 2.0 (TID 8) DEBUG BlockManager: Getting local block broadcast_4 21/01/17 17:43:08.385 Executor task launch worker for task 0.0 in stage 2.0 (TID 8) DEBUG BlockManager: Level for block broadcast_4 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:08.387 Executor task launch worker for task 2.0 in stage 2.0 (TID 10) DEBUG ExecutorMetricsPoller: stageTCMP: (2, 0) -> 2 21/01/17 17:43:08.393 Executor task launch worker for task 8.0 in stage 11.0 (TID 51) DEBUG BlockManager: Getting local block rdd_25_8 21/01/17 17:43:08.393 Executor task launch worker for task 8.0 in stage 11.0 (TID 51) DEBUG BlockManager: Level for block rdd_25_8 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:08.393 Executor task launch worker for task 8.0 in stage 11.0 (TID 51) INFO BlockManager: Found block rdd_25_8 locally 21/01/17 17:43:08.394 Executor task launch worker for task 3.0 in stage 2.0 (TID 11) DEBUG ExecutorMetricsPoller: stageTCMP: (2, 0) -> 3 21/01/17 17:43:08.401 Executor task launch worker for task 1.0 in stage 2.0 (TID 9) DEBUG ExecutorMetricsPoller: stageTCMP: (2, 0) -> 4 21/01/17 17:43:08.404 Executor task launch worker for task 8.0 in stage 11.0 (TID 51) DEBUG BlockManager: Getting local block rdd_24_8 21/01/17 17:43:08.404 Executor task launch worker for task 8.0 in stage 11.0 (TID 51) DEBUG BlockManager: Level for block rdd_24_8 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:08.404 Executor task launch worker for task 8.0 in stage 11.0 (TID 51) INFO BlockManager: Found block rdd_24_8 locally 21/01/17 17:43:08.405 Executor task launch worker for task 8.0 in stage 11.0 (TID 51) DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 4 21/01/17 17:43:08.405 Executor task launch worker for task 8.0 in stage 11.0 (TID 51) DEBUG MapOutputTrackerMaster: Convert map statuses for shuffle 4, mappers 0-10, partitions 8-9 21/01/17 17:43:08.416 Executor task launch worker for task 8.0 in stage 11.0 (TID 51) DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 21/01/17 17:43:08.417 Executor task launch worker for task 8.0 in stage 11.0 (TID 51) INFO ShuffleBlockFetcherIterator: Getting 3 (684.0 B) non-empty blocks including 3 (684.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 21/01/17 17:43:08.417 Executor task launch worker for task 8.0 in stage 11.0 (TID 51) INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 21/01/17 17:43:08.417 Executor task launch worker for task 8.0 in stage 11.0 (TID 51) DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_4_33_8,0), (shuffle_4_34_8,1), (shuffle_4_35_8,2) 21/01/17 17:43:08.417 Executor task launch worker for task 8.0 in stage 11.0 (TID 51) DEBUG BlockManager: Getting local shuffle block shuffle_4_33_8 21/01/17 17:43:08.417 Executor task launch worker for task 8.0 in stage 11.0 (TID 51) DEBUG BlockManager: Getting local shuffle block shuffle_4_34_8 21/01/17 17:43:08.417 Executor task launch worker for task 8.0 in stage 11.0 (TID 51) DEBUG BlockManager: Getting local shuffle block shuffle_4_35_8 21/01/17 17:43:08.417 Executor task launch worker for task 8.0 in stage 11.0 (TID 51) DEBUG ShuffleBlockFetcherIterator: Got local blocks in 1 ms 21/01/17 17:43:08.422 Executor task launch worker for task 8.0 in stage 11.0 (TID 51) DEBUG TaskMemoryManager: Task 51 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@4e7e0c23 21/01/17 17:43:08.422 Executor task launch worker for task 8.0 in stage 11.0 (TID 51) DEBUG TaskMemoryManager: Task 51 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@7e55e19d 21/01/17 17:43:08.424 Executor task launch worker for task 8.0 in stage 11.0 (TID 51) DEBUG TaskMemoryManager: Task 51 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@e9745f0 21/01/17 17:43:08.424 Executor task launch worker for task 8.0 in stage 11.0 (TID 51) DEBUG LocalDiskShuffleMapOutputWriter: Writing shuffle index file for mapId 51 with length 10 21/01/17 17:43:08.426 Executor task launch worker for task 8.0 in stage 11.0 (TID 51) DEBUG IndexShuffleBlockResolver: Shuffle index for mapId 51: [0,0,0,0,0,0,0,0,0,0] 21/01/17 17:43:08.429 Executor task launch worker for task 8.0 in stage 11.0 (TID 51) INFO Executor: Finished task 8.0 in stage 11.0 (TID 51). 2251 bytes result sent to driver 21/01/17 17:43:08.429 Executor task launch worker for task 8.0 in stage 11.0 (TID 51) DEBUG ExecutorMetricsPoller: stageTCMP: (11, 0) -> 3 21/01/17 17:43:08.429 dispatcher-event-loop-3 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_11.0, runningTasks: 3 21/01/17 17:43:08.430 dispatcher-event-loop-3 INFO TaskSetManager: Starting task 9.0 in stage 11.0 (TID 52) (192.168.1.225, executor driver, partition 9, PROCESS_LOCAL, 7422 bytes) taskResourceAssignments Map() 21/01/17 17:43:08.431 task-result-getter-1 INFO TaskSetManager: Finished task 8.0 in stage 11.0 (TID 51) in 64 ms on 192.168.1.225 (executor driver) (6/10) 21/01/17 17:43:08.432 dag-scheduler-event-loop DEBUG DAGScheduler: ShuffleMapTask finished on driver 21/01/17 17:43:08.433 Executor task launch worker for task 9.0 in stage 11.0 (TID 52) INFO Executor: Running task 9.0 in stage 11.0 (TID 52) 21/01/17 17:43:08.434 Executor task launch worker for task 9.0 in stage 11.0 (TID 52) DEBUG ExecutorMetricsPoller: stageTCMP: (11, 0) -> 4 21/01/17 17:43:08.453 Executor task launch worker for task 9.0 in stage 11.0 (TID 52) DEBUG BlockManager: Getting local block rdd_25_9 21/01/17 17:43:08.454 Executor task launch worker for task 9.0 in stage 11.0 (TID 52) DEBUG BlockManager: Level for block rdd_25_9 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:08.454 Executor task launch worker for task 9.0 in stage 11.0 (TID 52) INFO BlockManager: Found block rdd_25_9 locally 21/01/17 17:43:08.454 Executor task launch worker for task 9.0 in stage 11.0 (TID 52) DEBUG BlockManager: Getting local block rdd_24_9 21/01/17 17:43:08.454 Executor task launch worker for task 9.0 in stage 11.0 (TID 52) DEBUG BlockManager: Level for block rdd_24_9 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:08.454 Executor task launch worker for task 9.0 in stage 11.0 (TID 52) INFO BlockManager: Found block rdd_24_9 locally 21/01/17 17:43:08.454 Executor task launch worker for task 9.0 in stage 11.0 (TID 52) DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 4 21/01/17 17:43:08.454 Executor task launch worker for task 9.0 in stage 11.0 (TID 52) DEBUG MapOutputTrackerMaster: Convert map statuses for shuffle 4, mappers 0-10, partitions 9-10 21/01/17 17:43:08.454 Executor task launch worker for task 9.0 in stage 11.0 (TID 52) DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 21/01/17 17:43:08.455 Executor task launch worker for task 2.0 in stage 11.0 (TID 45) DEBUG TaskMemoryManager: Task 45 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@2c348d87 21/01/17 17:43:08.455 Executor task launch worker for task 9.0 in stage 11.0 (TID 52) INFO ShuffleBlockFetcherIterator: Getting 3 (684.0 B) non-empty blocks including 3 (684.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 21/01/17 17:43:08.456 Executor task launch worker for task 9.0 in stage 11.0 (TID 52) INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 1 ms 21/01/17 17:43:08.456 Executor task launch worker for task 9.0 in stage 11.0 (TID 52) DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_4_33_9,0), (shuffle_4_34_9,1), (shuffle_4_35_9,2) 21/01/17 17:43:08.456 Executor task launch worker for task 9.0 in stage 11.0 (TID 52) DEBUG BlockManager: Getting local shuffle block shuffle_4_33_9 21/01/17 17:43:08.457 Executor task launch worker for task 9.0 in stage 11.0 (TID 52) DEBUG BlockManager: Getting local shuffle block shuffle_4_34_9 21/01/17 17:43:08.457 Executor task launch worker for task 9.0 in stage 11.0 (TID 52) DEBUG BlockManager: Getting local shuffle block shuffle_4_35_9 21/01/17 17:43:08.457 Executor task launch worker for task 2.0 in stage 11.0 (TID 45) DEBUG TaskMemoryManager: Task 45 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@680e0d5f 21/01/17 17:43:08.459 Executor task launch worker for task 1.0 in stage 11.0 (TID 44) DEBUG TaskMemoryManager: Task 44 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@3fa7740a 21/01/17 17:43:08.460 Executor task launch worker for task 2.0 in stage 11.0 (TID 45) DEBUG LocalDiskShuffleMapOutputWriter: Writing shuffle index file for mapId 45 with length 10 21/01/17 17:43:08.460 Executor task launch worker for task 2.0 in stage 11.0 (TID 45) DEBUG IndexShuffleBlockResolver: Shuffle index for mapId 45: [214,231,228,215,215,214,215,215,215,215] 21/01/17 17:43:08.461 Executor task launch worker for task 2.0 in stage 11.0 (TID 45) INFO Executor: Finished task 2.0 in stage 11.0 (TID 45). 2423 bytes result sent to driver 21/01/17 17:43:08.461 Executor task launch worker for task 2.0 in stage 11.0 (TID 45) DEBUG ExecutorMetricsPoller: stageTCMP: (11, 0) -> 3 21/01/17 17:43:08.462 dispatcher-event-loop-1 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_11.0, runningTasks: 3 21/01/17 17:43:08.462 dispatcher-event-loop-1 DEBUG TaskSetManager: No tasks for locality level PROCESS_LOCAL, so moving to locality level NODE_LOCAL 21/01/17 17:43:08.462 dispatcher-event-loop-1 DEBUG TaskSetManager: No tasks for locality level NODE_LOCAL, so moving to locality level ANY 21/01/17 17:43:08.462 Executor task launch worker for task 1.0 in stage 11.0 (TID 44) DEBUG TaskMemoryManager: Task 44 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@7110dae3 21/01/17 17:43:08.462 task-result-getter-3 INFO TaskSetManager: Finished task 2.0 in stage 11.0 (TID 45) in 208 ms on 192.168.1.225 (executor driver) (7/10) 21/01/17 17:43:08.463 Executor task launch worker for task 9.0 in stage 11.0 (TID 52) DEBUG ShuffleBlockFetcherIterator: Got local blocks in 8 ms 21/01/17 17:43:08.463 dag-scheduler-event-loop DEBUG DAGScheduler: ShuffleMapTask finished on driver 21/01/17 17:43:08.464 Executor task launch worker for task 0.0 in stage 11.0 (TID 43) DEBUG TaskMemoryManager: Task 43 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@c3f931c 21/01/17 17:43:08.466 Executor task launch worker for task 9.0 in stage 11.0 (TID 52) DEBUG TaskMemoryManager: Task 52 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@2f574cfc 21/01/17 17:43:08.466 Executor task launch worker for task 9.0 in stage 11.0 (TID 52) DEBUG TaskMemoryManager: Task 52 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@818bd6 21/01/17 17:43:08.466 Executor task launch worker for task 9.0 in stage 11.0 (TID 52) DEBUG TaskMemoryManager: Task 52 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@39dd23bf 21/01/17 17:43:08.466 Executor task launch worker for task 9.0 in stage 11.0 (TID 52) DEBUG LocalDiskShuffleMapOutputWriter: Writing shuffle index file for mapId 52 with length 10 21/01/17 17:43:08.467 Executor task launch worker for task 0.0 in stage 11.0 (TID 43) DEBUG TaskMemoryManager: Task 43 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@6b8aca44 21/01/17 17:43:08.467 Executor task launch worker for task 9.0 in stage 11.0 (TID 52) DEBUG IndexShuffleBlockResolver: Shuffle index for mapId 52: [0,0,0,0,0,0,0,0,0,0] 21/01/17 17:43:08.468 Executor task launch worker for task 9.0 in stage 11.0 (TID 52) INFO Executor: Finished task 9.0 in stage 11.0 (TID 52). 2251 bytes result sent to driver 21/01/17 17:43:08.468 Executor task launch worker for task 0.0 in stage 11.0 (TID 43) DEBUG LocalDiskShuffleMapOutputWriter: Writing shuffle index file for mapId 43 with length 10 21/01/17 17:43:08.468 Executor task launch worker for task 0.0 in stage 11.0 (TID 43) DEBUG IndexShuffleBlockResolver: Shuffle index for mapId 43: [228,213,213,213,213,212,213,213,213,213] 21/01/17 17:43:08.469 task-result-getter-2 INFO TaskSetManager: Finished task 9.0 in stage 11.0 (TID 52) in 38 ms on 192.168.1.225 (executor driver) (8/10) 21/01/17 17:43:08.469 Executor task launch worker for task 9.0 in stage 11.0 (TID 52) DEBUG ExecutorMetricsPoller: stageTCMP: (11, 0) -> 2 21/01/17 17:43:08.469 dag-scheduler-event-loop DEBUG DAGScheduler: ShuffleMapTask finished on driver 21/01/17 17:43:08.469 Executor task launch worker for task 1.0 in stage 11.0 (TID 44) DEBUG LocalDiskShuffleMapOutputWriter: Writing shuffle index file for mapId 44 with length 10 21/01/17 17:43:08.470 Executor task launch worker for task 1.0 in stage 11.0 (TID 44) DEBUG IndexShuffleBlockResolver: Shuffle index for mapId 44: [230,226,231,215,215,214,215,215,215,215] 21/01/17 17:43:08.471 Executor task launch worker for task 1.0 in stage 11.0 (TID 44) INFO Executor: Finished task 1.0 in stage 11.0 (TID 44). 2423 bytes result sent to driver 21/01/17 17:43:08.471 Executor task launch worker for task 1.0 in stage 11.0 (TID 44) DEBUG ExecutorMetricsPoller: stageTCMP: (11, 0) -> 1 21/01/17 17:43:08.471 task-result-getter-0 INFO TaskSetManager: Finished task 1.0 in stage 11.0 (TID 44) in 217 ms on 192.168.1.225 (executor driver) (9/10) 21/01/17 17:43:08.472 Executor task launch worker for task 0.0 in stage 11.0 (TID 43) INFO Executor: Finished task 0.0 in stage 11.0 (TID 43). 2423 bytes result sent to driver 21/01/17 17:43:08.473 task-result-getter-1 INFO TaskSetManager: Finished task 0.0 in stage 11.0 (TID 43) in 219 ms on 192.168.1.225 (executor driver) (10/10) 21/01/17 17:43:08.473 task-result-getter-1 INFO TaskSchedulerImpl: Removed TaskSet 11.0, whose tasks have all completed, from pool 21/01/17 17:43:08.473 dag-scheduler-event-loop DEBUG DAGScheduler: ShuffleMapTask finished on driver 21/01/17 17:43:08.474 Executor task launch worker for task 0.0 in stage 11.0 (TID 43) DEBUG ExecutorMetricsPoller: removing (11, 0) from stageTCMP 21/01/17 17:43:08.474 dag-scheduler-event-loop DEBUG DAGScheduler: ShuffleMapTask finished on driver 21/01/17 17:43:08.475 dag-scheduler-event-loop INFO DAGScheduler: ShuffleMapStage 11 (flatMap at ALS.scala:1701) finished in 0.236 s 21/01/17 17:43:08.475 dag-scheduler-event-loop INFO DAGScheduler: looking for newly runnable stages 21/01/17 17:43:08.475 dag-scheduler-event-loop INFO DAGScheduler: running: Set() 21/01/17 17:43:08.475 dag-scheduler-event-loop INFO DAGScheduler: waiting: Set(ResultStage 12) 21/01/17 17:43:08.475 dag-scheduler-event-loop INFO DAGScheduler: failed: Set() 21/01/17 17:43:08.475 dag-scheduler-event-loop DEBUG MapOutputTrackerMaster: Increasing epoch to 5 21/01/17 17:43:08.475 dag-scheduler-event-loop DEBUG DAGScheduler: submitStage(ResultStage 12 (name=count at ALS.scala:1075;jobs=3)) 21/01/17 17:43:08.477 dag-scheduler-event-loop DEBUG DAGScheduler: missing: List() 21/01/17 17:43:08.477 dag-scheduler-event-loop INFO DAGScheduler: Submitting ResultStage 12 (userFactors MapPartitionsRDD[50] at mapPartitions at ALS.scala:1055), which has no missing parents 21/01/17 17:43:08.477 dag-scheduler-event-loop DEBUG DAGScheduler: submitMissingTasks(ResultStage 12) 21/01/17 17:43:08.483 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_8 stored as values in memory (estimated size 40.5 KiB, free 365.9 MiB) 21/01/17 17:43:08.483 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_8 locally took 0 ms 21/01/17 17:43:08.484 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_8 without replication took 0 ms 21/01/17 17:43:08.485 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_8_piece0 stored as bytes in memory (estimated size 17.9 KiB, free 365.9 MiB) 21/01/17 17:43:08.485 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master broadcast_8_piece0 for BlockManagerId(driver, 192.168.1.225, 40985, None) 21/01/17 17:43:08.485 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added broadcast_8_piece0 in memory on 192.168.1.225:40985 (size: 17.9 KiB, free: 366.2 MiB) 21/01/17 17:43:08.497 dag-scheduler-event-loop DEBUG BlockManagerMaster: Updated info of block broadcast_8_piece0 21/01/17 17:43:08.497 dag-scheduler-event-loop DEBUG BlockManager: Told master about block broadcast_8_piece0 21/01/17 17:43:08.497 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_8_piece0 locally took 12 ms 21/01/17 17:43:08.497 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_8_piece0 without replication took 12 ms 21/01/17 17:43:08.498 dag-scheduler-event-loop INFO SparkContext: Created broadcast 8 from broadcast at DAGScheduler.scala:1383 21/01/17 17:43:08.498 dag-scheduler-event-loop INFO DAGScheduler: Submitting 10 missing tasks from ResultStage 12 (userFactors MapPartitionsRDD[50] at mapPartitions at ALS.scala:1055) (first 15 tasks are for partitions Vector(0, 1, 2, 3, 4, 5, 6, 7, 8, 9)) 21/01/17 17:43:08.498 dag-scheduler-event-loop INFO TaskSchedulerImpl: Adding task set 12.0 with 10 tasks resource profile 0 21/01/17 17:43:08.499 dag-scheduler-event-loop DEBUG TaskSetManager: Epoch for TaskSet 12.0: 5 21/01/17 17:43:08.499 dag-scheduler-event-loop DEBUG TaskSetManager: Adding pending tasks took 0 ms 21/01/17 17:43:08.499 dag-scheduler-event-loop DEBUG TaskSetManager: Valid locality levels for TaskSet 12.0: PROCESS_LOCAL, NODE_LOCAL, ANY 21/01/17 17:43:08.500 dispatcher-event-loop-1 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_12.0, runningTasks: 0 21/01/17 17:43:08.500 dispatcher-event-loop-1 INFO TaskSetManager: Starting task 0.0 in stage 12.0 (TID 53) (192.168.1.225, executor driver, partition 0, PROCESS_LOCAL, 7433 bytes) taskResourceAssignments Map() 21/01/17 17:43:08.501 dispatcher-event-loop-1 INFO TaskSetManager: Starting task 1.0 in stage 12.0 (TID 54) (192.168.1.225, executor driver, partition 1, PROCESS_LOCAL, 7433 bytes) taskResourceAssignments Map() 21/01/17 17:43:08.501 dispatcher-event-loop-1 INFO TaskSetManager: Starting task 2.0 in stage 12.0 (TID 55) (192.168.1.225, executor driver, partition 2, PROCESS_LOCAL, 7433 bytes) taskResourceAssignments Map() 21/01/17 17:43:08.501 dispatcher-event-loop-1 INFO TaskSetManager: Starting task 3.0 in stage 12.0 (TID 56) (192.168.1.225, executor driver, partition 3, PROCESS_LOCAL, 7433 bytes) taskResourceAssignments Map() 21/01/17 17:43:08.502 Executor task launch worker for task 0.0 in stage 12.0 (TID 53) INFO Executor: Running task 0.0 in stage 12.0 (TID 53) 21/01/17 17:43:08.502 Executor task launch worker for task 1.0 in stage 12.0 (TID 54) INFO Executor: Running task 1.0 in stage 12.0 (TID 54) 21/01/17 17:43:08.502 Executor task launch worker for task 2.0 in stage 12.0 (TID 55) INFO Executor: Running task 2.0 in stage 12.0 (TID 55) 21/01/17 17:43:08.503 Executor task launch worker for task 3.0 in stage 12.0 (TID 56) INFO Executor: Running task 3.0 in stage 12.0 (TID 56) 21/01/17 17:43:08.504 Executor task launch worker for task 2.0 in stage 12.0 (TID 55) DEBUG ExecutorMetricsPoller: stageTCMP: (12, 0) -> 1 21/01/17 17:43:08.507 Executor task launch worker for task 1.0 in stage 12.0 (TID 54) DEBUG ExecutorMetricsPoller: stageTCMP: (12, 0) -> 2 21/01/17 17:43:08.509 Executor task launch worker for task 2.0 in stage 12.0 (TID 55) DEBUG BlockManager: Getting local block broadcast_8 21/01/17 17:43:08.509 Executor task launch worker for task 2.0 in stage 12.0 (TID 55) DEBUG BlockManager: Level for block broadcast_8 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:08.510 Executor task launch worker for task 0.0 in stage 12.0 (TID 53) DEBUG ExecutorMetricsPoller: stageTCMP: (12, 0) -> 3 21/01/17 17:43:08.511 Executor task launch worker for task 3.0 in stage 12.0 (TID 56) DEBUG ExecutorMetricsPoller: stageTCMP: (12, 0) -> 4 21/01/17 17:43:08.516 Executor task launch worker for task 2.0 in stage 12.0 (TID 55) DEBUG BlockManager: Getting local block rdd_50_2 21/01/17 17:43:08.516 Executor task launch worker for task 2.0 in stage 12.0 (TID 55) DEBUG BlockManager: Block rdd_50_2 was not found 21/01/17 17:43:08.516 Executor task launch worker for task 2.0 in stage 12.0 (TID 55) DEBUG BlockManager: Getting remote block rdd_50_2 21/01/17 17:43:08.517 Executor task launch worker for task 1.0 in stage 12.0 (TID 54) DEBUG BlockManager: Getting local block rdd_50_1 21/01/17 17:43:08.517 Executor task launch worker for task 1.0 in stage 12.0 (TID 54) DEBUG BlockManager: Block rdd_50_1 was not found 21/01/17 17:43:08.517 Executor task launch worker for task 1.0 in stage 12.0 (TID 54) DEBUG BlockManager: Getting remote block rdd_50_1 21/01/17 17:43:08.519 Executor task launch worker for task 0.0 in stage 12.0 (TID 53) DEBUG BlockManager: Getting local block rdd_50_0 21/01/17 17:43:08.519 Executor task launch worker for task 0.0 in stage 12.0 (TID 53) DEBUG BlockManager: Block rdd_50_0 was not found 21/01/17 17:43:08.519 Executor task launch worker for task 0.0 in stage 12.0 (TID 53) DEBUG BlockManager: Getting remote block rdd_50_0 21/01/17 17:43:08.519 Executor task launch worker for task 0.0 in stage 12.0 (TID 53) DEBUG BlockManager: Block rdd_50_0 is unknown by block manager master 21/01/17 17:43:08.519 Executor task launch worker for task 1.0 in stage 12.0 (TID 54) DEBUG BlockManager: Block rdd_50_1 is unknown by block manager master 21/01/17 17:43:08.519 Executor task launch worker for task 0.0 in stage 12.0 (TID 53) DEBUG BlockManager: Getting local block rdd_19_0 21/01/17 17:43:08.519 Executor task launch worker for task 3.0 in stage 12.0 (TID 56) DEBUG BlockManager: Getting local block rdd_50_3 21/01/17 17:43:08.519 Executor task launch worker for task 1.0 in stage 12.0 (TID 54) DEBUG BlockManager: Getting local block rdd_19_1 21/01/17 17:43:08.519 Executor task launch worker for task 0.0 in stage 12.0 (TID 53) DEBUG BlockManager: Level for block rdd_19_0 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:08.520 Executor task launch worker for task 1.0 in stage 12.0 (TID 54) DEBUG BlockManager: Level for block rdd_19_1 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:08.520 Executor task launch worker for task 1.0 in stage 12.0 (TID 54) INFO BlockManager: Found block rdd_19_1 locally 21/01/17 17:43:08.520 Executor task launch worker for task 2.0 in stage 12.0 (TID 55) DEBUG BlockManager: Block rdd_50_2 is unknown by block manager master 21/01/17 17:43:08.520 Executor task launch worker for task 1.0 in stage 12.0 (TID 54) DEBUG BlockManager: Getting local block rdd_19_1 21/01/17 17:43:08.520 Executor task launch worker for task 2.0 in stage 12.0 (TID 55) DEBUG BlockManager: Getting local block rdd_19_2 21/01/17 17:43:08.520 Executor task launch worker for task 2.0 in stage 12.0 (TID 55) DEBUG BlockManager: Level for block rdd_19_2 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:08.520 Executor task launch worker for task 2.0 in stage 12.0 (TID 55) INFO BlockManager: Found block rdd_19_2 locally 21/01/17 17:43:08.520 Executor task launch worker for task 2.0 in stage 12.0 (TID 55) DEBUG BlockManager: Getting local block rdd_19_2 21/01/17 17:43:08.520 Executor task launch worker for task 2.0 in stage 12.0 (TID 55) DEBUG BlockManager: Level for block rdd_19_2 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:08.520 Executor task launch worker for task 2.0 in stage 12.0 (TID 55) INFO BlockManager: Found block rdd_19_2 locally 21/01/17 17:43:08.520 Executor task launch worker for task 2.0 in stage 12.0 (TID 55) DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 3 21/01/17 17:43:08.520 Executor task launch worker for task 2.0 in stage 12.0 (TID 55) DEBUG MapOutputTrackerMaster: Convert map statuses for shuffle 3, mappers 0-10, partitions 2-3 21/01/17 17:43:08.521 Executor task launch worker for task 2.0 in stage 12.0 (TID 55) DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 21/01/17 17:43:08.521 Executor task launch worker for task 2.0 in stage 12.0 (TID 55) INFO ShuffleBlockFetcherIterator: Getting 3 (707.0 B) non-empty blocks including 3 (707.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 21/01/17 17:43:08.521 Executor task launch worker for task 2.0 in stage 12.0 (TID 55) INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 21/01/17 17:43:08.521 Executor task launch worker for task 2.0 in stage 12.0 (TID 55) DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_3_43_2,0), (shuffle_3_44_2,1), (shuffle_3_45_2,2) 21/01/17 17:43:08.521 Executor task launch worker for task 2.0 in stage 12.0 (TID 55) DEBUG BlockManager: Getting local shuffle block shuffle_3_43_2 21/01/17 17:43:08.519 Executor task launch worker for task 3.0 in stage 12.0 (TID 56) DEBUG BlockManager: Block rdd_50_3 was not found 21/01/17 17:43:08.520 Executor task launch worker for task 1.0 in stage 12.0 (TID 54) DEBUG BlockManager: Level for block rdd_19_1 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:08.521 Executor task launch worker for task 2.0 in stage 12.0 (TID 55) DEBUG BlockManager: Getting local shuffle block shuffle_3_44_2 21/01/17 17:43:08.522 Executor task launch worker for task 2.0 in stage 12.0 (TID 55) DEBUG BlockManager: Getting local shuffle block shuffle_3_45_2 21/01/17 17:43:08.522 Executor task launch worker for task 2.0 in stage 12.0 (TID 55) DEBUG ShuffleBlockFetcherIterator: Got local blocks in 1 ms 21/01/17 17:43:08.524 Executor task launch worker for task 2.0 in stage 12.0 (TID 55) DEBUG TaskMemoryManager: Task 55 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@6dad5deb 21/01/17 17:43:08.524 Executor task launch worker for task 2.0 in stage 12.0 (TID 55) DEBUG TaskMemoryManager: Task 55 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@42b800e 21/01/17 17:43:08.526 Executor task launch worker for task 2.0 in stage 12.0 (TID 55) DEBUG TaskMemoryManager: Task 55 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@11f624db 21/01/17 17:43:08.526 Executor task launch worker for task 2.0 in stage 12.0 (TID 55) INFO MemoryStore: Block rdd_50_2 stored as values in memory (estimated size 88.0 B, free 365.9 MiB) 21/01/17 17:43:08.520 Executor task launch worker for task 0.0 in stage 12.0 (TID 53) INFO BlockManager: Found block rdd_19_0 locally 21/01/17 17:43:08.521 Executor task launch worker for task 1.0 in stage 12.0 (TID 54) INFO BlockManager: Found block rdd_19_1 locally 21/01/17 17:43:08.527 Executor task launch worker for task 0.0 in stage 12.0 (TID 53) DEBUG BlockManager: Getting local block rdd_19_0 21/01/17 17:43:08.521 Executor task launch worker for task 3.0 in stage 12.0 (TID 56) DEBUG BlockManager: Getting remote block rdd_50_3 21/01/17 17:43:08.527 Executor task launch worker for task 1.0 in stage 12.0 (TID 54) DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 3 21/01/17 17:43:08.527 Executor task launch worker for task 0.0 in stage 12.0 (TID 53) DEBUG BlockManager: Level for block rdd_19_0 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:08.527 Executor task launch worker for task 1.0 in stage 12.0 (TID 54) DEBUG MapOutputTrackerMaster: Convert map statuses for shuffle 3, mappers 0-10, partitions 1-2 21/01/17 17:43:08.527 Executor task launch worker for task 0.0 in stage 12.0 (TID 53) INFO BlockManager: Found block rdd_19_0 locally 21/01/17 17:43:08.527 Executor task launch worker for task 0.0 in stage 12.0 (TID 53) DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 3 21/01/17 17:43:08.527 Executor task launch worker for task 0.0 in stage 12.0 (TID 53) DEBUG MapOutputTrackerMaster: Convert map statuses for shuffle 3, mappers 0-10, partitions 0-1 21/01/17 17:43:08.528 Executor task launch worker for task 1.0 in stage 12.0 (TID 54) DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 21/01/17 17:43:08.528 Executor task launch worker for task 1.0 in stage 12.0 (TID 54) INFO ShuffleBlockFetcherIterator: Getting 3 (707.0 B) non-empty blocks including 3 (707.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 21/01/17 17:43:08.528 Executor task launch worker for task 1.0 in stage 12.0 (TID 54) INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 21/01/17 17:43:08.528 Executor task launch worker for task 1.0 in stage 12.0 (TID 54) DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_3_43_1,0), (shuffle_3_44_1,1), (shuffle_3_45_1,2) 21/01/17 17:43:08.528 Executor task launch worker for task 1.0 in stage 12.0 (TID 54) DEBUG BlockManager: Getting local shuffle block shuffle_3_43_1 21/01/17 17:43:08.529 Executor task launch worker for task 1.0 in stage 12.0 (TID 54) DEBUG BlockManager: Getting local shuffle block shuffle_3_44_1 21/01/17 17:43:08.529 Executor task launch worker for task 1.0 in stage 12.0 (TID 54) DEBUG BlockManager: Getting local shuffle block shuffle_3_45_1 21/01/17 17:43:08.529 Executor task launch worker for task 1.0 in stage 12.0 (TID 54) DEBUG ShuffleBlockFetcherIterator: Got local blocks in 1 ms 21/01/17 17:43:08.529 Executor task launch worker for task 0.0 in stage 12.0 (TID 53) DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 21/01/17 17:43:08.529 Executor task launch worker for task 0.0 in stage 12.0 (TID 53) INFO ShuffleBlockFetcherIterator: Getting 3 (707.0 B) non-empty blocks including 3 (707.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 21/01/17 17:43:08.529 Executor task launch worker for task 0.0 in stage 12.0 (TID 53) INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 21/01/17 17:43:08.530 Executor task launch worker for task 0.0 in stage 12.0 (TID 53) DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_3_43_0,0), (shuffle_3_44_0,1), (shuffle_3_45_0,2) 21/01/17 17:43:08.530 Executor task launch worker for task 0.0 in stage 12.0 (TID 53) DEBUG BlockManager: Getting local shuffle block shuffle_3_43_0 21/01/17 17:43:08.531 Executor task launch worker for task 0.0 in stage 12.0 (TID 53) DEBUG BlockManager: Getting local shuffle block shuffle_3_44_0 21/01/17 17:43:08.531 Executor task launch worker for task 1.0 in stage 12.0 (TID 54) DEBUG TaskMemoryManager: Task 54 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@5a86ff84 21/01/17 17:43:08.531 Executor task launch worker for task 0.0 in stage 12.0 (TID 53) DEBUG BlockManager: Getting local shuffle block shuffle_3_45_0 21/01/17 17:43:08.531 Executor task launch worker for task 0.0 in stage 12.0 (TID 53) DEBUG ShuffleBlockFetcherIterator: Got local blocks in 2 ms 21/01/17 17:43:08.531 Executor task launch worker for task 1.0 in stage 12.0 (TID 54) DEBUG TaskMemoryManager: Task 54 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@3daf424b 21/01/17 17:43:08.532 Executor task launch worker for task 1.0 in stage 12.0 (TID 54) DEBUG TaskMemoryManager: Task 54 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@181cb05a 21/01/17 17:43:08.532 Executor task launch worker for task 1.0 in stage 12.0 (TID 54) INFO MemoryStore: Block rdd_50_1 stored as values in memory (estimated size 88.0 B, free 365.9 MiB) 21/01/17 17:43:08.533 Executor task launch worker for task 0.0 in stage 12.0 (TID 53) DEBUG TaskMemoryManager: Task 53 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@6362093c 21/01/17 17:43:08.534 Executor task launch worker for task 0.0 in stage 12.0 (TID 53) DEBUG TaskMemoryManager: Task 53 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@67797206 21/01/17 17:43:08.534 Executor task launch worker for task 0.0 in stage 12.0 (TID 53) DEBUG TaskMemoryManager: Task 53 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@13f47207 21/01/17 17:43:08.534 Executor task launch worker for task 0.0 in stage 12.0 (TID 53) INFO MemoryStore: Block rdd_50_0 stored as values in memory (estimated size 88.0 B, free 365.9 MiB) 21/01/17 17:43:08.535 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_50_2 for BlockManagerId(driver, 192.168.1.225, 40985, None) 21/01/17 17:43:08.535 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_50_2 in memory on 192.168.1.225:40985 (size: 88.0 B, free: 366.2 MiB) 21/01/17 17:43:08.537 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_50_0 for BlockManagerId(driver, 192.168.1.225, 40985, None) 21/01/17 17:43:08.537 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_50_0 in memory on 192.168.1.225:40985 (size: 88.0 B, free: 366.2 MiB) 21/01/17 17:43:08.537 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_50_1 for BlockManagerId(driver, 192.168.1.225, 40985, None) 21/01/17 17:43:08.537 Executor task launch worker for task 0.0 in stage 12.0 (TID 53) DEBUG BlockManagerMaster: Updated info of block rdd_50_0 21/01/17 17:43:08.538 Executor task launch worker for task 0.0 in stage 12.0 (TID 53) DEBUG BlockManager: Told master about block rdd_50_0 21/01/17 17:43:08.538 Executor task launch worker for task 0.0 in stage 12.0 (TID 53) DEBUG BlockManager: Put block rdd_50_0 locally took 18 ms 21/01/17 17:43:08.538 Executor task launch worker for task 0.0 in stage 12.0 (TID 53) DEBUG BlockManager: Putting block rdd_50_0 without replication took 18 ms 21/01/17 17:43:08.538 Executor task launch worker for task 0.0 in stage 12.0 (TID 53) DEBUG BlockManager: Getting local block rdd_50_0 21/01/17 17:43:08.538 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_50_1 in memory on 192.168.1.225:40985 (size: 88.0 B, free: 366.2 MiB) 21/01/17 17:43:08.538 Executor task launch worker for task 0.0 in stage 12.0 (TID 53) DEBUG BlockManager: Level for block rdd_50_0 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:08.538 Executor task launch worker for task 1.0 in stage 12.0 (TID 54) DEBUG BlockManagerMaster: Updated info of block rdd_50_1 21/01/17 17:43:08.538 Executor task launch worker for task 1.0 in stage 12.0 (TID 54) DEBUG BlockManager: Told master about block rdd_50_1 21/01/17 17:43:08.538 Executor task launch worker for task 1.0 in stage 12.0 (TID 54) DEBUG BlockManager: Put block rdd_50_1 locally took 18 ms 21/01/17 17:43:08.538 Executor task launch worker for task 1.0 in stage 12.0 (TID 54) DEBUG BlockManager: Putting block rdd_50_1 without replication took 18 ms 21/01/17 17:43:08.538 Executor task launch worker for task 1.0 in stage 12.0 (TID 54) DEBUG BlockManager: Getting local block rdd_50_1 21/01/17 17:43:08.538 Executor task launch worker for task 3.0 in stage 12.0 (TID 56) DEBUG BlockManager: Block rdd_50_3 is unknown by block manager master 21/01/17 17:43:08.538 Executor task launch worker for task 1.0 in stage 12.0 (TID 54) DEBUG BlockManager: Level for block rdd_50_1 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:08.538 Executor task launch worker for task 3.0 in stage 12.0 (TID 56) DEBUG BlockManager: Getting local block rdd_19_3 21/01/17 17:43:08.538 Executor task launch worker for task 3.0 in stage 12.0 (TID 56) DEBUG BlockManager: Level for block rdd_19_3 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:08.538 Executor task launch worker for task 3.0 in stage 12.0 (TID 56) INFO BlockManager: Found block rdd_19_3 locally 21/01/17 17:43:08.538 Executor task launch worker for task 3.0 in stage 12.0 (TID 56) DEBUG BlockManager: Getting local block rdd_19_3 21/01/17 17:43:08.539 Executor task launch worker for task 3.0 in stage 12.0 (TID 56) DEBUG BlockManager: Level for block rdd_19_3 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:08.539 Executor task launch worker for task 3.0 in stage 12.0 (TID 56) INFO BlockManager: Found block rdd_19_3 locally 21/01/17 17:43:08.539 Executor task launch worker for task 3.0 in stage 12.0 (TID 56) DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 3 21/01/17 17:43:08.539 Executor task launch worker for task 3.0 in stage 12.0 (TID 56) DEBUG MapOutputTrackerMaster: Convert map statuses for shuffle 3, mappers 0-10, partitions 3-4 21/01/17 17:43:08.539 Executor task launch worker for task 1.0 in stage 12.0 (TID 54) INFO Executor: Finished task 1.0 in stage 12.0 (TID 54). 2256 bytes result sent to driver 21/01/17 17:43:08.539 Executor task launch worker for task 3.0 in stage 12.0 (TID 56) DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 21/01/17 17:43:08.539 Executor task launch worker for task 1.0 in stage 12.0 (TID 54) DEBUG ExecutorMetricsPoller: stageTCMP: (12, 0) -> 3 21/01/17 17:43:08.539 Executor task launch worker for task 3.0 in stage 12.0 (TID 56) INFO ShuffleBlockFetcherIterator: Getting 3 (684.0 B) non-empty blocks including 3 (684.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 21/01/17 17:43:08.539 Executor task launch worker for task 3.0 in stage 12.0 (TID 56) INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 21/01/17 17:43:08.539 Executor task launch worker for task 3.0 in stage 12.0 (TID 56) DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_3_43_3,0), (shuffle_3_44_3,1), (shuffle_3_45_3,2) 21/01/17 17:43:08.540 Executor task launch worker for task 3.0 in stage 12.0 (TID 56) DEBUG BlockManager: Getting local shuffle block shuffle_3_43_3 21/01/17 17:43:08.540 Executor task launch worker for task 3.0 in stage 12.0 (TID 56) DEBUG BlockManager: Getting local shuffle block shuffle_3_44_3 21/01/17 17:43:08.540 Executor task launch worker for task 0.0 in stage 12.0 (TID 53) INFO Executor: Finished task 0.0 in stage 12.0 (TID 53). 2256 bytes result sent to driver 21/01/17 17:43:08.540 Executor task launch worker for task 0.0 in stage 12.0 (TID 53) DEBUG ExecutorMetricsPoller: stageTCMP: (12, 0) -> 2 21/01/17 17:43:08.540 Executor task launch worker for task 3.0 in stage 12.0 (TID 56) DEBUG BlockManager: Getting local shuffle block shuffle_3_45_3 21/01/17 17:43:08.540 Executor task launch worker for task 3.0 in stage 12.0 (TID 56) DEBUG ShuffleBlockFetcherIterator: Got local blocks in 0 ms 21/01/17 17:43:08.540 Executor task launch worker for task 2.0 in stage 12.0 (TID 55) DEBUG BlockManagerMaster: Updated info of block rdd_50_2 21/01/17 17:43:08.540 Executor task launch worker for task 2.0 in stage 12.0 (TID 55) DEBUG BlockManager: Told master about block rdd_50_2 21/01/17 17:43:08.540 Executor task launch worker for task 2.0 in stage 12.0 (TID 55) DEBUG BlockManager: Put block rdd_50_2 locally took 20 ms 21/01/17 17:43:08.540 Executor task launch worker for task 2.0 in stage 12.0 (TID 55) DEBUG BlockManager: Putting block rdd_50_2 without replication took 20 ms 21/01/17 17:43:08.540 Executor task launch worker for task 2.0 in stage 12.0 (TID 55) DEBUG BlockManager: Getting local block rdd_50_2 21/01/17 17:43:08.540 Executor task launch worker for task 2.0 in stage 12.0 (TID 55) DEBUG BlockManager: Level for block rdd_50_2 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:08.541 dispatcher-event-loop-2 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_12.0, runningTasks: 3 21/01/17 17:43:08.541 Executor task launch worker for task 2.0 in stage 12.0 (TID 55) INFO Executor: Finished task 2.0 in stage 12.0 (TID 55). 2256 bytes result sent to driver 21/01/17 17:43:08.541 Executor task launch worker for task 2.0 in stage 12.0 (TID 55) DEBUG ExecutorMetricsPoller: stageTCMP: (12, 0) -> 1 21/01/17 17:43:08.542 Executor task launch worker for task 3.0 in stage 12.0 (TID 56) DEBUG TaskMemoryManager: Task 56 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@44b74075 21/01/17 17:43:08.542 Executor task launch worker for task 3.0 in stage 12.0 (TID 56) DEBUG TaskMemoryManager: Task 56 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@69c7ce5d 21/01/17 17:43:08.542 Executor task launch worker for task 3.0 in stage 12.0 (TID 56) DEBUG TaskMemoryManager: Task 56 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@4e02a9a7 21/01/17 17:43:08.543 Executor task launch worker for task 3.0 in stage 12.0 (TID 56) INFO MemoryStore: Block rdd_50_3 stored as values in memory (estimated size 16.0 B, free 365.9 MiB) 21/01/17 17:43:08.543 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_50_3 for BlockManagerId(driver, 192.168.1.225, 40985, None) 21/01/17 17:43:08.543 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_50_3 in memory on 192.168.1.225:40985 (size: 16.0 B, free: 366.2 MiB) 21/01/17 17:43:08.543 Executor task launch worker for task 3.0 in stage 12.0 (TID 56) DEBUG BlockManagerMaster: Updated info of block rdd_50_3 21/01/17 17:43:08.543 Executor task launch worker for task 3.0 in stage 12.0 (TID 56) DEBUG BlockManager: Told master about block rdd_50_3 21/01/17 17:43:08.543 Executor task launch worker for task 3.0 in stage 12.0 (TID 56) DEBUG BlockManager: Put block rdd_50_3 locally took 5 ms 21/01/17 17:43:08.543 Executor task launch worker for task 3.0 in stage 12.0 (TID 56) DEBUG BlockManager: Putting block rdd_50_3 without replication took 5 ms 21/01/17 17:43:08.544 Executor task launch worker for task 3.0 in stage 12.0 (TID 56) DEBUG BlockManager: Getting local block rdd_50_3 21/01/17 17:43:08.544 Executor task launch worker for task 3.0 in stage 12.0 (TID 56) DEBUG BlockManager: Level for block rdd_50_3 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:08.544 Executor task launch worker for task 3.0 in stage 12.0 (TID 56) INFO Executor: Finished task 3.0 in stage 12.0 (TID 56). 2213 bytes result sent to driver 21/01/17 17:43:08.545 Executor task launch worker for task 3.0 in stage 12.0 (TID 56) DEBUG ExecutorMetricsPoller: removing (12, 0) from stageTCMP 21/01/17 17:43:08.545 dispatcher-event-loop-2 INFO TaskSetManager: Starting task 4.0 in stage 12.0 (TID 57) (192.168.1.225, executor driver, partition 4, PROCESS_LOCAL, 7433 bytes) taskResourceAssignments Map() 21/01/17 17:43:08.546 dispatcher-event-loop-2 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_12.0, runningTasks: 3 21/01/17 17:43:08.546 dispatcher-event-loop-2 INFO TaskSetManager: Starting task 5.0 in stage 12.0 (TID 58) (192.168.1.225, executor driver, partition 5, PROCESS_LOCAL, 7433 bytes) taskResourceAssignments Map() 21/01/17 17:43:08.546 task-result-getter-2 INFO TaskSetManager: Finished task 0.0 in stage 12.0 (TID 53) in 46 ms on 192.168.1.225 (executor driver) (1/10) 21/01/17 17:43:08.546 Executor task launch worker for task 5.0 in stage 12.0 (TID 58) INFO Executor: Running task 5.0 in stage 12.0 (TID 58) 21/01/17 17:43:08.546 Executor task launch worker for task 4.0 in stage 12.0 (TID 57) INFO Executor: Running task 4.0 in stage 12.0 (TID 57) 21/01/17 17:43:08.547 dispatcher-event-loop-2 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_12.0, runningTasks: 3 21/01/17 17:43:08.547 dispatcher-event-loop-2 INFO TaskSetManager: Starting task 6.0 in stage 12.0 (TID 59) (192.168.1.225, executor driver, partition 6, PROCESS_LOCAL, 7433 bytes) taskResourceAssignments Map() 21/01/17 17:43:08.547 Executor task launch worker for task 4.0 in stage 12.0 (TID 57) DEBUG ExecutorMetricsPoller: stageTCMP: (12, 0) -> 1 21/01/17 17:43:08.547 task-result-getter-3 INFO TaskSetManager: Finished task 1.0 in stage 12.0 (TID 54) in 47 ms on 192.168.1.225 (executor driver) (2/10) 21/01/17 17:43:08.548 Executor task launch worker for task 6.0 in stage 12.0 (TID 59) INFO Executor: Running task 6.0 in stage 12.0 (TID 59) 21/01/17 17:43:08.548 dispatcher-event-loop-2 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_12.0, runningTasks: 3 21/01/17 17:43:08.548 Executor task launch worker for task 5.0 in stage 12.0 (TID 58) DEBUG ExecutorMetricsPoller: stageTCMP: (12, 0) -> 3 21/01/17 17:43:08.548 dispatcher-event-loop-2 INFO TaskSetManager: Starting task 7.0 in stage 12.0 (TID 60) (192.168.1.225, executor driver, partition 7, PROCESS_LOCAL, 7433 bytes) taskResourceAssignments Map() 21/01/17 17:43:08.549 Executor task launch worker for task 7.0 in stage 12.0 (TID 60) INFO Executor: Running task 7.0 in stage 12.0 (TID 60) 21/01/17 17:43:08.549 task-result-getter-0 INFO TaskSetManager: Finished task 2.0 in stage 12.0 (TID 55) in 48 ms on 192.168.1.225 (executor driver) (3/10) 21/01/17 17:43:08.549 task-result-getter-1 INFO TaskSetManager: Finished task 3.0 in stage 12.0 (TID 56) in 48 ms on 192.168.1.225 (executor driver) (4/10) 21/01/17 17:43:08.549 Executor task launch worker for task 7.0 in stage 12.0 (TID 60) DEBUG ExecutorMetricsPoller: stageTCMP: (12, 0) -> 4 21/01/17 17:43:08.553 Executor task launch worker for task 4.0 in stage 12.0 (TID 57) DEBUG BlockManager: Getting local block rdd_50_4 21/01/17 17:43:08.554 Executor task launch worker for task 4.0 in stage 12.0 (TID 57) DEBUG BlockManager: Block rdd_50_4 was not found 21/01/17 17:43:08.554 Executor task launch worker for task 4.0 in stage 12.0 (TID 57) DEBUG BlockManager: Getting remote block rdd_50_4 21/01/17 17:43:08.554 Executor task launch worker for task 4.0 in stage 12.0 (TID 57) DEBUG BlockManager: Block rdd_50_4 is unknown by block manager master 21/01/17 17:43:08.554 Executor task launch worker for task 4.0 in stage 12.0 (TID 57) DEBUG BlockManager: Getting local block rdd_19_4 21/01/17 17:43:08.554 Executor task launch worker for task 4.0 in stage 12.0 (TID 57) DEBUG BlockManager: Level for block rdd_19_4 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:08.554 Executor task launch worker for task 4.0 in stage 12.0 (TID 57) INFO BlockManager: Found block rdd_19_4 locally 21/01/17 17:43:08.554 Executor task launch worker for task 4.0 in stage 12.0 (TID 57) DEBUG BlockManager: Getting local block rdd_19_4 21/01/17 17:43:08.554 Executor task launch worker for task 4.0 in stage 12.0 (TID 57) DEBUG BlockManager: Level for block rdd_19_4 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:08.554 Executor task launch worker for task 4.0 in stage 12.0 (TID 57) INFO BlockManager: Found block rdd_19_4 locally 21/01/17 17:43:08.554 Executor task launch worker for task 4.0 in stage 12.0 (TID 57) DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 3 21/01/17 17:43:08.554 Executor task launch worker for task 4.0 in stage 12.0 (TID 57) DEBUG MapOutputTrackerMaster: Convert map statuses for shuffle 3, mappers 0-10, partitions 4-5 21/01/17 17:43:08.555 Executor task launch worker for task 4.0 in stage 12.0 (TID 57) DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 21/01/17 17:43:08.555 Executor task launch worker for task 7.0 in stage 12.0 (TID 60) DEBUG BlockManager: Getting local block rdd_50_7 21/01/17 17:43:08.555 Executor task launch worker for task 7.0 in stage 12.0 (TID 60) DEBUG BlockManager: Block rdd_50_7 was not found 21/01/17 17:43:08.555 Executor task launch worker for task 7.0 in stage 12.0 (TID 60) DEBUG BlockManager: Getting remote block rdd_50_7 21/01/17 17:43:08.555 Executor task launch worker for task 4.0 in stage 12.0 (TID 57) INFO ShuffleBlockFetcherIterator: Getting 3 (684.0 B) non-empty blocks including 3 (684.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 21/01/17 17:43:08.555 Executor task launch worker for task 4.0 in stage 12.0 (TID 57) INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 21/01/17 17:43:08.555 Executor task launch worker for task 4.0 in stage 12.0 (TID 57) DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_3_43_4,0), (shuffle_3_44_4,1), (shuffle_3_45_4,2) 21/01/17 17:43:08.555 Executor task launch worker for task 4.0 in stage 12.0 (TID 57) DEBUG BlockManager: Getting local shuffle block shuffle_3_43_4 21/01/17 17:43:08.556 Executor task launch worker for task 4.0 in stage 12.0 (TID 57) DEBUG BlockManager: Getting local shuffle block shuffle_3_44_4 21/01/17 17:43:08.556 Executor task launch worker for task 4.0 in stage 12.0 (TID 57) DEBUG BlockManager: Getting local shuffle block shuffle_3_45_4 21/01/17 17:43:08.556 Executor task launch worker for task 4.0 in stage 12.0 (TID 57) DEBUG ShuffleBlockFetcherIterator: Got local blocks in 1 ms 21/01/17 17:43:08.548 Executor task launch worker for task 6.0 in stage 12.0 (TID 59) DEBUG ExecutorMetricsPoller: stageTCMP: (12, 0) -> 2 21/01/17 17:43:08.566 Executor task launch worker for task 6.0 in stage 12.0 (TID 59) DEBUG BlockManager: Getting local block rdd_50_6 21/01/17 17:43:08.567 Executor task launch worker for task 6.0 in stage 12.0 (TID 59) DEBUG BlockManager: Block rdd_50_6 was not found 21/01/17 17:43:08.567 Executor task launch worker for task 6.0 in stage 12.0 (TID 59) DEBUG BlockManager: Getting remote block rdd_50_6 21/01/17 17:43:08.567 Executor task launch worker for task 7.0 in stage 12.0 (TID 60) DEBUG BlockManager: Block rdd_50_7 is unknown by block manager master 21/01/17 17:43:08.567 Executor task launch worker for task 7.0 in stage 12.0 (TID 60) DEBUG BlockManager: Getting local block rdd_19_7 21/01/17 17:43:08.567 Executor task launch worker for task 7.0 in stage 12.0 (TID 60) DEBUG BlockManager: Level for block rdd_19_7 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:08.567 Executor task launch worker for task 7.0 in stage 12.0 (TID 60) INFO BlockManager: Found block rdd_19_7 locally 21/01/17 17:43:08.567 Executor task launch worker for task 7.0 in stage 12.0 (TID 60) DEBUG BlockManager: Getting local block rdd_19_7 21/01/17 17:43:08.567 Executor task launch worker for task 7.0 in stage 12.0 (TID 60) DEBUG BlockManager: Level for block rdd_19_7 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:08.567 Executor task launch worker for task 7.0 in stage 12.0 (TID 60) INFO BlockManager: Found block rdd_19_7 locally 21/01/17 17:43:08.567 Executor task launch worker for task 7.0 in stage 12.0 (TID 60) DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 3 21/01/17 17:43:08.567 Executor task launch worker for task 7.0 in stage 12.0 (TID 60) DEBUG MapOutputTrackerMaster: Convert map statuses for shuffle 3, mappers 0-10, partitions 7-8 21/01/17 17:43:08.568 Executor task launch worker for task 7.0 in stage 12.0 (TID 60) DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 21/01/17 17:43:08.568 Executor task launch worker for task 4.0 in stage 12.0 (TID 57) DEBUG TaskMemoryManager: Task 57 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@7b8bcc49 21/01/17 17:43:08.568 Executor task launch worker for task 7.0 in stage 12.0 (TID 60) INFO ShuffleBlockFetcherIterator: Getting 3 (684.0 B) non-empty blocks including 3 (684.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 21/01/17 17:43:08.568 Executor task launch worker for task 7.0 in stage 12.0 (TID 60) INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 21/01/17 17:43:08.568 Executor task launch worker for task 7.0 in stage 12.0 (TID 60) DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_3_43_7,0), (shuffle_3_44_7,1), (shuffle_3_45_7,2) 21/01/17 17:43:08.568 Executor task launch worker for task 7.0 in stage 12.0 (TID 60) DEBUG BlockManager: Getting local shuffle block shuffle_3_43_7 21/01/17 17:43:08.569 Executor task launch worker for task 4.0 in stage 12.0 (TID 57) DEBUG TaskMemoryManager: Task 57 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@1ca53c82 21/01/17 17:43:08.569 Executor task launch worker for task 7.0 in stage 12.0 (TID 60) DEBUG BlockManager: Getting local shuffle block shuffle_3_44_7 21/01/17 17:43:08.569 Executor task launch worker for task 7.0 in stage 12.0 (TID 60) DEBUG BlockManager: Getting local shuffle block shuffle_3_45_7 21/01/17 17:43:08.569 Executor task launch worker for task 4.0 in stage 12.0 (TID 57) DEBUG TaskMemoryManager: Task 57 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@573f9875 21/01/17 17:43:08.569 Executor task launch worker for task 4.0 in stage 12.0 (TID 57) INFO MemoryStore: Block rdd_50_4 stored as values in memory (estimated size 16.0 B, free 365.9 MiB) 21/01/17 17:43:08.569 Executor task launch worker for task 6.0 in stage 12.0 (TID 59) DEBUG BlockManager: Block rdd_50_6 is unknown by block manager master 21/01/17 17:43:08.570 Executor task launch worker for task 6.0 in stage 12.0 (TID 59) DEBUG BlockManager: Getting local block rdd_19_6 21/01/17 17:43:08.569 Executor task launch worker for task 7.0 in stage 12.0 (TID 60) DEBUG ShuffleBlockFetcherIterator: Got local blocks in 1 ms 21/01/17 17:43:08.570 Executor task launch worker for task 6.0 in stage 12.0 (TID 59) DEBUG BlockManager: Level for block rdd_19_6 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:08.570 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_50_4 for BlockManagerId(driver, 192.168.1.225, 40985, None) 21/01/17 17:43:08.570 Executor task launch worker for task 6.0 in stage 12.0 (TID 59) INFO BlockManager: Found block rdd_19_6 locally 21/01/17 17:43:08.570 Executor task launch worker for task 6.0 in stage 12.0 (TID 59) DEBUG BlockManager: Getting local block rdd_19_6 21/01/17 17:43:08.571 Executor task launch worker for task 6.0 in stage 12.0 (TID 59) DEBUG BlockManager: Level for block rdd_19_6 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:08.571 Executor task launch worker for task 6.0 in stage 12.0 (TID 59) INFO BlockManager: Found block rdd_19_6 locally 21/01/17 17:43:08.571 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_50_4 in memory on 192.168.1.225:40985 (size: 16.0 B, free: 366.2 MiB) 21/01/17 17:43:08.571 Executor task launch worker for task 6.0 in stage 12.0 (TID 59) DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 3 21/01/17 17:43:08.571 Executor task launch worker for task 6.0 in stage 12.0 (TID 59) DEBUG MapOutputTrackerMaster: Convert map statuses for shuffle 3, mappers 0-10, partitions 6-7 21/01/17 17:43:08.571 Executor task launch worker for task 6.0 in stage 12.0 (TID 59) DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 21/01/17 17:43:08.571 Executor task launch worker for task 7.0 in stage 12.0 (TID 60) DEBUG TaskMemoryManager: Task 60 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@3095152 21/01/17 17:43:08.572 Executor task launch worker for task 6.0 in stage 12.0 (TID 59) INFO ShuffleBlockFetcherIterator: Getting 3 (684.0 B) non-empty blocks including 3 (684.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 21/01/17 17:43:08.572 Executor task launch worker for task 6.0 in stage 12.0 (TID 59) INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 21/01/17 17:43:08.572 Executor task launch worker for task 6.0 in stage 12.0 (TID 59) DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_3_43_6,0), (shuffle_3_44_6,1), (shuffle_3_45_6,2) 21/01/17 17:43:08.572 Executor task launch worker for task 6.0 in stage 12.0 (TID 59) DEBUG BlockManager: Getting local shuffle block shuffle_3_43_6 21/01/17 17:43:08.572 Executor task launch worker for task 7.0 in stage 12.0 (TID 60) DEBUG TaskMemoryManager: Task 60 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@1e2bd623 21/01/17 17:43:08.572 Executor task launch worker for task 6.0 in stage 12.0 (TID 59) DEBUG BlockManager: Getting local shuffle block shuffle_3_44_6 21/01/17 17:43:08.572 Executor task launch worker for task 6.0 in stage 12.0 (TID 59) DEBUG BlockManager: Getting local shuffle block shuffle_3_45_6 21/01/17 17:43:08.572 Executor task launch worker for task 7.0 in stage 12.0 (TID 60) DEBUG TaskMemoryManager: Task 60 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@7b034693 21/01/17 17:43:08.572 Executor task launch worker for task 6.0 in stage 12.0 (TID 59) DEBUG ShuffleBlockFetcherIterator: Got local blocks in 1 ms 21/01/17 17:43:08.573 Executor task launch worker for task 7.0 in stage 12.0 (TID 60) INFO MemoryStore: Block rdd_50_7 stored as values in memory (estimated size 16.0 B, free 365.9 MiB) 21/01/17 17:43:08.573 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_50_7 for BlockManagerId(driver, 192.168.1.225, 40985, None) 21/01/17 17:43:08.573 Executor task launch worker for task 4.0 in stage 12.0 (TID 57) DEBUG BlockManagerMaster: Updated info of block rdd_50_4 21/01/17 17:43:08.573 Executor task launch worker for task 4.0 in stage 12.0 (TID 57) DEBUG BlockManager: Told master about block rdd_50_4 21/01/17 17:43:08.573 Executor task launch worker for task 4.0 in stage 12.0 (TID 57) DEBUG BlockManager: Put block rdd_50_4 locally took 19 ms 21/01/17 17:43:08.573 Executor task launch worker for task 5.0 in stage 12.0 (TID 58) DEBUG BlockManager: Getting local block rdd_50_5 21/01/17 17:43:08.573 Executor task launch worker for task 4.0 in stage 12.0 (TID 57) DEBUG BlockManager: Putting block rdd_50_4 without replication took 19 ms 21/01/17 17:43:08.573 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_50_7 in memory on 192.168.1.225:40985 (size: 16.0 B, free: 366.2 MiB) 21/01/17 17:43:08.573 Executor task launch worker for task 5.0 in stage 12.0 (TID 58) DEBUG BlockManager: Block rdd_50_5 was not found 21/01/17 17:43:08.574 Executor task launch worker for task 5.0 in stage 12.0 (TID 58) DEBUG BlockManager: Getting remote block rdd_50_5 21/01/17 17:43:08.573 Executor task launch worker for task 4.0 in stage 12.0 (TID 57) DEBUG BlockManager: Getting local block rdd_50_4 21/01/17 17:43:08.577 Executor task launch worker for task 4.0 in stage 12.0 (TID 57) DEBUG BlockManager: Level for block rdd_50_4 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:08.577 Executor task launch worker for task 7.0 in stage 12.0 (TID 60) DEBUG BlockManagerMaster: Updated info of block rdd_50_7 21/01/17 17:43:08.577 Executor task launch worker for task 7.0 in stage 12.0 (TID 60) DEBUG BlockManager: Told master about block rdd_50_7 21/01/17 17:43:08.577 Executor task launch worker for task 7.0 in stage 12.0 (TID 60) DEBUG BlockManager: Put block rdd_50_7 locally took 10 ms 21/01/17 17:43:08.577 Executor task launch worker for task 7.0 in stage 12.0 (TID 60) DEBUG BlockManager: Putting block rdd_50_7 without replication took 10 ms 21/01/17 17:43:08.577 Executor task launch worker for task 7.0 in stage 12.0 (TID 60) DEBUG BlockManager: Getting local block rdd_50_7 21/01/17 17:43:08.577 Executor task launch worker for task 7.0 in stage 12.0 (TID 60) DEBUG BlockManager: Level for block rdd_50_7 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:08.578 Executor task launch worker for task 4.0 in stage 12.0 (TID 57) INFO Executor: Finished task 4.0 in stage 12.0 (TID 57). 2213 bytes result sent to driver 21/01/17 17:43:08.578 Executor task launch worker for task 4.0 in stage 12.0 (TID 57) DEBUG ExecutorMetricsPoller: stageTCMP: (12, 0) -> 3 21/01/17 17:43:08.578 Executor task launch worker for task 7.0 in stage 12.0 (TID 60) INFO Executor: Finished task 7.0 in stage 12.0 (TID 60). 2213 bytes result sent to driver 21/01/17 17:43:08.578 Executor task launch worker for task 7.0 in stage 12.0 (TID 60) DEBUG ExecutorMetricsPoller: stageTCMP: (12, 0) -> 2 21/01/17 17:43:08.578 dispatcher-event-loop-3 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_12.0, runningTasks: 3 21/01/17 17:43:08.576 Executor task launch worker for task 6.0 in stage 12.0 (TID 59) DEBUG TaskMemoryManager: Task 59 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@347b5bbb 21/01/17 17:43:08.579 dispatcher-event-loop-3 INFO TaskSetManager: Starting task 8.0 in stage 12.0 (TID 61) (192.168.1.225, executor driver, partition 8, PROCESS_LOCAL, 7433 bytes) taskResourceAssignments Map() 21/01/17 17:43:08.579 Executor task launch worker for task 6.0 in stage 12.0 (TID 59) DEBUG TaskMemoryManager: Task 59 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@5f815876 21/01/17 17:43:08.579 dispatcher-event-loop-3 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_12.0, runningTasks: 3 21/01/17 17:43:08.580 Executor task launch worker for task 6.0 in stage 12.0 (TID 59) DEBUG TaskMemoryManager: Task 59 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@185816f6 21/01/17 17:43:08.580 dispatcher-event-loop-3 INFO TaskSetManager: Starting task 9.0 in stage 12.0 (TID 62) (192.168.1.225, executor driver, partition 9, PROCESS_LOCAL, 7433 bytes) taskResourceAssignments Map() 21/01/17 17:43:08.580 Executor task launch worker for task 6.0 in stage 12.0 (TID 59) INFO MemoryStore: Block rdd_50_6 stored as values in memory (estimated size 16.0 B, free 365.9 MiB) 21/01/17 17:43:08.580 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_50_6 for BlockManagerId(driver, 192.168.1.225, 40985, None) 21/01/17 17:43:08.580 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_50_6 in memory on 192.168.1.225:40985 (size: 16.0 B, free: 366.2 MiB) 21/01/17 17:43:08.580 task-result-getter-2 INFO TaskSetManager: Finished task 4.0 in stage 12.0 (TID 57) in 39 ms on 192.168.1.225 (executor driver) (5/10) 21/01/17 17:43:08.581 Executor task launch worker for task 6.0 in stage 12.0 (TID 59) DEBUG BlockManagerMaster: Updated info of block rdd_50_6 21/01/17 17:43:08.581 Executor task launch worker for task 6.0 in stage 12.0 (TID 59) DEBUG BlockManager: Told master about block rdd_50_6 21/01/17 17:43:08.581 Executor task launch worker for task 6.0 in stage 12.0 (TID 59) DEBUG BlockManager: Put block rdd_50_6 locally took 11 ms 21/01/17 17:43:08.581 Executor task launch worker for task 6.0 in stage 12.0 (TID 59) DEBUG BlockManager: Putting block rdd_50_6 without replication took 11 ms 21/01/17 17:43:08.581 Executor task launch worker for task 6.0 in stage 12.0 (TID 59) DEBUG BlockManager: Getting local block rdd_50_6 21/01/17 17:43:08.581 Executor task launch worker for task 6.0 in stage 12.0 (TID 59) DEBUG BlockManager: Level for block rdd_50_6 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:08.575 Executor task launch worker for task 5.0 in stage 12.0 (TID 58) DEBUG BlockManager: Block rdd_50_5 is unknown by block manager master 21/01/17 17:43:08.581 Executor task launch worker for task 5.0 in stage 12.0 (TID 58) DEBUG BlockManager: Getting local block rdd_19_5 21/01/17 17:43:08.581 Executor task launch worker for task 5.0 in stage 12.0 (TID 58) DEBUG BlockManager: Level for block rdd_19_5 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:08.581 Executor task launch worker for task 5.0 in stage 12.0 (TID 58) INFO BlockManager: Found block rdd_19_5 locally 21/01/17 17:43:08.581 Executor task launch worker for task 5.0 in stage 12.0 (TID 58) DEBUG BlockManager: Getting local block rdd_19_5 21/01/17 17:43:08.581 Executor task launch worker for task 5.0 in stage 12.0 (TID 58) DEBUG BlockManager: Level for block rdd_19_5 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:08.581 Executor task launch worker for task 5.0 in stage 12.0 (TID 58) INFO BlockManager: Found block rdd_19_5 locally 21/01/17 17:43:08.581 Executor task launch worker for task 5.0 in stage 12.0 (TID 58) DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 3 21/01/17 17:43:08.582 Executor task launch worker for task 5.0 in stage 12.0 (TID 58) DEBUG MapOutputTrackerMaster: Convert map statuses for shuffle 3, mappers 0-10, partitions 5-6 21/01/17 17:43:08.581 Executor task launch worker for task 8.0 in stage 12.0 (TID 61) INFO Executor: Running task 8.0 in stage 12.0 (TID 61) 21/01/17 17:43:08.581 task-result-getter-3 INFO TaskSetManager: Finished task 7.0 in stage 12.0 (TID 60) in 33 ms on 192.168.1.225 (executor driver) (6/10) 21/01/17 17:43:08.582 Executor task launch worker for task 6.0 in stage 12.0 (TID 59) INFO Executor: Finished task 6.0 in stage 12.0 (TID 59). 2213 bytes result sent to driver 21/01/17 17:43:08.582 Executor task launch worker for task 6.0 in stage 12.0 (TID 59) DEBUG ExecutorMetricsPoller: stageTCMP: (12, 0) -> 1 21/01/17 17:43:08.582 Executor task launch worker for task 5.0 in stage 12.0 (TID 58) DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 21/01/17 17:43:08.582 dispatcher-event-loop-1 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_12.0, runningTasks: 3 21/01/17 17:43:08.582 Executor task launch worker for task 9.0 in stage 12.0 (TID 62) INFO Executor: Running task 9.0 in stage 12.0 (TID 62) 21/01/17 17:43:08.582 Executor task launch worker for task 8.0 in stage 12.0 (TID 61) DEBUG ExecutorMetricsPoller: stageTCMP: (12, 0) -> 2 21/01/17 17:43:08.582 dispatcher-event-loop-1 DEBUG TaskSetManager: No tasks for locality level PROCESS_LOCAL, so moving to locality level NODE_LOCAL 21/01/17 17:43:08.583 dispatcher-event-loop-1 DEBUG TaskSetManager: No tasks for locality level NODE_LOCAL, so moving to locality level ANY 21/01/17 17:43:08.583 Executor task launch worker for task 5.0 in stage 12.0 (TID 58) INFO ShuffleBlockFetcherIterator: Getting 3 (684.0 B) non-empty blocks including 3 (684.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 21/01/17 17:43:08.583 Executor task launch worker for task 5.0 in stage 12.0 (TID 58) INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 21/01/17 17:43:08.583 Executor task launch worker for task 5.0 in stage 12.0 (TID 58) DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_3_43_5,0), (shuffle_3_44_5,1), (shuffle_3_45_5,2) 21/01/17 17:43:08.583 Executor task launch worker for task 5.0 in stage 12.0 (TID 58) DEBUG BlockManager: Getting local shuffle block shuffle_3_43_5 21/01/17 17:43:08.583 Executor task launch worker for task 5.0 in stage 12.0 (TID 58) DEBUG BlockManager: Getting local shuffle block shuffle_3_44_5 21/01/17 17:43:08.583 task-result-getter-0 INFO TaskSetManager: Finished task 6.0 in stage 12.0 (TID 59) in 36 ms on 192.168.1.225 (executor driver) (7/10) 21/01/17 17:43:08.583 Executor task launch worker for task 5.0 in stage 12.0 (TID 58) DEBUG BlockManager: Getting local shuffle block shuffle_3_45_5 21/01/17 17:43:08.583 Executor task launch worker for task 5.0 in stage 12.0 (TID 58) DEBUG ShuffleBlockFetcherIterator: Got local blocks in 1 ms 21/01/17 17:43:08.585 Executor task launch worker for task 5.0 in stage 12.0 (TID 58) DEBUG TaskMemoryManager: Task 58 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@43aa1cbf 21/01/17 17:43:08.585 Executor task launch worker for task 5.0 in stage 12.0 (TID 58) DEBUG TaskMemoryManager: Task 58 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@45f70bdc 21/01/17 17:43:08.585 Executor task launch worker for task 5.0 in stage 12.0 (TID 58) DEBUG TaskMemoryManager: Task 58 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@28ee2e7a 21/01/17 17:43:08.586 Executor task launch worker for task 5.0 in stage 12.0 (TID 58) INFO MemoryStore: Block rdd_50_5 stored as values in memory (estimated size 16.0 B, free 365.9 MiB) 21/01/17 17:43:08.586 Executor task launch worker for task 9.0 in stage 12.0 (TID 62) DEBUG ExecutorMetricsPoller: stageTCMP: (12, 0) -> 3 21/01/17 17:43:08.589 Executor task launch worker for task 8.0 in stage 12.0 (TID 61) DEBUG BlockManager: Getting local block rdd_50_8 21/01/17 17:43:08.589 Executor task launch worker for task 8.0 in stage 12.0 (TID 61) DEBUG BlockManager: Block rdd_50_8 was not found 21/01/17 17:43:08.589 Executor task launch worker for task 8.0 in stage 12.0 (TID 61) DEBUG BlockManager: Getting remote block rdd_50_8 21/01/17 17:43:08.589 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_50_5 for BlockManagerId(driver, 192.168.1.225, 40985, None) 21/01/17 17:43:08.590 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_50_5 in memory on 192.168.1.225:40985 (size: 16.0 B, free: 366.2 MiB) 21/01/17 17:43:08.591 Executor task launch worker for task 8.0 in stage 12.0 (TID 61) DEBUG BlockManager: Block rdd_50_8 is unknown by block manager master 21/01/17 17:43:08.591 Executor task launch worker for task 5.0 in stage 12.0 (TID 58) DEBUG BlockManagerMaster: Updated info of block rdd_50_5 21/01/17 17:43:08.591 Executor task launch worker for task 5.0 in stage 12.0 (TID 58) DEBUG BlockManager: Told master about block rdd_50_5 21/01/17 17:43:08.591 Executor task launch worker for task 5.0 in stage 12.0 (TID 58) DEBUG BlockManager: Put block rdd_50_5 locally took 9 ms 21/01/17 17:43:08.591 Executor task launch worker for task 8.0 in stage 12.0 (TID 61) DEBUG BlockManager: Getting local block rdd_19_8 21/01/17 17:43:08.591 Executor task launch worker for task 5.0 in stage 12.0 (TID 58) DEBUG BlockManager: Putting block rdd_50_5 without replication took 9 ms 21/01/17 17:43:08.591 Executor task launch worker for task 5.0 in stage 12.0 (TID 58) DEBUG BlockManager: Getting local block rdd_50_5 21/01/17 17:43:08.591 Executor task launch worker for task 8.0 in stage 12.0 (TID 61) DEBUG BlockManager: Level for block rdd_19_8 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:08.591 Executor task launch worker for task 5.0 in stage 12.0 (TID 58) DEBUG BlockManager: Level for block rdd_50_5 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:08.591 Executor task launch worker for task 8.0 in stage 12.0 (TID 61) INFO BlockManager: Found block rdd_19_8 locally 21/01/17 17:43:08.591 Executor task launch worker for task 8.0 in stage 12.0 (TID 61) DEBUG BlockManager: Getting local block rdd_19_8 21/01/17 17:43:08.591 Executor task launch worker for task 8.0 in stage 12.0 (TID 61) DEBUG BlockManager: Level for block rdd_19_8 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:08.591 Executor task launch worker for task 8.0 in stage 12.0 (TID 61) INFO BlockManager: Found block rdd_19_8 locally 21/01/17 17:43:08.591 Executor task launch worker for task 8.0 in stage 12.0 (TID 61) DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 3 21/01/17 17:43:08.591 Executor task launch worker for task 8.0 in stage 12.0 (TID 61) DEBUG MapOutputTrackerMaster: Convert map statuses for shuffle 3, mappers 0-10, partitions 8-9 21/01/17 17:43:08.592 Executor task launch worker for task 8.0 in stage 12.0 (TID 61) DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 21/01/17 17:43:08.592 Executor task launch worker for task 5.0 in stage 12.0 (TID 58) INFO Executor: Finished task 5.0 in stage 12.0 (TID 58). 2213 bytes result sent to driver 21/01/17 17:43:08.592 Executor task launch worker for task 8.0 in stage 12.0 (TID 61) INFO ShuffleBlockFetcherIterator: Getting 3 (684.0 B) non-empty blocks including 3 (684.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 21/01/17 17:43:08.592 Executor task launch worker for task 8.0 in stage 12.0 (TID 61) INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 21/01/17 17:43:08.592 Executor task launch worker for task 8.0 in stage 12.0 (TID 61) DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_3_43_8,0), (shuffle_3_44_8,1), (shuffle_3_45_8,2) 21/01/17 17:43:08.592 Executor task launch worker for task 8.0 in stage 12.0 (TID 61) DEBUG BlockManager: Getting local shuffle block shuffle_3_43_8 21/01/17 17:43:08.593 Executor task launch worker for task 8.0 in stage 12.0 (TID 61) DEBUG BlockManager: Getting local shuffle block shuffle_3_44_8 21/01/17 17:43:08.593 Executor task launch worker for task 8.0 in stage 12.0 (TID 61) DEBUG BlockManager: Getting local shuffle block shuffle_3_45_8 21/01/17 17:43:08.593 task-result-getter-1 INFO TaskSetManager: Finished task 5.0 in stage 12.0 (TID 58) in 47 ms on 192.168.1.225 (executor driver) (8/10) 21/01/17 17:43:08.593 Executor task launch worker for task 5.0 in stage 12.0 (TID 58) DEBUG ExecutorMetricsPoller: stageTCMP: (12, 0) -> 2 21/01/17 17:43:08.593 Executor task launch worker for task 9.0 in stage 12.0 (TID 62) DEBUG BlockManager: Getting local block rdd_50_9 21/01/17 17:43:08.593 Executor task launch worker for task 9.0 in stage 12.0 (TID 62) DEBUG BlockManager: Block rdd_50_9 was not found 21/01/17 17:43:08.593 Executor task launch worker for task 9.0 in stage 12.0 (TID 62) DEBUG BlockManager: Getting remote block rdd_50_9 21/01/17 17:43:08.593 Executor task launch worker for task 9.0 in stage 12.0 (TID 62) DEBUG BlockManager: Block rdd_50_9 is unknown by block manager master 21/01/17 17:43:08.593 Executor task launch worker for task 9.0 in stage 12.0 (TID 62) DEBUG BlockManager: Getting local block rdd_19_9 21/01/17 17:43:08.593 Executor task launch worker for task 9.0 in stage 12.0 (TID 62) DEBUG BlockManager: Level for block rdd_19_9 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:08.595 Executor task launch worker for task 9.0 in stage 12.0 (TID 62) INFO BlockManager: Found block rdd_19_9 locally 21/01/17 17:43:08.595 Executor task launch worker for task 9.0 in stage 12.0 (TID 62) DEBUG BlockManager: Getting local block rdd_19_9 21/01/17 17:43:08.595 Executor task launch worker for task 9.0 in stage 12.0 (TID 62) DEBUG BlockManager: Level for block rdd_19_9 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:08.595 Executor task launch worker for task 9.0 in stage 12.0 (TID 62) INFO BlockManager: Found block rdd_19_9 locally 21/01/17 17:43:08.595 Executor task launch worker for task 9.0 in stage 12.0 (TID 62) DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 3 21/01/17 17:43:08.594 Executor task launch worker for task 8.0 in stage 12.0 (TID 61) DEBUG ShuffleBlockFetcherIterator: Got local blocks in 1 ms 21/01/17 17:43:08.595 Executor task launch worker for task 9.0 in stage 12.0 (TID 62) DEBUG MapOutputTrackerMaster: Convert map statuses for shuffle 3, mappers 0-10, partitions 9-10 21/01/17 17:43:08.597 Executor task launch worker for task 9.0 in stage 12.0 (TID 62) DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 21/01/17 17:43:08.597 Executor task launch worker for task 9.0 in stage 12.0 (TID 62) INFO ShuffleBlockFetcherIterator: Getting 3 (684.0 B) non-empty blocks including 3 (684.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 21/01/17 17:43:08.597 Executor task launch worker for task 9.0 in stage 12.0 (TID 62) INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 21/01/17 17:43:08.598 Executor task launch worker for task 9.0 in stage 12.0 (TID 62) DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_3_43_9,0), (shuffle_3_44_9,1), (shuffle_3_45_9,2) 21/01/17 17:43:08.598 Executor task launch worker for task 9.0 in stage 12.0 (TID 62) DEBUG BlockManager: Getting local shuffle block shuffle_3_43_9 21/01/17 17:43:08.598 Executor task launch worker for task 9.0 in stage 12.0 (TID 62) DEBUG BlockManager: Getting local shuffle block shuffle_3_44_9 21/01/17 17:43:08.598 Executor task launch worker for task 9.0 in stage 12.0 (TID 62) DEBUG BlockManager: Getting local shuffle block shuffle_3_45_9 21/01/17 17:43:08.598 Executor task launch worker for task 9.0 in stage 12.0 (TID 62) DEBUG ShuffleBlockFetcherIterator: Got local blocks in 1 ms 21/01/17 17:43:08.601 Executor task launch worker for task 9.0 in stage 12.0 (TID 62) DEBUG TaskMemoryManager: Task 62 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@41115952 21/01/17 17:43:08.602 Executor task launch worker for task 9.0 in stage 12.0 (TID 62) DEBUG TaskMemoryManager: Task 62 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@358e04a 21/01/17 17:43:08.602 Executor task launch worker for task 9.0 in stage 12.0 (TID 62) DEBUG TaskMemoryManager: Task 62 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@3e097f5c 21/01/17 17:43:08.602 Executor task launch worker for task 8.0 in stage 12.0 (TID 61) DEBUG TaskMemoryManager: Task 61 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@63b6eaca 21/01/17 17:43:08.602 Executor task launch worker for task 9.0 in stage 12.0 (TID 62) INFO MemoryStore: Block rdd_50_9 stored as values in memory (estimated size 16.0 B, free 365.9 MiB) 21/01/17 17:43:08.603 Executor task launch worker for task 8.0 in stage 12.0 (TID 61) DEBUG TaskMemoryManager: Task 61 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@1306b021 21/01/17 17:43:08.603 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_50_9 for BlockManagerId(driver, 192.168.1.225, 40985, None) 21/01/17 17:43:08.603 Executor task launch worker for task 8.0 in stage 12.0 (TID 61) DEBUG TaskMemoryManager: Task 61 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@7e0d0191 21/01/17 17:43:08.603 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_50_9 in memory on 192.168.1.225:40985 (size: 16.0 B, free: 366.2 MiB) 21/01/17 17:43:08.603 Executor task launch worker for task 8.0 in stage 12.0 (TID 61) INFO MemoryStore: Block rdd_50_8 stored as values in memory (estimated size 16.0 B, free 365.9 MiB) 21/01/17 17:43:08.603 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_50_8 for BlockManagerId(driver, 192.168.1.225, 40985, None) 21/01/17 17:43:08.604 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_50_8 in memory on 192.168.1.225:40985 (size: 16.0 B, free: 366.2 MiB) 21/01/17 17:43:08.604 Executor task launch worker for task 8.0 in stage 12.0 (TID 61) DEBUG BlockManagerMaster: Updated info of block rdd_50_8 21/01/17 17:43:08.604 Executor task launch worker for task 8.0 in stage 12.0 (TID 61) DEBUG BlockManager: Told master about block rdd_50_8 21/01/17 17:43:08.604 Executor task launch worker for task 8.0 in stage 12.0 (TID 61) DEBUG BlockManager: Put block rdd_50_8 locally took 13 ms 21/01/17 17:43:08.605 Executor task launch worker for task 8.0 in stage 12.0 (TID 61) DEBUG BlockManager: Putting block rdd_50_8 without replication took 13 ms 21/01/17 17:43:08.605 Executor task launch worker for task 8.0 in stage 12.0 (TID 61) DEBUG BlockManager: Getting local block rdd_50_8 21/01/17 17:43:08.605 Executor task launch worker for task 8.0 in stage 12.0 (TID 61) DEBUG BlockManager: Level for block rdd_50_8 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:08.605 Executor task launch worker for task 9.0 in stage 12.0 (TID 62) DEBUG BlockManagerMaster: Updated info of block rdd_50_9 21/01/17 17:43:08.605 Executor task launch worker for task 9.0 in stage 12.0 (TID 62) DEBUG BlockManager: Told master about block rdd_50_9 21/01/17 17:43:08.606 Executor task launch worker for task 9.0 in stage 12.0 (TID 62) DEBUG BlockManager: Put block rdd_50_9 locally took 12 ms 21/01/17 17:43:08.607 Executor task launch worker for task 8.0 in stage 12.0 (TID 61) INFO Executor: Finished task 8.0 in stage 12.0 (TID 61). 2256 bytes result sent to driver 21/01/17 17:43:08.608 task-result-getter-2 INFO TaskSetManager: Finished task 8.0 in stage 12.0 (TID 61) in 29 ms on 192.168.1.225 (executor driver) (9/10) 21/01/17 17:43:08.609 Executor task launch worker for task 8.0 in stage 12.0 (TID 61) DEBUG ExecutorMetricsPoller: stageTCMP: (12, 0) -> 1 21/01/17 17:43:08.615 Executor task launch worker for task 9.0 in stage 12.0 (TID 62) DEBUG BlockManager: Putting block rdd_50_9 without replication took 21 ms 21/01/17 17:43:08.615 Executor task launch worker for task 9.0 in stage 12.0 (TID 62) DEBUG BlockManager: Getting local block rdd_50_9 21/01/17 17:43:08.615 Executor task launch worker for task 9.0 in stage 12.0 (TID 62) DEBUG BlockManager: Level for block rdd_50_9 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:08.616 Executor task launch worker for task 9.0 in stage 12.0 (TID 62) INFO Executor: Finished task 9.0 in stage 12.0 (TID 62). 2213 bytes result sent to driver 21/01/17 17:43:08.616 Executor task launch worker for task 9.0 in stage 12.0 (TID 62) DEBUG ExecutorMetricsPoller: removing (12, 0) from stageTCMP 21/01/17 17:43:08.617 task-result-getter-3 INFO TaskSetManager: Finished task 9.0 in stage 12.0 (TID 62) in 37 ms on 192.168.1.225 (executor driver) (10/10) 21/01/17 17:43:08.617 task-result-getter-3 INFO TaskSchedulerImpl: Removed TaskSet 12.0, whose tasks have all completed, from pool 21/01/17 17:43:08.618 dag-scheduler-event-loop INFO DAGScheduler: ResultStage 12 (count at ALS.scala:1075) finished in 0.139 s 21/01/17 17:43:08.618 dag-scheduler-event-loop DEBUG DAGScheduler: After removal of stage 8, remaining stages = 5 21/01/17 17:43:08.619 dag-scheduler-event-loop DEBUG DAGScheduler: After removal of stage 11, remaining stages = 4 21/01/17 17:43:08.619 dag-scheduler-event-loop DEBUG DAGScheduler: After removal of stage 7, remaining stages = 3 21/01/17 17:43:08.619 dag-scheduler-event-loop DEBUG DAGScheduler: After removal of stage 10, remaining stages = 2 21/01/17 17:43:08.619 dag-scheduler-event-loop DEBUG DAGScheduler: After removal of stage 9, remaining stages = 1 21/01/17 17:43:08.619 dag-scheduler-event-loop DEBUG DAGScheduler: After removal of stage 12, remaining stages = 0 21/01/17 17:43:08.619 dag-scheduler-event-loop INFO DAGScheduler: Job 3 is finished. Cancelling potential speculative or zombie tasks for this job 21/01/17 17:43:08.619 dag-scheduler-event-loop INFO TaskSchedulerImpl: Killing all running tasks in stage 12: Stage finished 21/01/17 17:43:08.619 Thread-3 INFO DAGScheduler: Job 3 finished: count at ALS.scala:1075, took 0.550888 s 21/01/17 17:43:08.622 Executor task launch worker for task 2.0 in stage 2.0 (TID 10) DEBUG GenerateSafeProjection: code for createexternalrow(staticinvoke(class scala.collection.mutable.WrappedArray$, ObjectType(interface scala.collection.Seq), make, mapobjects(lambdavariable(MapObject, StringType, true, -1), lambdavariable(MapObject, StringType, true, -1).toString, input[0, array, true], None).array, true, false), StructField(words,ArrayType(StringType,true),true)): /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificSafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificSafeProjection extends org.apache.spark.sql.catalyst.expressions.codegen.BaseProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private InternalRow mutableRow; /* 009 */ private boolean resultIsNull_0; /* 010 */ private UTF8String value_MapObject_lambda_variable_1; /* 011 */ private boolean isNull_MapObject_lambda_variable_1; /* 012 */ private boolean globalIsNull_0; /* 013 */ private java.lang.Object[] mutableStateArray_0 = new java.lang.Object[1]; /* 014 */ /* 015 */ public SpecificSafeProjection(Object[] references) { /* 016 */ this.references = references; /* 017 */ mutableRow = (InternalRow) references[references.length - 1]; /* 018 */ /* 019 */ /* 020 */ } /* 021 */ /* 022 */ public void initialize(int partitionIndex) { /* 023 */ /* 024 */ } /* 025 */ /* 026 */ public java.lang.Object apply(java.lang.Object _i) { /* 027 */ InternalRow i = (InternalRow) _i; /* 028 */ org.apache.spark.sql.Row value_7 = CreateExternalRow_0(i); /* 029 */ if (false) { /* 030 */ mutableRow.setNullAt(0); /* 031 */ } else { /* 032 */ /* 033 */ mutableRow.update(0, value_7); /* 034 */ } /* 035 */ /* 036 */ return mutableRow; /* 037 */ } /* 038 */ /* 039 */ /* 040 */ private org.apache.spark.sql.Row CreateExternalRow_0(InternalRow i) { /* 041 */ Object[] values_0 = new Object[1]; /* 042 */ /* 043 */ resultIsNull_0 = false; /* 044 */ if (!resultIsNull_0) { /* 045 */ ArrayData value_6 = MapObjects_0(i); /* 046 */ boolean isNull_2 = true; /* 047 */ java.lang.Object value_2 = null; /* 048 */ if (!globalIsNull_0) { /* 049 */ /* 050 */ isNull_2 = false; /* 051 */ if (!isNull_2) { /* 052 */ /* 053 */ Object funcResult_1 = null; /* 054 */ funcResult_1 = value_6.array(); /* 055 */ value_2 = (java.lang.Object) funcResult_1; /* 056 */ /* 057 */ } /* 058 */ } /* 059 */ resultIsNull_0 = isNull_2; /* 060 */ mutableStateArray_0[0] = value_2; /* 061 */ } /* 062 */ /* 063 */ boolean isNull_1 = resultIsNull_0; /* 064 */ scala.collection.Seq value_1 = null; /* 065 */ if (!resultIsNull_0) { /* 066 */ value_1 = scala.collection.mutable.WrappedArray.make(mutableStateArray_0[0]); /* 067 */ } /* 068 */ if (isNull_1) { /* 069 */ values_0[0] = null; /* 070 */ } else { /* 071 */ values_0[0] = value_1; /* 072 */ } /* 073 */ /* 074 */ final org.apache.spark.sql.Row value_0 = new org.apache.spark.sql.catalyst.expressions.GenericRowWithSchema(values_0, ((org.apache.spark.sql.types.StructType) references[0] /* schema */)); /* 075 */ /* 076 */ return value_0; /* 077 */ } /* 078 */ /* 079 */ /* 080 */ private ArrayData MapObjects_0(InternalRow i) { /* 081 */ boolean isNull_4 = i.isNullAt(0); /* 082 */ ArrayData value_4 = isNull_4 ? /* 083 */ null : (i.getArray(0)); /* 084 */ ArrayData value_3 = null; /* 085 */ /* 086 */ if (!isNull_4) { /* 087 */ /* 088 */ int dataLength_0 = value_4.numElements(); /* 089 */ /* 090 */ java.lang.String[] convertedArray_0 = null; /* 091 */ convertedArray_0 = new java.lang.String[dataLength_0]; /* 092 */ /* 093 */ /* 094 */ int loopIndex_0 = 0; /* 095 */ /* 096 */ while (loopIndex_0 < dataLength_0) { /* 097 */ value_MapObject_lambda_variable_1 = (UTF8String) (value_4.getUTF8String(loopIndex_0)); /* 098 */ isNull_MapObject_lambda_variable_1 = value_4.isNullAt(loopIndex_0); /* 099 */ /* 100 */ boolean isNull_5 = true; /* 101 */ java.lang.String value_5 = null; /* 102 */ if (!isNull_MapObject_lambda_variable_1) { /* 103 */ /* 104 */ isNull_5 = false; /* 105 */ if (!isNull_5) { /* 106 */ /* 107 */ Object funcResult_0 = null; /* 108 */ funcResult_0 = value_MapObject_lambda_variable_1.toString(); /* 109 */ value_5 = (java.lang.String) funcResult_0; /* 110 */ /* 111 */ } /* 112 */ } /* 113 */ if (isNull_5) { /* 114 */ convertedArray_0[loopIndex_0] = null; /* 115 */ } else { /* 116 */ convertedArray_0[loopIndex_0] = value_5; /* 117 */ } /* 118 */ /* 119 */ loopIndex_0 += 1; /* 120 */ } /* 121 */ /* 122 */ value_3 = new org.apache.spark.sql.catalyst.util.GenericArrayData(convertedArray_0); /* 123 */ } /* 124 */ globalIsNull_0 = isNull_4; /* 125 */ return value_3; /* 126 */ } /* 127 */ /* 128 */ } 21/01/17 17:43:08.622 Thread-3 INFO MapPartitionsRDD: Removing RDD 19 from persistence list 21/01/17 17:43:08.626 Executor task launch worker for task 0.0 in stage 2.0 (TID 8) DEBUG GenerateSafeProjection: code for createexternalrow(staticinvoke(class scala.collection.mutable.WrappedArray$, ObjectType(interface scala.collection.Seq), make, mapobjects(lambdavariable(MapObject, StringType, true, -1), lambdavariable(MapObject, StringType, true, -1).toString, input[0, array, true], None).array, true, false), StructField(words,ArrayType(StringType,true),true)): /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificSafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificSafeProjection extends org.apache.spark.sql.catalyst.expressions.codegen.BaseProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private InternalRow mutableRow; /* 009 */ private boolean resultIsNull_0; /* 010 */ private UTF8String value_MapObject_lambda_variable_1; /* 011 */ private boolean isNull_MapObject_lambda_variable_1; /* 012 */ private boolean globalIsNull_0; /* 013 */ private java.lang.Object[] mutableStateArray_0 = new java.lang.Object[1]; /* 014 */ /* 015 */ public SpecificSafeProjection(Object[] references) { /* 016 */ this.references = references; /* 017 */ mutableRow = (InternalRow) references[references.length - 1]; /* 018 */ /* 019 */ /* 020 */ } /* 021 */ /* 022 */ public void initialize(int partitionIndex) { /* 023 */ /* 024 */ } /* 025 */ /* 026 */ public java.lang.Object apply(java.lang.Object _i) { /* 027 */ InternalRow i = (InternalRow) _i; /* 028 */ org.apache.spark.sql.Row value_7 = CreateExternalRow_0(i); /* 029 */ if (false) { /* 030 */ mutableRow.setNullAt(0); /* 031 */ } else { /* 032 */ /* 033 */ mutableRow.update(0, value_7); /* 034 */ } /* 035 */ /* 036 */ return mutableRow; /* 037 */ } /* 038 */ /* 039 */ /* 040 */ private org.apache.spark.sql.Row CreateExternalRow_0(InternalRow i) { /* 041 */ Object[] values_0 = new Object[1]; /* 042 */ /* 043 */ resultIsNull_0 = false; /* 044 */ if (!resultIsNull_0) { /* 045 */ ArrayData value_6 = MapObjects_0(i); /* 046 */ boolean isNull_2 = true; /* 047 */ java.lang.Object value_2 = null; /* 048 */ if (!globalIsNull_0) { /* 049 */ /* 050 */ isNull_2 = false; /* 051 */ if (!isNull_2) { /* 052 */ /* 053 */ Object funcResult_1 = null; /* 054 */ funcResult_1 = value_6.array(); /* 055 */ value_2 = (java.lang.Object) funcResult_1; /* 056 */ /* 057 */ } /* 058 */ } /* 059 */ resultIsNull_0 = isNull_2; /* 060 */ mutableStateArray_0[0] = value_2; /* 061 */ } /* 062 */ /* 063 */ boolean isNull_1 = resultIsNull_0; /* 064 */ scala.collection.Seq value_1 = null; /* 065 */ if (!resultIsNull_0) { /* 066 */ value_1 = scala.collection.mutable.WrappedArray.make(mutableStateArray_0[0]); /* 067 */ } /* 068 */ if (isNull_1) { /* 069 */ values_0[0] = null; /* 070 */ } else { /* 071 */ values_0[0] = value_1; /* 072 */ } /* 073 */ /* 074 */ final org.apache.spark.sql.Row value_0 = new org.apache.spark.sql.catalyst.expressions.GenericRowWithSchema(values_0, ((org.apache.spark.sql.types.StructType) references[0] /* schema */)); /* 075 */ /* 076 */ return value_0; /* 077 */ } /* 078 */ /* 079 */ /* 080 */ private ArrayData MapObjects_0(InternalRow i) { /* 081 */ boolean isNull_4 = i.isNullAt(0); /* 082 */ ArrayData value_4 = isNull_4 ? /* 083 */ null : (i.getArray(0)); /* 084 */ ArrayData value_3 = null; /* 085 */ /* 086 */ if (!isNull_4) { /* 087 */ /* 088 */ int dataLength_0 = value_4.numElements(); /* 089 */ /* 090 */ java.lang.String[] convertedArray_0 = null; /* 091 */ convertedArray_0 = new java.lang.String[dataLength_0]; /* 092 */ /* 093 */ /* 094 */ int loopIndex_0 = 0; /* 095 */ /* 096 */ while (loopIndex_0 < dataLength_0) { /* 097 */ value_MapObject_lambda_variable_1 = (UTF8String) (value_4.getUTF8String(loopIndex_0)); /* 098 */ isNull_MapObject_lambda_variable_1 = value_4.isNullAt(loopIndex_0); /* 099 */ /* 100 */ boolean isNull_5 = true; /* 101 */ java.lang.String value_5 = null; /* 102 */ if (!isNull_MapObject_lambda_variable_1) { /* 103 */ /* 104 */ isNull_5 = false; /* 105 */ if (!isNull_5) { /* 106 */ /* 107 */ Object funcResult_0 = null; /* 108 */ funcResult_0 = value_MapObject_lambda_variable_1.toString(); /* 109 */ value_5 = (java.lang.String) funcResult_0; /* 110 */ /* 111 */ } /* 112 */ } /* 113 */ if (isNull_5) { /* 114 */ convertedArray_0[loopIndex_0] = null; /* 115 */ } else { /* 116 */ convertedArray_0[loopIndex_0] = value_5; /* 117 */ } /* 118 */ /* 119 */ loopIndex_0 += 1; /* 120 */ } /* 121 */ /* 122 */ value_3 = new org.apache.spark.sql.catalyst.util.GenericArrayData(convertedArray_0); /* 123 */ } /* 124 */ globalIsNull_0 = isNull_4; /* 125 */ return value_3; /* 126 */ } /* 127 */ /* 128 */ } 21/01/17 17:43:08.627 Executor task launch worker for task 1.0 in stage 2.0 (TID 9) DEBUG GenerateSafeProjection: code for createexternalrow(staticinvoke(class scala.collection.mutable.WrappedArray$, ObjectType(interface scala.collection.Seq), make, mapobjects(lambdavariable(MapObject, StringType, true, -1), lambdavariable(MapObject, StringType, true, -1).toString, input[0, array, true], None).array, true, false), StructField(words,ArrayType(StringType,true),true)): /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificSafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificSafeProjection extends org.apache.spark.sql.catalyst.expressions.codegen.BaseProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private InternalRow mutableRow; /* 009 */ private boolean resultIsNull_0; /* 010 */ private UTF8String value_MapObject_lambda_variable_1; /* 011 */ private boolean isNull_MapObject_lambda_variable_1; /* 012 */ private boolean globalIsNull_0; /* 013 */ private java.lang.Object[] mutableStateArray_0 = new java.lang.Object[1]; /* 014 */ /* 015 */ public SpecificSafeProjection(Object[] references) { /* 016 */ this.references = references; /* 017 */ mutableRow = (InternalRow) references[references.length - 1]; /* 018 */ /* 019 */ /* 020 */ } /* 021 */ /* 022 */ public void initialize(int partitionIndex) { /* 023 */ /* 024 */ } /* 025 */ /* 026 */ public java.lang.Object apply(java.lang.Object _i) { /* 027 */ InternalRow i = (InternalRow) _i; /* 028 */ org.apache.spark.sql.Row value_7 = CreateExternalRow_0(i); /* 029 */ if (false) { /* 030 */ mutableRow.setNullAt(0); /* 031 */ } else { /* 032 */ /* 033 */ mutableRow.update(0, value_7); /* 034 */ } /* 035 */ /* 036 */ return mutableRow; /* 037 */ } /* 038 */ /* 039 */ /* 040 */ private org.apache.spark.sql.Row CreateExternalRow_0(InternalRow i) { /* 041 */ Object[] values_0 = new Object[1]; /* 042 */ /* 043 */ resultIsNull_0 = false; /* 044 */ if (!resultIsNull_0) { /* 045 */ ArrayData value_6 = MapObjects_0(i); /* 046 */ boolean isNull_2 = true; /* 047 */ java.lang.Object value_2 = null; /* 048 */ if (!globalIsNull_0) { /* 049 */ /* 050 */ isNull_2 = false; /* 051 */ if (!isNull_2) { /* 052 */ /* 053 */ Object funcResult_1 = null; /* 054 */ funcResult_1 = value_6.array(); /* 055 */ value_2 = (java.lang.Object) funcResult_1; /* 056 */ /* 057 */ } /* 058 */ } /* 059 */ resultIsNull_0 = isNull_2; /* 060 */ mutableStateArray_0[0] = value_2; /* 061 */ } /* 062 */ /* 063 */ boolean isNull_1 = resultIsNull_0; /* 064 */ scala.collection.Seq value_1 = null; /* 065 */ if (!resultIsNull_0) { /* 066 */ value_1 = scala.collection.mutable.WrappedArray.make(mutableStateArray_0[0]); /* 067 */ } /* 068 */ if (isNull_1) { /* 069 */ values_0[0] = null; /* 070 */ } else { /* 071 */ values_0[0] = value_1; /* 072 */ } /* 073 */ /* 074 */ final org.apache.spark.sql.Row value_0 = new org.apache.spark.sql.catalyst.expressions.GenericRowWithSchema(values_0, ((org.apache.spark.sql.types.StructType) references[0] /* schema */)); /* 075 */ /* 076 */ return value_0; /* 077 */ } /* 078 */ /* 079 */ /* 080 */ private ArrayData MapObjects_0(InternalRow i) { /* 081 */ boolean isNull_4 = i.isNullAt(0); /* 082 */ ArrayData value_4 = isNull_4 ? /* 083 */ null : (i.getArray(0)); /* 084 */ ArrayData value_3 = null; /* 085 */ /* 086 */ if (!isNull_4) { /* 087 */ /* 088 */ int dataLength_0 = value_4.numElements(); /* 089 */ /* 090 */ java.lang.String[] convertedArray_0 = null; /* 091 */ convertedArray_0 = new java.lang.String[dataLength_0]; /* 092 */ /* 093 */ /* 094 */ int loopIndex_0 = 0; /* 095 */ /* 096 */ while (loopIndex_0 < dataLength_0) { /* 097 */ value_MapObject_lambda_variable_1 = (UTF8String) (value_4.getUTF8String(loopIndex_0)); /* 098 */ isNull_MapObject_lambda_variable_1 = value_4.isNullAt(loopIndex_0); /* 099 */ /* 100 */ boolean isNull_5 = true; /* 101 */ java.lang.String value_5 = null; /* 102 */ if (!isNull_MapObject_lambda_variable_1) { /* 103 */ /* 104 */ isNull_5 = false; /* 105 */ if (!isNull_5) { /* 106 */ /* 107 */ Object funcResult_0 = null; /* 108 */ funcResult_0 = value_MapObject_lambda_variable_1.toString(); /* 109 */ value_5 = (java.lang.String) funcResult_0; /* 110 */ /* 111 */ } /* 112 */ } /* 113 */ if (isNull_5) { /* 114 */ convertedArray_0[loopIndex_0] = null; /* 115 */ } else { /* 116 */ convertedArray_0[loopIndex_0] = value_5; /* 117 */ } /* 118 */ /* 119 */ loopIndex_0 += 1; /* 120 */ } /* 121 */ /* 122 */ value_3 = new org.apache.spark.sql.catalyst.util.GenericArrayData(convertedArray_0); /* 123 */ } /* 124 */ globalIsNull_0 = isNull_4; /* 125 */ return value_3; /* 126 */ } /* 127 */ /* 128 */ } 21/01/17 17:43:08.628 Executor task launch worker for task 2.0 in stage 2.0 (TID 10) DEBUG CodeGenerator: /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificSafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificSafeProjection extends org.apache.spark.sql.catalyst.expressions.codegen.BaseProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private InternalRow mutableRow; /* 009 */ private boolean resultIsNull_0; /* 010 */ private UTF8String value_MapObject_lambda_variable_1; /* 011 */ private boolean isNull_MapObject_lambda_variable_1; /* 012 */ private boolean globalIsNull_0; /* 013 */ private java.lang.Object[] mutableStateArray_0 = new java.lang.Object[1]; /* 014 */ /* 015 */ public SpecificSafeProjection(Object[] references) { /* 016 */ this.references = references; /* 017 */ mutableRow = (InternalRow) references[references.length - 1]; /* 018 */ /* 019 */ /* 020 */ } /* 021 */ /* 022 */ public void initialize(int partitionIndex) { /* 023 */ /* 024 */ } /* 025 */ /* 026 */ public java.lang.Object apply(java.lang.Object _i) { /* 027 */ InternalRow i = (InternalRow) _i; /* 028 */ org.apache.spark.sql.Row value_7 = CreateExternalRow_0(i); /* 029 */ if (false) { /* 030 */ mutableRow.setNullAt(0); /* 031 */ } else { /* 032 */ /* 033 */ mutableRow.update(0, value_7); /* 034 */ } /* 035 */ /* 036 */ return mutableRow; /* 037 */ } /* 038 */ /* 039 */ /* 040 */ private org.apache.spark.sql.Row CreateExternalRow_0(InternalRow i) { /* 041 */ Object[] values_0 = new Object[1]; /* 042 */ /* 043 */ resultIsNull_0 = false; /* 044 */ if (!resultIsNull_0) { /* 045 */ ArrayData value_6 = MapObjects_0(i); /* 046 */ boolean isNull_2 = true; /* 047 */ java.lang.Object value_2 = null; /* 048 */ if (!globalIsNull_0) { /* 049 */ /* 050 */ isNull_2 = false; /* 051 */ if (!isNull_2) { /* 052 */ /* 053 */ Object funcResult_1 = null; /* 054 */ funcResult_1 = value_6.array(); /* 055 */ value_2 = (java.lang.Object) funcResult_1; /* 056 */ /* 057 */ } /* 058 */ } /* 059 */ resultIsNull_0 = isNull_2; /* 060 */ mutableStateArray_0[0] = value_2; /* 061 */ } /* 062 */ /* 063 */ boolean isNull_1 = resultIsNull_0; /* 064 */ scala.collection.Seq value_1 = null; /* 065 */ if (!resultIsNull_0) { /* 066 */ value_1 = scala.collection.mutable.WrappedArray.make(mutableStateArray_0[0]); /* 067 */ } /* 068 */ if (isNull_1) { /* 069 */ values_0[0] = null; /* 070 */ } else { /* 071 */ values_0[0] = value_1; /* 072 */ } /* 073 */ /* 074 */ final org.apache.spark.sql.Row value_0 = new org.apache.spark.sql.catalyst.expressions.GenericRowWithSchema(values_0, ((org.apache.spark.sql.types.StructType) references[0] /* schema */)); /* 075 */ /* 076 */ return value_0; /* 077 */ } /* 078 */ /* 079 */ /* 080 */ private ArrayData MapObjects_0(InternalRow i) { /* 081 */ boolean isNull_4 = i.isNullAt(0); /* 082 */ ArrayData value_4 = isNull_4 ? /* 083 */ null : (i.getArray(0)); /* 084 */ ArrayData value_3 = null; /* 085 */ /* 086 */ if (!isNull_4) { /* 087 */ /* 088 */ int dataLength_0 = value_4.numElements(); /* 089 */ /* 090 */ java.lang.String[] convertedArray_0 = null; /* 091 */ convertedArray_0 = new java.lang.String[dataLength_0]; /* 092 */ /* 093 */ /* 094 */ int loopIndex_0 = 0; /* 095 */ /* 096 */ while (loopIndex_0 < dataLength_0) { /* 097 */ value_MapObject_lambda_variable_1 = (UTF8String) (value_4.getUTF8String(loopIndex_0)); /* 098 */ isNull_MapObject_lambda_variable_1 = value_4.isNullAt(loopIndex_0); /* 099 */ /* 100 */ boolean isNull_5 = true; /* 101 */ java.lang.String value_5 = null; /* 102 */ if (!isNull_MapObject_lambda_variable_1) { /* 103 */ /* 104 */ isNull_5 = false; /* 105 */ if (!isNull_5) { /* 106 */ /* 107 */ Object funcResult_0 = null; /* 108 */ funcResult_0 = value_MapObject_lambda_variable_1.toString(); /* 109 */ value_5 = (java.lang.String) funcResult_0; /* 110 */ /* 111 */ } /* 112 */ } /* 113 */ if (isNull_5) { /* 114 */ convertedArray_0[loopIndex_0] = null; /* 115 */ } else { /* 116 */ convertedArray_0[loopIndex_0] = value_5; /* 117 */ } /* 118 */ /* 119 */ loopIndex_0 += 1; /* 120 */ } /* 121 */ /* 122 */ value_3 = new org.apache.spark.sql.catalyst.util.GenericArrayData(convertedArray_0); /* 123 */ } /* 124 */ globalIsNull_0 = isNull_4; /* 125 */ return value_3; /* 126 */ } /* 127 */ /* 128 */ } 21/01/17 17:43:08.627 Executor task launch worker for task 3.0 in stage 2.0 (TID 11) DEBUG GenerateSafeProjection: code for createexternalrow(staticinvoke(class scala.collection.mutable.WrappedArray$, ObjectType(interface scala.collection.Seq), make, mapobjects(lambdavariable(MapObject, StringType, true, -1), lambdavariable(MapObject, StringType, true, -1).toString, input[0, array, true], None).array, true, false), StructField(words,ArrayType(StringType,true),true)): /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificSafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificSafeProjection extends org.apache.spark.sql.catalyst.expressions.codegen.BaseProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private InternalRow mutableRow; /* 009 */ private boolean resultIsNull_0; /* 010 */ private UTF8String value_MapObject_lambda_variable_1; /* 011 */ private boolean isNull_MapObject_lambda_variable_1; /* 012 */ private boolean globalIsNull_0; /* 013 */ private java.lang.Object[] mutableStateArray_0 = new java.lang.Object[1]; /* 014 */ /* 015 */ public SpecificSafeProjection(Object[] references) { /* 016 */ this.references = references; /* 017 */ mutableRow = (InternalRow) references[references.length - 1]; /* 018 */ /* 019 */ /* 020 */ } /* 021 */ /* 022 */ public void initialize(int partitionIndex) { /* 023 */ /* 024 */ } /* 025 */ /* 026 */ public java.lang.Object apply(java.lang.Object _i) { /* 027 */ InternalRow i = (InternalRow) _i; /* 028 */ org.apache.spark.sql.Row value_7 = CreateExternalRow_0(i); /* 029 */ if (false) { /* 030 */ mutableRow.setNullAt(0); /* 031 */ } else { /* 032 */ /* 033 */ mutableRow.update(0, value_7); /* 034 */ } /* 035 */ /* 036 */ return mutableRow; /* 037 */ } /* 038 */ /* 039 */ /* 040 */ private org.apache.spark.sql.Row CreateExternalRow_0(InternalRow i) { /* 041 */ Object[] values_0 = new Object[1]; /* 042 */ /* 043 */ resultIsNull_0 = false; /* 044 */ if (!resultIsNull_0) { /* 045 */ ArrayData value_6 = MapObjects_0(i); /* 046 */ boolean isNull_2 = true; /* 047 */ java.lang.Object value_2 = null; /* 048 */ if (!globalIsNull_0) { /* 049 */ /* 050 */ isNull_2 = false; /* 051 */ if (!isNull_2) { /* 052 */ /* 053 */ Object funcResult_1 = null; /* 054 */ funcResult_1 = value_6.array(); /* 055 */ value_2 = (java.lang.Object) funcResult_1; /* 056 */ /* 057 */ } /* 058 */ } /* 059 */ resultIsNull_0 = isNull_2; /* 060 */ mutableStateArray_0[0] = value_2; /* 061 */ } /* 062 */ /* 063 */ boolean isNull_1 = resultIsNull_0; /* 064 */ scala.collection.Seq value_1 = null; /* 065 */ if (!resultIsNull_0) { /* 066 */ value_1 = scala.collection.mutable.WrappedArray.make(mutableStateArray_0[0]); /* 067 */ } /* 068 */ if (isNull_1) { /* 069 */ values_0[0] = null; /* 070 */ } else { /* 071 */ values_0[0] = value_1; /* 072 */ } /* 073 */ /* 074 */ final org.apache.spark.sql.Row value_0 = new org.apache.spark.sql.catalyst.expressions.GenericRowWithSchema(values_0, ((org.apache.spark.sql.types.StructType) references[0] /* schema */)); /* 075 */ /* 076 */ return value_0; /* 077 */ } /* 078 */ /* 079 */ /* 080 */ private ArrayData MapObjects_0(InternalRow i) { /* 081 */ boolean isNull_4 = i.isNullAt(0); /* 082 */ ArrayData value_4 = isNull_4 ? /* 083 */ null : (i.getArray(0)); /* 084 */ ArrayData value_3 = null; /* 085 */ /* 086 */ if (!isNull_4) { /* 087 */ /* 088 */ int dataLength_0 = value_4.numElements(); /* 089 */ /* 090 */ java.lang.String[] convertedArray_0 = null; /* 091 */ convertedArray_0 = new java.lang.String[dataLength_0]; /* 092 */ /* 093 */ /* 094 */ int loopIndex_0 = 0; /* 095 */ /* 096 */ while (loopIndex_0 < dataLength_0) { /* 097 */ value_MapObject_lambda_variable_1 = (UTF8String) (value_4.getUTF8String(loopIndex_0)); /* 098 */ isNull_MapObject_lambda_variable_1 = value_4.isNullAt(loopIndex_0); /* 099 */ /* 100 */ boolean isNull_5 = true; /* 101 */ java.lang.String value_5 = null; /* 102 */ if (!isNull_MapObject_lambda_variable_1) { /* 103 */ /* 104 */ isNull_5 = false; /* 105 */ if (!isNull_5) { /* 106 */ /* 107 */ Object funcResult_0 = null; /* 108 */ funcResult_0 = value_MapObject_lambda_variable_1.toString(); /* 109 */ value_5 = (java.lang.String) funcResult_0; /* 110 */ /* 111 */ } /* 112 */ } /* 113 */ if (isNull_5) { /* 114 */ convertedArray_0[loopIndex_0] = null; /* 115 */ } else { /* 116 */ convertedArray_0[loopIndex_0] = value_5; /* 117 */ } /* 118 */ /* 119 */ loopIndex_0 += 1; /* 120 */ } /* 121 */ /* 122 */ value_3 = new org.apache.spark.sql.catalyst.util.GenericArrayData(convertedArray_0); /* 123 */ } /* 124 */ globalIsNull_0 = isNull_4; /* 125 */ return value_3; /* 126 */ } /* 127 */ /* 128 */ } 21/01/17 17:43:08.660 Thread-3 INFO MapPartitionsRDD: Removing RDD 20 from persistence list 21/01/17 17:43:08.660 Thread-3 DEBUG InsertAdaptiveSparkPlan: Adaptive execution enabled for plan: HashAggregate(keys=[], functions=[sum(if (isnull(prediction#1)) null else (UDF(normalizedFeatures#11, knownnotnull(prediction#1), 1.0) * 1.0)), sum(1.0)], output=[(sum((UDF(normalizedFeatures, cast(prediction as double), weightCol) * weightCol)) / sum(weightCol))#36]) +- HashAggregate(keys=[], functions=[partial_sum(if (isnull(prediction#1)) null else (UDF(normalizedFeatures#11, knownnotnull(prediction#1), 1.0) * 1.0)), partial_sum(1.0)], output=[sum#58, sum#59]) +- Project [prediction#1, UDF(features#0) AS normalizedFeatures#11] +- Scan ExistingRDD[features#0,prediction#1] 21/01/17 17:43:08.663 block-manager-storage-async-thread-pool-0 DEBUG BlockManagerStorageEndpoint: removing RDD 19 21/01/17 17:43:08.664 block-manager-storage-async-thread-pool-0 INFO BlockManager: Removing RDD 19 21/01/17 17:43:08.666 block-manager-storage-async-thread-pool-0 DEBUG BlockManager: Removing block rdd_19_7 21/01/17 17:43:08.666 Thread-3 INFO MapPartitionsRDD: Removing RDD 25 from persistence list 21/01/17 17:43:08.666 block-manager-storage-async-thread-pool-0 DEBUG MemoryStore: Block rdd_19_7 of size 16 dropped from memory (free 383621980) 21/01/17 17:43:08.667 block-manager-storage-async-thread-pool-0 DEBUG BlockManager: Removing block rdd_19_6 21/01/17 17:43:08.667 block-manager-storage-async-thread-pool-0 DEBUG MemoryStore: Block rdd_19_6 of size 16 dropped from memory (free 383621996) 21/01/17 17:43:08.668 block-manager-storage-async-thread-pool-0 DEBUG BlockManager: Removing block rdd_19_0 21/01/17 17:43:08.668 block-manager-storage-async-thread-pool-0 DEBUG MemoryStore: Block rdd_19_0 of size 192 dropped from memory (free 383622188) 21/01/17 17:43:08.668 Thread-3 INFO MapPartitionsRDD: Removing RDD 16 from persistence list 21/01/17 17:43:08.668 block-manager-storage-async-thread-pool-0 DEBUG BlockManager: Removing block rdd_19_9 21/01/17 17:43:08.668 block-manager-storage-async-thread-pool-0 DEBUG MemoryStore: Block rdd_19_9 of size 16 dropped from memory (free 383622204) 21/01/17 17:43:08.668 block-manager-storage-async-thread-pool-0 DEBUG BlockManager: Removing block rdd_19_4 21/01/17 17:43:08.668 block-manager-storage-async-thread-pool-0 DEBUG MemoryStore: Block rdd_19_4 of size 16 dropped from memory (free 383622220) 21/01/17 17:43:08.668 block-manager-storage-async-thread-pool-0 DEBUG BlockManager: Removing block rdd_19_5 21/01/17 17:43:08.668 block-manager-storage-async-thread-pool-0 DEBUG MemoryStore: Block rdd_19_5 of size 16 dropped from memory (free 383622236) 21/01/17 17:43:08.668 block-manager-storage-async-thread-pool-0 DEBUG BlockManager: Removing block rdd_19_1 21/01/17 17:43:08.668 block-manager-storage-async-thread-pool-0 DEBUG MemoryStore: Block rdd_19_1 of size 192 dropped from memory (free 383622428) 21/01/17 17:43:08.669 block-manager-storage-async-thread-pool-0 DEBUG BlockManager: Removing block rdd_19_2 21/01/17 17:43:08.669 block-manager-storage-async-thread-pool-0 DEBUG MemoryStore: Block rdd_19_2 of size 192 dropped from memory (free 383622620) 21/01/17 17:43:08.669 block-manager-storage-async-thread-pool-0 DEBUG BlockManager: Removing block rdd_19_8 21/01/17 17:43:08.669 Thread-3 DEBUG CoGroupedRDD: Adding one-to-one dependency with MapPartitionsRDD[51] at mapValues at ALS.scala:1065 21/01/17 17:43:08.669 block-manager-storage-async-thread-pool-0 DEBUG MemoryStore: Block rdd_19_8 of size 16 dropped from memory (free 383622636) 21/01/17 17:43:08.669 Thread-3 DEBUG CoGroupedRDD: Adding one-to-one dependency with MapPartitionsRDD[36] at mapValues at ALS.scala:1715 21/01/17 17:43:08.669 block-manager-storage-async-thread-pool-0 DEBUG BlockManager: Removing block rdd_19_3 21/01/17 17:43:08.669 block-manager-storage-async-thread-pool-0 DEBUG MemoryStore: Block rdd_19_3 of size 16 dropped from memory (free 383622652) 21/01/17 17:43:08.669 Thread-3 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$count$1$adapted 21/01/17 17:43:08.672 Thread-3 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$count$1$adapted) is now cleaned +++ 21/01/17 17:43:08.672 Thread-3 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$runJob$5 21/01/17 17:43:08.675 Thread-3 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$runJob$5) is now cleaned +++ 21/01/17 17:43:08.675 Thread-3 INFO SparkContext: Starting job: count at ALS.scala:1080 21/01/17 17:43:08.675 dag-scheduler-event-loop DEBUG DAGScheduler: Merging stage rdd profiles: Set() 21/01/17 17:43:08.676 dag-scheduler-event-loop DEBUG DAGScheduler: Merging stage rdd profiles: Set() 21/01/17 17:43:08.676 dag-scheduler-event-loop DEBUG DAGScheduler: Merging stage rdd profiles: Set() 21/01/17 17:43:08.677 dag-scheduler-event-loop DEBUG DAGScheduler: Merging stage rdd profiles: Set() 21/01/17 17:43:08.677 dag-scheduler-event-loop DEBUG DAGScheduler: Merging stage rdd profiles: Set() 21/01/17 17:43:08.678 dag-scheduler-event-loop INFO DAGScheduler: Got job 4 (count at ALS.scala:1080) with 10 output partitions 21/01/17 17:43:08.678 dag-scheduler-event-loop INFO DAGScheduler: Final stage: ResultStage 17 (count at ALS.scala:1080) 21/01/17 17:43:08.678 dag-scheduler-event-loop INFO DAGScheduler: Parents of final stage: List(ShuffleMapStage 15, ShuffleMapStage 16) 21/01/17 17:43:08.681 dag-scheduler-event-loop INFO DAGScheduler: Missing parents: List() 21/01/17 17:43:08.681 dag-scheduler-event-loop DEBUG DAGScheduler: submitStage(ResultStage 17 (name=count at ALS.scala:1080;jobs=4)) 21/01/17 17:43:08.681 dag-scheduler-event-loop DEBUG DAGScheduler: missing: List() 21/01/17 17:43:08.681 dag-scheduler-event-loop INFO DAGScheduler: Submitting ResultStage 17 (itemFactors MapPartitionsRDD[55] at mapPartitions at ALS.scala:1067), which has no missing parents 21/01/17 17:43:08.681 dag-scheduler-event-loop DEBUG DAGScheduler: submitMissingTasks(ResultStage 17) 21/01/17 17:43:08.712 Executor task launch worker for task 2.0 in stage 2.0 (TID 10) INFO CodeGenerator: Code generated in 77.723776 ms 21/01/17 17:43:08.720 block-manager-storage-async-thread-pool-2 DEBUG BlockManagerStorageEndpoint: removing RDD 25 21/01/17 17:43:08.720 block-manager-storage-async-thread-pool-2 INFO BlockManager: Removing RDD 25 21/01/17 17:43:08.720 block-manager-storage-async-thread-pool-2 DEBUG BlockManager: Removing block rdd_25_8 21/01/17 17:43:08.721 block-manager-storage-async-thread-pool-2 DEBUG MemoryStore: Block rdd_25_8 of size 16 dropped from memory (free 383622668) 21/01/17 17:43:08.721 block-manager-storage-async-thread-pool-2 DEBUG BlockManager: Removing block rdd_25_1 21/01/17 17:43:08.721 block-manager-storage-async-thread-pool-2 DEBUG MemoryStore: Block rdd_25_1 of size 304 dropped from memory (free 383622972) 21/01/17 17:43:08.721 block-manager-storage-async-thread-pool-2 DEBUG BlockManager: Removing block rdd_25_9 21/01/17 17:43:08.721 block-manager-storage-async-thread-pool-2 DEBUG MemoryStore: Block rdd_25_9 of size 16 dropped from memory (free 383622988) 21/01/17 17:43:08.721 block-manager-storage-async-thread-pool-2 DEBUG BlockManager: Removing block rdd_25_4 21/01/17 17:43:08.721 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(182) 21/01/17 17:43:08.722 block-manager-storage-async-thread-pool-2 DEBUG MemoryStore: Block rdd_25_4 of size 16 dropped from memory (free 383623004) 21/01/17 17:43:08.722 block-manager-storage-async-thread-pool-2 DEBUG BlockManager: Removing block rdd_25_0 21/01/17 17:43:08.722 block-manager-storage-async-thread-pool-2 DEBUG MemoryStore: Block rdd_25_0 of size 288 dropped from memory (free 383623292) 21/01/17 17:43:08.722 block-manager-storage-async-thread-pool-1 DEBUG BlockManagerStorageEndpoint: removing RDD 20 21/01/17 17:43:08.722 block-manager-storage-async-thread-pool-1 INFO BlockManager: Removing RDD 20 21/01/17 17:43:08.722 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 182 21/01/17 17:43:08.722 block-manager-storage-async-thread-pool-1 DEBUG BlockManager: Removing block rdd_20_6 21/01/17 17:43:08.722 block-manager-storage-async-thread-pool-1 DEBUG MemoryStore: Block rdd_20_6 of size 16 dropped from memory (free 383623308) 21/01/17 17:43:08.722 block-manager-storage-async-thread-pool-1 DEBUG BlockManager: Removing block rdd_20_9 21/01/17 17:43:08.722 block-manager-storage-async-thread-pool-1 DEBUG MemoryStore: Block rdd_20_9 of size 16 dropped from memory (free 383623324) 21/01/17 17:43:08.722 block-manager-storage-async-thread-pool-1 DEBUG BlockManager: Removing block rdd_20_5 21/01/17 17:43:08.722 block-manager-storage-async-thread-pool-1 DEBUG MemoryStore: Block rdd_20_5 of size 16 dropped from memory (free 383623340) 21/01/17 17:43:08.722 block-manager-storage-async-thread-pool-1 DEBUG BlockManager: Removing block rdd_20_3 21/01/17 17:43:08.722 block-manager-storage-async-thread-pool-1 DEBUG MemoryStore: Block rdd_20_3 of size 16 dropped from memory (free 383623356) 21/01/17 17:43:08.723 block-manager-storage-async-thread-pool-1 DEBUG BlockManager: Removing block rdd_20_2 21/01/17 17:43:08.723 block-manager-storage-async-thread-pool-5 DEBUG BlockManagerStorageEndpoint: Done removing RDD 19, response is 0 21/01/17 17:43:08.720 block-manager-storage-async-thread-pool-3 DEBUG BlockManagerStorageEndpoint: removing RDD 16 21/01/17 17:43:08.723 block-manager-storage-async-thread-pool-3 INFO BlockManager: Removing RDD 16 21/01/17 17:43:08.723 block-manager-storage-async-thread-pool-1 DEBUG MemoryStore: Block rdd_20_2 of size 296 dropped from memory (free 383623652) 21/01/17 17:43:08.723 block-manager-storage-async-thread-pool-1 DEBUG BlockManager: Removing block rdd_20_7 21/01/17 17:43:08.723 block-manager-storage-async-thread-pool-1 DEBUG MemoryStore: Block rdd_20_7 of size 16 dropped from memory (free 383623668) 21/01/17 17:43:08.723 block-manager-storage-async-thread-pool-1 DEBUG BlockManager: Removing block rdd_20_1 21/01/17 17:43:08.723 block-manager-storage-async-thread-pool-1 DEBUG MemoryStore: Block rdd_20_1 of size 296 dropped from memory (free 383623964) 21/01/17 17:43:08.723 block-manager-storage-async-thread-pool-1 DEBUG BlockManager: Removing block rdd_20_0 21/01/17 17:43:08.723 block-manager-storage-async-thread-pool-1 DEBUG MemoryStore: Block rdd_20_0 of size 296 dropped from memory (free 383624260) 21/01/17 17:43:08.723 block-manager-storage-async-thread-pool-1 DEBUG BlockManager: Removing block rdd_20_4 21/01/17 17:43:08.723 block-manager-storage-async-thread-pool-1 DEBUG MemoryStore: Block rdd_20_4 of size 16 dropped from memory (free 383624276) 21/01/17 17:43:08.723 block-manager-storage-async-thread-pool-1 DEBUG BlockManager: Removing block rdd_20_8 21/01/17 17:43:08.723 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 182 21/01/17 17:43:08.724 block-manager-storage-async-thread-pool-1 DEBUG MemoryStore: Block rdd_20_8 of size 16 dropped from memory (free 383624292) 21/01/17 17:43:08.724 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(151) 21/01/17 17:43:08.724 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 151 21/01/17 17:43:08.724 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 151 21/01/17 17:43:08.724 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(193) 21/01/17 17:43:08.724 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 193 21/01/17 17:43:08.724 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 193 21/01/17 17:43:08.724 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(74) 21/01/17 17:43:08.724 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 74 21/01/17 17:43:08.724 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 74 21/01/17 17:43:08.724 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(109) 21/01/17 17:43:08.724 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 109 21/01/17 17:43:08.724 block-manager-storage-async-thread-pool-3 DEBUG BlockManager: Removing block rdd_16_0 21/01/17 17:43:08.724 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 109 21/01/17 17:43:08.724 block-manager-storage-async-thread-pool-3 DEBUG MemoryStore: Block rdd_16_0 of size 176 dropped from memory (free 383624468) 21/01/17 17:43:08.724 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanBroadcast(5) 21/01/17 17:43:08.724 block-manager-storage-async-thread-pool-3 DEBUG BlockManager: Removing block rdd_16_2 21/01/17 17:43:08.724 block-manager-storage-async-thread-pool-3 DEBUG MemoryStore: Block rdd_16_2 of size 328 dropped from memory (free 383624796) 21/01/17 17:43:08.724 block-manager-storage-async-thread-pool-3 DEBUG BlockManager: Removing block rdd_16_3 21/01/17 17:43:08.724 block-manager-storage-async-thread-pool-3 DEBUG MemoryStore: Block rdd_16_3 of size 328 dropped from memory (free 383625124) 21/01/17 17:43:08.724 Spark Context Cleaner DEBUG ContextCleaner: Cleaning broadcast 5 21/01/17 17:43:08.724 block-manager-storage-async-thread-pool-3 DEBUG BlockManager: Removing block rdd_16_1 21/01/17 17:43:08.724 block-manager-storage-async-thread-pool-3 DEBUG MemoryStore: Block rdd_16_1 of size 176 dropped from memory (free 383625300) 21/01/17 17:43:08.725 block-manager-storage-async-thread-pool-2 DEBUG BlockManager: Removing block rdd_25_5 21/01/17 17:43:08.725 block-manager-storage-async-thread-pool-2 DEBUG MemoryStore: Block rdd_25_5 of size 16 dropped from memory (free 383625316) 21/01/17 17:43:08.725 Spark Context Cleaner DEBUG TorrentBroadcast: Unpersisting TorrentBroadcast 5 21/01/17 17:43:08.725 block-manager-storage-async-thread-pool-2 DEBUG BlockManager: Removing block rdd_25_3 21/01/17 17:43:08.725 block-manager-storage-async-thread-pool-2 DEBUG MemoryStore: Block rdd_25_3 of size 16 dropped from memory (free 383625332) 21/01/17 17:43:08.725 block-manager-storage-async-thread-pool-2 DEBUG BlockManager: Removing block rdd_25_7 21/01/17 17:43:08.727 block-manager-storage-async-thread-pool-2 DEBUG MemoryStore: Block rdd_25_7 of size 16 dropped from memory (free 383625348) 21/01/17 17:43:08.727 block-manager-storage-async-thread-pool-2 DEBUG BlockManager: Removing block rdd_25_6 21/01/17 17:43:08.727 block-manager-storage-async-thread-pool-2 DEBUG MemoryStore: Block rdd_25_6 of size 16 dropped from memory (free 383625364) 21/01/17 17:43:08.727 block-manager-storage-async-thread-pool-2 DEBUG BlockManager: Removing block rdd_25_2 21/01/17 17:43:08.727 block-manager-storage-async-thread-pool-2 DEBUG MemoryStore: Block rdd_25_2 of size 296 dropped from memory (free 383625660) 21/01/17 17:43:08.729 block-manager-storage-async-thread-pool-6 DEBUG BlockManagerStorageEndpoint: removing broadcast 5 21/01/17 17:43:08.730 block-manager-storage-async-thread-pool-6 DEBUG BlockManager: Removing broadcast 5 21/01/17 17:43:08.733 block-manager-storage-async-thread-pool-6 DEBUG BlockManager: Removing block broadcast_5 21/01/17 17:43:08.733 block-manager-storage-async-thread-pool-6 DEBUG MemoryStore: Block broadcast_5 of size 35640 dropped from memory (free 383661300) 21/01/17 17:43:08.733 block-manager-storage-async-thread-pool-6 DEBUG BlockManager: Removing block broadcast_5_piece0 21/01/17 17:43:08.740 block-manager-storage-async-thread-pool-5 DEBUG BlockManagerStorageEndpoint: Sent response: 0 to 192.168.1.225:41897 21/01/17 17:43:08.754 Executor task launch worker for task 2.0 in stage 2.0 (TID 10) INFO PythonRunner: Times: total = 59, boot = -1002, init = 1061, finish = 0 21/01/17 17:43:08.760 block-manager-storage-async-thread-pool-6 DEBUG MemoryStore: Block broadcast_5_piece0 of size 16613 dropped from memory (free 383677913) 21/01/17 17:43:08.754 Executor task launch worker for task 0.0 in stage 2.0 (TID 8) INFO PythonRunner: Times: total = 56, boot = -1005, init = 1061, finish = 0 21/01/17 17:43:08.760 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master broadcast_5_piece0 for BlockManagerId(driver, 192.168.1.225, 40985, None) 21/01/17 17:43:08.756 Executor task launch worker for task 1.0 in stage 2.0 (TID 9) INFO PythonRunner: Times: total = 77, boot = -1043, init = 1120, finish = 0 21/01/17 17:43:08.756 Executor task launch worker for task 3.0 in stage 2.0 (TID 11) INFO PythonRunner: Times: total = 89, boot = -1036, init = 1125, finish = 0 21/01/17 17:43:08.761 block-manager-storage-async-thread-pool-10 DEBUG BlockManagerStorageEndpoint: Done removing RDD 20, response is 0 21/01/17 17:43:08.762 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Removed broadcast_5_piece0 on 192.168.1.225:40985 in memory (size: 16.2 KiB, free: 366.2 MiB) 21/01/17 17:43:08.779 block-manager-storage-async-thread-pool-6 DEBUG BlockManagerMaster: Updated info of block broadcast_5_piece0 21/01/17 17:43:08.779 block-manager-storage-async-thread-pool-6 DEBUG BlockManager: Told master about block broadcast_5_piece0 21/01/17 17:43:08.784 block-manager-storage-async-thread-pool-12 DEBUG BlockManagerStorageEndpoint: Done removing RDD 16, response is 0 21/01/17 17:43:08.785 Executor task launch worker for task 1.0 in stage 2.0 (TID 9) DEBUG LocalDiskShuffleMapOutputWriter: Writing shuffle index file for mapId 9 with length 4 21/01/17 17:43:08.786 block-manager-storage-async-thread-pool-11 DEBUG BlockManagerStorageEndpoint: Done removing RDD 25, response is 0 21/01/17 17:43:08.787 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_9 stored as values in memory (estimated size 38.9 KiB, free 365.9 MiB) 21/01/17 17:43:08.787 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_9 locally took 0 ms 21/01/17 17:43:08.787 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_9 without replication took 0 ms 21/01/17 17:43:08.787 block-manager-storage-async-thread-pool-10 DEBUG BlockManagerStorageEndpoint: Sent response: 0 to 192.168.1.225:41897 21/01/17 17:43:08.785 Executor task launch worker for task 0.0 in stage 2.0 (TID 8) DEBUG LocalDiskShuffleMapOutputWriter: Writing shuffle index file for mapId 8 with length 4 21/01/17 17:43:08.788 Executor task launch worker for task 2.0 in stage 2.0 (TID 10) DEBUG LocalDiskShuffleMapOutputWriter: Writing shuffle index file for mapId 10 with length 4 21/01/17 17:43:08.789 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_9_piece0 stored as bytes in memory (estimated size 17.6 KiB, free 365.8 MiB) 21/01/17 17:43:08.789 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master broadcast_9_piece0 for BlockManagerId(driver, 192.168.1.225, 40985, None) 21/01/17 17:43:08.789 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added broadcast_9_piece0 in memory on 192.168.1.225:40985 (size: 17.6 KiB, free: 366.2 MiB) 21/01/17 17:43:08.789 dag-scheduler-event-loop DEBUG BlockManagerMaster: Updated info of block broadcast_9_piece0 21/01/17 17:43:08.789 dag-scheduler-event-loop DEBUG BlockManager: Told master about block broadcast_9_piece0 21/01/17 17:43:08.790 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_9_piece0 locally took 1 ms 21/01/17 17:43:08.790 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_9_piece0 without replication took 1 ms 21/01/17 17:43:08.790 dag-scheduler-event-loop INFO SparkContext: Created broadcast 9 from broadcast at DAGScheduler.scala:1383 21/01/17 17:43:08.790 dag-scheduler-event-loop INFO DAGScheduler: Submitting 10 missing tasks from ResultStage 17 (itemFactors MapPartitionsRDD[55] at mapPartitions at ALS.scala:1067) (first 15 tasks are for partitions Vector(0, 1, 2, 3, 4, 5, 6, 7, 8, 9)) 21/01/17 17:43:08.790 dag-scheduler-event-loop INFO TaskSchedulerImpl: Adding task set 17.0 with 10 tasks resource profile 0 21/01/17 17:43:08.791 dag-scheduler-event-loop DEBUG TaskSetManager: Epoch for TaskSet 17.0: 5 21/01/17 17:43:08.791 dag-scheduler-event-loop DEBUG TaskSetManager: Adding pending tasks took 0 ms 21/01/17 17:43:08.791 dag-scheduler-event-loop DEBUG TaskSetManager: Valid locality levels for TaskSet 17.0: PROCESS_LOCAL, NODE_LOCAL, ANY 21/01/17 17:43:08.792 Executor task launch worker for task 1.0 in stage 2.0 (TID 9) DEBUG IndexShuffleBlockResolver: Shuffle index for mapId 9: [0,179,0,0] 21/01/17 17:43:08.792 dispatcher-event-loop-3 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_17.0, runningTasks: 0 21/01/17 17:43:08.792 dispatcher-event-loop-3 INFO TaskSetManager: Starting task 0.0 in stage 17.0 (TID 63) (192.168.1.225, executor driver, partition 0, PROCESS_LOCAL, 7433 bytes) taskResourceAssignments Map() 21/01/17 17:43:08.793 dispatcher-event-loop-3 INFO TaskSetManager: Starting task 1.0 in stage 17.0 (TID 64) (192.168.1.225, executor driver, partition 1, PROCESS_LOCAL, 7433 bytes) taskResourceAssignments Map() 21/01/17 17:43:08.794 Executor task launch worker for task 3.0 in stage 2.0 (TID 11) DEBUG LocalDiskShuffleMapOutputWriter: Writing shuffle index file for mapId 11 with length 4 21/01/17 17:43:08.795 Executor task launch worker for task 3.0 in stage 2.0 (TID 11) DEBUG IndexShuffleBlockResolver: Shuffle index for mapId 11: [0,0,0,179] 21/01/17 17:43:08.794 Executor task launch worker for task 0.0 in stage 2.0 (TID 8) DEBUG IndexShuffleBlockResolver: Shuffle index for mapId 8: [0,179,179,179] 21/01/17 17:43:08.797 Executor task launch worker for task 1.0 in stage 2.0 (TID 9) DEBUG TaskMemoryManager: Task 9 release 0.0 B from org.apache.spark.util.collection.ExternalSorter@748cdd88 21/01/17 17:43:08.797 Executor task launch worker for task 3.0 in stage 2.0 (TID 11) DEBUG TaskMemoryManager: Task 11 release 0.0 B from org.apache.spark.util.collection.ExternalSorter@2f42643d 21/01/17 17:43:08.798 Executor task launch worker for task 0.0 in stage 2.0 (TID 8) DEBUG TaskMemoryManager: Task 8 release 0.0 B from org.apache.spark.util.collection.ExternalSorter@56dbb036 21/01/17 17:43:08.801 Executor task launch worker for task 2.0 in stage 2.0 (TID 10) DEBUG IndexShuffleBlockResolver: Shuffle index for mapId 10: [0,179,179,0] 21/01/17 17:43:08.801 Executor task launch worker for task 2.0 in stage 2.0 (TID 10) DEBUG TaskMemoryManager: Task 10 release 0.0 B from org.apache.spark.util.collection.ExternalSorter@58813e37 21/01/17 17:43:08.803 Executor task launch worker for task 2.0 in stage 2.0 (TID 10) INFO Executor: Finished task 2.0 in stage 2.0 (TID 10). 2030 bytes result sent to driver 21/01/17 17:43:08.803 Executor task launch worker for task 2.0 in stage 2.0 (TID 10) DEBUG ExecutorMetricsPoller: stageTCMP: (2, 0) -> 3 21/01/17 17:43:08.804 dispatcher-event-loop-2 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_2.0, runningTasks: 3 21/01/17 17:43:08.804 dispatcher-event-loop-2 DEBUG TaskSetManager: No tasks for locality level NO_PREF, so moving to locality level ANY 21/01/17 17:43:08.806 Executor task launch worker for task 1.0 in stage 2.0 (TID 9) INFO Executor: Finished task 1.0 in stage 2.0 (TID 9). 2073 bytes result sent to driver 21/01/17 17:43:08.806 Executor task launch worker for task 1.0 in stage 2.0 (TID 9) DEBUG ExecutorMetricsPoller: stageTCMP: (2, 0) -> 2 21/01/17 17:43:08.806 Executor task launch worker for task 0.0 in stage 2.0 (TID 8) INFO Executor: Finished task 0.0 in stage 2.0 (TID 8). 2073 bytes result sent to driver 21/01/17 17:43:08.806 Executor task launch worker for task 0.0 in stage 2.0 (TID 8) DEBUG ExecutorMetricsPoller: stageTCMP: (2, 0) -> 1 21/01/17 17:43:08.807 Executor task launch worker for task 3.0 in stage 2.0 (TID 11) INFO Executor: Finished task 3.0 in stage 2.0 (TID 11). 2073 bytes result sent to driver 21/01/17 17:43:08.807 task-result-getter-0 INFO TaskSetManager: Finished task 2.0 in stage 2.0 (TID 10) in 428 ms on 192.168.1.225 (executor driver) (1/4) 21/01/17 17:43:08.807 Executor task launch worker for task 3.0 in stage 2.0 (TID 11) DEBUG ExecutorMetricsPoller: removing (2, 0) from stageTCMP 21/01/17 17:43:08.807 task-result-getter-2 INFO TaskSetManager: Finished task 1.0 in stage 2.0 (TID 9) in 428 ms on 192.168.1.225 (executor driver) (2/4) 21/01/17 17:43:08.807 task-result-getter-1 INFO TaskSetManager: Finished task 0.0 in stage 2.0 (TID 8) in 430 ms on 192.168.1.225 (executor driver) (3/4) 21/01/17 17:43:08.808 task-result-getter-3 INFO TaskSetManager: Finished task 3.0 in stage 2.0 (TID 11) in 428 ms on 192.168.1.225 (executor driver) (4/4) 21/01/17 17:43:08.808 task-result-getter-3 INFO TaskSchedulerImpl: Removed TaskSet 2.0, whose tasks have all completed, from pool 21/01/17 17:43:08.808 dag-scheduler-event-loop DEBUG DAGScheduler: ShuffleMapTask finished on driver 21/01/17 17:43:08.810 dag-scheduler-event-loop DEBUG DAGScheduler: ShuffleMapTask finished on driver 21/01/17 17:43:08.811 dag-scheduler-event-loop DEBUG DAGScheduler: ShuffleMapTask finished on driver 21/01/17 17:43:08.812 dag-scheduler-event-loop DEBUG DAGScheduler: ShuffleMapTask finished on driver 21/01/17 17:43:08.812 dag-scheduler-event-loop INFO DAGScheduler: ShuffleMapStage 2 (flatMap at CountVectorizer.scala:212) finished in 0.555 s 21/01/17 17:43:08.812 dag-scheduler-event-loop INFO DAGScheduler: looking for newly runnable stages 21/01/17 17:43:08.813 dag-scheduler-event-loop INFO DAGScheduler: running: Set() 21/01/17 17:43:08.813 dag-scheduler-event-loop INFO DAGScheduler: waiting: Set(ResultStage 3) 21/01/17 17:43:08.814 dag-scheduler-event-loop INFO DAGScheduler: failed: Set() 21/01/17 17:43:08.814 block-manager-storage-async-thread-pool-11 DEBUG BlockManagerStorageEndpoint: Sent response: 0 to 192.168.1.225:41897 21/01/17 17:43:08.814 dag-scheduler-event-loop DEBUG MapOutputTrackerMaster: Increasing epoch to 1 21/01/17 17:43:08.817 dag-scheduler-event-loop DEBUG DAGScheduler: submitStage(ResultStage 3 (name=count at CountVectorizer.scala:233;jobs=2)) 21/01/17 17:43:08.818 dag-scheduler-event-loop DEBUG DAGScheduler: missing: List() 21/01/17 17:43:08.818 dag-scheduler-event-loop INFO DAGScheduler: Submitting ResultStage 3 (MapPartitionsRDD[21] at map at CountVectorizer.scala:230), which has no missing parents 21/01/17 17:43:08.818 dag-scheduler-event-loop DEBUG DAGScheduler: submitMissingTasks(ResultStage 3) 21/01/17 17:43:08.823 Thread-3 DEBUG WholeStageCodegenExec: /* 001 */ public Object generate(Object[] references) { /* 002 */ return new GeneratedIteratorForCodegenStage1(references); /* 003 */ } /* 004 */ /* 005 */ // codegenStageId=1 /* 006 */ final class GeneratedIteratorForCodegenStage1 extends org.apache.spark.sql.execution.BufferedRowIterator { /* 007 */ private Object[] references; /* 008 */ private scala.collection.Iterator[] inputs; /* 009 */ private boolean agg_initAgg_0; /* 010 */ private boolean agg_bufIsNull_0; /* 011 */ private double agg_bufValue_0; /* 012 */ private boolean agg_bufIsNull_1; /* 013 */ private double agg_bufValue_1; /* 014 */ private scala.collection.Iterator rdd_input_0; /* 015 */ private boolean agg_agg_isNull_4_0; /* 016 */ private boolean agg_agg_isNull_6_0; /* 017 */ private boolean agg_agg_isNull_22_0; /* 018 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[] rdd_mutableStateArray_0 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[7]; /* 019 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[] rdd_mutableStateArray_1 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[6]; /* 020 */ /* 021 */ public GeneratedIteratorForCodegenStage1(Object[] references) { /* 022 */ this.references = references; /* 023 */ } /* 024 */ /* 025 */ public void init(int index, scala.collection.Iterator[] inputs) { /* 026 */ partitionIndex = index; /* 027 */ this.inputs = inputs; /* 028 */ wholestagecodegen_init_0_0(); /* 029 */ wholestagecodegen_init_0_1(); /* 030 */ /* 031 */ } /* 032 */ /* 033 */ private void agg_doAggregate_sum_0(boolean agg_exprIsNull_1_0, org.apache.spark.sql.catalyst.InternalRow agg_expr_1_0, double agg_expr_0_0, boolean agg_exprIsNull_0_0) throws java.io.IOException { /* 034 */ agg_agg_isNull_4_0 = true; /* 035 */ double agg_value_4 = -1.0; /* 036 */ do { /* 037 */ boolean agg_isNull_5 = true; /* 038 */ double agg_value_5 = -1.0; /* 039 */ agg_agg_isNull_6_0 = true; /* 040 */ double agg_value_6 = -1.0; /* 041 */ do { /* 042 */ if (!agg_bufIsNull_0) { /* 043 */ agg_agg_isNull_6_0 = false; /* 044 */ agg_value_6 = agg_bufValue_0; /* 045 */ continue; /* 046 */ } /* 047 */ /* 048 */ if (!false) { /* 049 */ agg_agg_isNull_6_0 = false; /* 050 */ agg_value_6 = 0.0D; /* 051 */ continue; /* 052 */ } /* 053 */ /* 054 */ } while (false); /* 055 */ boolean agg_isNull_9 = false; /* 056 */ double agg_value_9 = -1.0; /* 057 */ if (!false && agg_exprIsNull_0_0) { /* 058 */ agg_isNull_9 = true; /* 059 */ agg_value_9 = -1.0; /* 060 */ } else { /* 061 */ Object agg_arg_0 = null; /* 062 */ if (agg_exprIsNull_1_0) { /* 063 */ agg_arg_0 = ((scala.Function1[]) references[4] /* converters */)[0].apply(null); /* 064 */ } else { /* 065 */ agg_arg_0 = ((scala.Function1[]) references[4] /* converters */)[0].apply(agg_expr_1_0); /* 066 */ } /* 067 */ /* 068 */ Double agg_conv_0 = agg_expr_0_0; /* 069 */ Object agg_arg_1 = false ? null : agg_conv_0; /* 070 */ /* 071 */ Double agg_conv_1 = 1.0D; /* 072 */ Object agg_arg_2 = false ? null : agg_conv_1; /* 073 */ /* 074 */ Double agg_result_0 = null; /* 075 */ try { /* 076 */ agg_result_0 = (Double)((scala.Function3) references[6] /* udf */).apply(agg_arg_0, agg_arg_1, agg_arg_2); /* 077 */ } catch (Exception e) { /* 078 */ throw new org.apache.spark.SparkException(((java.lang.String) references[5] /* errMsg */), e); /* 079 */ } /* 080 */ /* 081 */ boolean agg_isNull_14 = agg_result_0 == null; /* 082 */ double agg_value_14 = -1.0; /* 083 */ if (!agg_isNull_14) { /* 084 */ agg_value_14 = agg_result_0; /* 085 */ } /* 086 */ /* 087 */ double agg_value_13 = -1.0; /* 088 */ /* 089 */ agg_value_13 = agg_value_14 * 1.0D; /* 090 */ agg_isNull_9 = false; /* 091 */ agg_value_9 = agg_value_13; /* 092 */ } /* 093 */ if (!agg_isNull_9) { /* 094 */ agg_isNull_5 = false; // resultCode could change nullability. /* 095 */ /* 096 */ agg_value_5 = agg_value_6 + agg_value_9; /* 097 */ /* 098 */ } /* 099 */ if (!agg_isNull_5) { /* 100 */ agg_agg_isNull_4_0 = false; /* 101 */ agg_value_4 = agg_value_5; /* 102 */ continue; /* 103 */ } /* 104 */ /* 105 */ if (!agg_bufIsNull_0) { /* 106 */ agg_agg_isNull_4_0 = false; /* 107 */ agg_value_4 = agg_bufValue_0; /* 108 */ continue; /* 109 */ } /* 110 */ /* 111 */ } while (false); /* 112 */ /* 113 */ agg_bufIsNull_0 = agg_agg_isNull_4_0; /* 114 */ agg_bufValue_0 = agg_value_4; /* 115 */ } /* 116 */ /* 117 */ private void agg_doAggregateWithoutKey_0() throws java.io.IOException { /* 118 */ // initialize aggregation buffer /* 119 */ agg_bufIsNull_0 = true; /* 120 */ agg_bufValue_0 = -1.0; /* 121 */ agg_bufIsNull_1 = true; /* 122 */ agg_bufValue_1 = -1.0; /* 123 */ /* 124 */ while ( rdd_input_0.hasNext()) { /* 125 */ InternalRow rdd_row_0 = (InternalRow) rdd_input_0.next(); /* 126 */ ((org.apache.spark.sql.execution.metric.SQLMetric) references[0] /* numOutputRows */).add(1); /* 127 */ // common sub-expressions /* 128 */ /* 129 */ boolean rdd_isNull_1 = rdd_row_0.isNullAt(1); /* 130 */ double rdd_value_1 = rdd_isNull_1 ? /* 131 */ -1.0 : (rdd_row_0.getDouble(1)); /* 132 */ boolean rdd_isNull_0 = rdd_row_0.isNullAt(0); /* 133 */ InternalRow rdd_value_0 = rdd_isNull_0 ? /* 134 */ null : (rdd_row_0.getStruct(0, 4)); /* 135 */ /* 136 */ Object project_arg_0 = null; /* 137 */ if (rdd_isNull_0) { /* 138 */ project_arg_0 = ((scala.Function1[]) references[1] /* converters */)[0].apply(null); /* 139 */ } else { /* 140 */ project_arg_0 = ((scala.Function1[]) references[1] /* converters */)[0].apply(rdd_value_0); /* 141 */ } /* 142 */ /* 143 */ InternalRow project_result_0 = null; /* 144 */ try { /* 145 */ project_result_0 = (InternalRow)((scala.Function1[]) references[1] /* converters */)[1].apply(((scala.Function1) references[3] /* udf */).apply(project_arg_0)); /* 146 */ } catch (Exception e) { /* 147 */ throw new org.apache.spark.SparkException(((java.lang.String) references[2] /* errMsg */), e); /* 148 */ } /* 149 */ /* 150 */ boolean project_isNull_1 = project_result_0 == null; /* 151 */ InternalRow project_value_1 = null; /* 152 */ if (!project_isNull_1) { /* 153 */ project_value_1 = project_result_0; /* 154 */ } /* 155 */ /* 156 */ agg_doConsume_0(rdd_value_1, rdd_isNull_1, project_value_1, project_isNull_1); /* 157 */ // shouldStop check is eliminated /* 158 */ } /* 159 */ /* 160 */ } /* 161 */ /* 162 */ private void project_writeFields_0_1(InternalRow project_tmpInput_0) { /* 163 */ if ((project_tmpInput_0.isNullAt(3))) { /* 164 */ rdd_mutableStateArray_0[3].setNullAt(3); /* 165 */ } else { /* 166 */ // Remember the current cursor so that we can calculate how many bytes are /* 167 */ // written later. /* 168 */ final int project_previousCursor_2 = rdd_mutableStateArray_0[3].cursor(); /* 169 */ /* 170 */ final ArrayData project_tmpInput_2 = (project_tmpInput_0.getArray(3)); /* 171 */ if (project_tmpInput_2 instanceof UnsafeArrayData) { /* 172 */ rdd_mutableStateArray_0[3].write((UnsafeArrayData) project_tmpInput_2); /* 173 */ } else { /* 174 */ final int project_numElements_1 = project_tmpInput_2.numElements(); /* 175 */ rdd_mutableStateArray_1[3].initialize(project_numElements_1); /* 176 */ /* 177 */ for (int project_index_1 = 0; project_index_1 < project_numElements_1; project_index_1++) { /* 178 */ rdd_mutableStateArray_1[3].write(project_index_1, project_tmpInput_2.getDouble(project_index_1)); /* 179 */ } /* 180 */ } /* 181 */ /* 182 */ rdd_mutableStateArray_0[3].setOffsetAndSizeFromPreviousCursor(3, project_previousCursor_2); /* 183 */ } /* 184 */ /* 185 */ } /* 186 */ /* 187 */ private void wholestagecodegen_init_0_1() { /* 188 */ rdd_mutableStateArray_0[4] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(2, 32); /* 189 */ rdd_mutableStateArray_0[5] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(rdd_mutableStateArray_0[4], 4); /* 190 */ rdd_mutableStateArray_1[4] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[5], 4); /* 191 */ rdd_mutableStateArray_1[5] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[5], 8); /* 192 */ rdd_mutableStateArray_0[6] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(2, 0); /* 193 */ /* 194 */ } /* 195 */ /* 196 */ private void agg_doConsume_0(double agg_expr_0_0, boolean agg_exprIsNull_0_0, InternalRow agg_expr_1_0, boolean agg_exprIsNull_1_0) throws java.io.IOException { /* 197 */ // do aggregate /* 198 */ // common sub-expressions /* 199 */ /* 200 */ // evaluate aggregate functions and update aggregation buffers /* 201 */ agg_doAggregate_sum_0(agg_exprIsNull_1_0, agg_expr_1_0, agg_expr_0_0, agg_exprIsNull_0_0); /* 202 */ agg_doAggregate_sum_1(); /* 203 */ /* 204 */ } /* 205 */ /* 206 */ private void agg_doAggregate_sum_1() throws java.io.IOException { /* 207 */ agg_agg_isNull_22_0 = true; /* 208 */ double agg_value_22 = -1.0; /* 209 */ do { /* 210 */ if (!agg_bufIsNull_1) { /* 211 */ agg_agg_isNull_22_0 = false; /* 212 */ agg_value_22 = agg_bufValue_1; /* 213 */ continue; /* 214 */ } /* 215 */ /* 216 */ if (!false) { /* 217 */ agg_agg_isNull_22_0 = false; /* 218 */ agg_value_22 = 0.0D; /* 219 */ continue; /* 220 */ } /* 221 */ /* 222 */ } while (false); /* 223 */ /* 224 */ double agg_value_21 = -1.0; /* 225 */ /* 226 */ agg_value_21 = agg_value_22 + 1.0D; /* 227 */ /* 228 */ agg_bufIsNull_1 = false; /* 229 */ agg_bufValue_1 = agg_value_21; /* 230 */ } /* 231 */ /* 232 */ private void project_writeFields_0_0(InternalRow project_tmpInput_0) { /* 233 */ rdd_mutableStateArray_0[3].write(0, (project_tmpInput_0.getByte(0))); /* 234 */ /* 235 */ if ((project_tmpInput_0.isNullAt(1))) { /* 236 */ rdd_mutableStateArray_0[3].setNullAt(1); /* 237 */ } else { /* 238 */ rdd_mutableStateArray_0[3].write(1, (project_tmpInput_0.getInt(1))); /* 239 */ } /* 240 */ /* 241 */ if ((project_tmpInput_0.isNullAt(2))) { /* 242 */ rdd_mutableStateArray_0[3].setNullAt(2); /* 243 */ } else { /* 244 */ // Remember the current cursor so that we can calculate how many bytes are /* 245 */ // written later. /* 246 */ final int project_previousCursor_1 = rdd_mutableStateArray_0[3].cursor(); /* 247 */ /* 248 */ final ArrayData project_tmpInput_1 = (project_tmpInput_0.getArray(2)); /* 249 */ if (project_tmpInput_1 instanceof UnsafeArrayData) { /* 250 */ rdd_mutableStateArray_0[3].write((UnsafeArrayData) project_tmpInput_1); /* 251 */ } else { /* 252 */ final int project_numElements_0 = project_tmpInput_1.numElements(); /* 253 */ rdd_mutableStateArray_1[2].initialize(project_numElements_0); /* 254 */ /* 255 */ for (int project_index_0 = 0; project_index_0 < project_numElements_0; project_index_0++) { /* 256 */ rdd_mutableStateArray_1[2].write(project_index_0, project_tmpInput_1.getInt(project_index_0)); /* 257 */ } /* 258 */ } /* 259 */ /* 260 */ rdd_mutableStateArray_0[3].setOffsetAndSizeFromPreviousCursor(2, project_previousCursor_1); /* 261 */ } /* 262 */ /* 263 */ } /* 264 */ /* 265 */ protected void processNext() throws java.io.IOException { /* 266 */ while (!agg_initAgg_0) { /* 267 */ agg_initAgg_0 = true; /* 268 */ long agg_beforeAgg_0 = System.nanoTime(); /* 269 */ agg_doAggregateWithoutKey_0(); /* 270 */ ((org.apache.spark.sql.execution.metric.SQLMetric) references[8] /* aggTime */).add((System.nanoTime() - agg_beforeAgg_0) / 1000000); /* 271 */ /* 272 */ // output the result /* 273 */ /* 274 */ ((org.apache.spark.sql.execution.metric.SQLMetric) references[7] /* numOutputRows */).add(1); /* 275 */ rdd_mutableStateArray_0[6].reset(); /* 276 */ /* 277 */ rdd_mutableStateArray_0[6].zeroOutNullBytes(); /* 278 */ /* 279 */ if (agg_bufIsNull_0) { /* 280 */ rdd_mutableStateArray_0[6].setNullAt(0); /* 281 */ } else { /* 282 */ rdd_mutableStateArray_0[6].write(0, agg_bufValue_0); /* 283 */ } /* 284 */ /* 285 */ if (agg_bufIsNull_1) { /* 286 */ rdd_mutableStateArray_0[6].setNullAt(1); /* 287 */ } else { /* 288 */ rdd_mutableStateArray_0[6].write(1, agg_bufValue_1); /* 289 */ } /* 290 */ append((rdd_mutableStateArray_0[6].getRow())); /* 291 */ } /* 292 */ } /* 293 */ /* 294 */ private void wholestagecodegen_init_0_0() { /* 295 */ rdd_input_0 = inputs[0]; /* 296 */ rdd_mutableStateArray_0[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(2, 32); /* 297 */ rdd_mutableStateArray_0[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(rdd_mutableStateArray_0[0], 4); /* 298 */ rdd_mutableStateArray_1[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[1], 4); /* 299 */ rdd_mutableStateArray_1[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[1], 8); /* 300 */ rdd_mutableStateArray_0[2] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(2, 32); /* 301 */ rdd_mutableStateArray_0[3] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(rdd_mutableStateArray_0[2], 4); /* 302 */ rdd_mutableStateArray_1[2] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[3], 4); /* 303 */ rdd_mutableStateArray_1[3] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[3], 8); /* 304 */ /* 305 */ } /* 306 */ /* 307 */ } 21/01/17 17:43:08.828 Thread-3 DEBUG CodeGenerator: /* 001 */ public Object generate(Object[] references) { /* 002 */ return new GeneratedIteratorForCodegenStage1(references); /* 003 */ } /* 004 */ /* 005 */ // codegenStageId=1 /* 006 */ final class GeneratedIteratorForCodegenStage1 extends org.apache.spark.sql.execution.BufferedRowIterator { /* 007 */ private Object[] references; /* 008 */ private scala.collection.Iterator[] inputs; /* 009 */ private boolean agg_initAgg_0; /* 010 */ private boolean agg_bufIsNull_0; /* 011 */ private double agg_bufValue_0; /* 012 */ private boolean agg_bufIsNull_1; /* 013 */ private double agg_bufValue_1; /* 014 */ private scala.collection.Iterator rdd_input_0; /* 015 */ private boolean agg_agg_isNull_4_0; /* 016 */ private boolean agg_agg_isNull_6_0; /* 017 */ private boolean agg_agg_isNull_22_0; /* 018 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[] rdd_mutableStateArray_0 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[7]; /* 019 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[] rdd_mutableStateArray_1 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[6]; /* 020 */ /* 021 */ public GeneratedIteratorForCodegenStage1(Object[] references) { /* 022 */ this.references = references; /* 023 */ } /* 024 */ /* 025 */ public void init(int index, scala.collection.Iterator[] inputs) { /* 026 */ partitionIndex = index; /* 027 */ this.inputs = inputs; /* 028 */ wholestagecodegen_init_0_0(); /* 029 */ wholestagecodegen_init_0_1(); /* 030 */ /* 031 */ } /* 032 */ /* 033 */ private void agg_doAggregate_sum_0(boolean agg_exprIsNull_1_0, org.apache.spark.sql.catalyst.InternalRow agg_expr_1_0, double agg_expr_0_0, boolean agg_exprIsNull_0_0) throws java.io.IOException { /* 034 */ agg_agg_isNull_4_0 = true; /* 035 */ double agg_value_4 = -1.0; /* 036 */ do { /* 037 */ boolean agg_isNull_5 = true; /* 038 */ double agg_value_5 = -1.0; /* 039 */ agg_agg_isNull_6_0 = true; /* 040 */ double agg_value_6 = -1.0; /* 041 */ do { /* 042 */ if (!agg_bufIsNull_0) { /* 043 */ agg_agg_isNull_6_0 = false; /* 044 */ agg_value_6 = agg_bufValue_0; /* 045 */ continue; /* 046 */ } /* 047 */ /* 048 */ if (!false) { /* 049 */ agg_agg_isNull_6_0 = false; /* 050 */ agg_value_6 = 0.0D; /* 051 */ continue; /* 052 */ } /* 053 */ /* 054 */ } while (false); /* 055 */ boolean agg_isNull_9 = false; /* 056 */ double agg_value_9 = -1.0; /* 057 */ if (!false && agg_exprIsNull_0_0) { /* 058 */ agg_isNull_9 = true; /* 059 */ agg_value_9 = -1.0; /* 060 */ } else { /* 061 */ Object agg_arg_0 = null; /* 062 */ if (agg_exprIsNull_1_0) { /* 063 */ agg_arg_0 = ((scala.Function1[]) references[4] /* converters */)[0].apply(null); /* 064 */ } else { /* 065 */ agg_arg_0 = ((scala.Function1[]) references[4] /* converters */)[0].apply(agg_expr_1_0); /* 066 */ } /* 067 */ /* 068 */ Double agg_conv_0 = agg_expr_0_0; /* 069 */ Object agg_arg_1 = false ? null : agg_conv_0; /* 070 */ /* 071 */ Double agg_conv_1 = 1.0D; /* 072 */ Object agg_arg_2 = false ? null : agg_conv_1; /* 073 */ /* 074 */ Double agg_result_0 = null; /* 075 */ try { /* 076 */ agg_result_0 = (Double)((scala.Function3) references[6] /* udf */).apply(agg_arg_0, agg_arg_1, agg_arg_2); /* 077 */ } catch (Exception e) { /* 078 */ throw new org.apache.spark.SparkException(((java.lang.String) references[5] /* errMsg */), e); /* 079 */ } /* 080 */ /* 081 */ boolean agg_isNull_14 = agg_result_0 == null; /* 082 */ double agg_value_14 = -1.0; /* 083 */ if (!agg_isNull_14) { /* 084 */ agg_value_14 = agg_result_0; /* 085 */ } /* 086 */ /* 087 */ double agg_value_13 = -1.0; /* 088 */ /* 089 */ agg_value_13 = agg_value_14 * 1.0D; /* 090 */ agg_isNull_9 = false; /* 091 */ agg_value_9 = agg_value_13; /* 092 */ } /* 093 */ if (!agg_isNull_9) { /* 094 */ agg_isNull_5 = false; // resultCode could change nullability. /* 095 */ /* 096 */ agg_value_5 = agg_value_6 + agg_value_9; /* 097 */ /* 098 */ } /* 099 */ if (!agg_isNull_5) { /* 100 */ agg_agg_isNull_4_0 = false; /* 101 */ agg_value_4 = agg_value_5; /* 102 */ continue; /* 103 */ } /* 104 */ /* 105 */ if (!agg_bufIsNull_0) { /* 106 */ agg_agg_isNull_4_0 = false; /* 107 */ agg_value_4 = agg_bufValue_0; /* 108 */ continue; /* 109 */ } /* 110 */ /* 111 */ } while (false); /* 112 */ /* 113 */ agg_bufIsNull_0 = agg_agg_isNull_4_0; /* 114 */ agg_bufValue_0 = agg_value_4; /* 115 */ } /* 116 */ /* 117 */ private void agg_doAggregateWithoutKey_0() throws java.io.IOException { /* 118 */ // initialize aggregation buffer /* 119 */ agg_bufIsNull_0 = true; /* 120 */ agg_bufValue_0 = -1.0; /* 121 */ agg_bufIsNull_1 = true; /* 122 */ agg_bufValue_1 = -1.0; /* 123 */ /* 124 */ while ( rdd_input_0.hasNext()) { /* 125 */ InternalRow rdd_row_0 = (InternalRow) rdd_input_0.next(); /* 126 */ ((org.apache.spark.sql.execution.metric.SQLMetric) references[0] /* numOutputRows */).add(1); /* 127 */ // common sub-expressions /* 128 */ /* 129 */ boolean rdd_isNull_1 = rdd_row_0.isNullAt(1); /* 130 */ double rdd_value_1 = rdd_isNull_1 ? /* 131 */ -1.0 : (rdd_row_0.getDouble(1)); /* 132 */ boolean rdd_isNull_0 = rdd_row_0.isNullAt(0); /* 133 */ InternalRow rdd_value_0 = rdd_isNull_0 ? /* 134 */ null : (rdd_row_0.getStruct(0, 4)); /* 135 */ /* 136 */ Object project_arg_0 = null; /* 137 */ if (rdd_isNull_0) { /* 138 */ project_arg_0 = ((scala.Function1[]) references[1] /* converters */)[0].apply(null); /* 139 */ } else { /* 140 */ project_arg_0 = ((scala.Function1[]) references[1] /* converters */)[0].apply(rdd_value_0); /* 141 */ } /* 142 */ /* 143 */ InternalRow project_result_0 = null; /* 144 */ try { /* 145 */ project_result_0 = (InternalRow)((scala.Function1[]) references[1] /* converters */)[1].apply(((scala.Function1) references[3] /* udf */).apply(project_arg_0)); /* 146 */ } catch (Exception e) { /* 147 */ throw new org.apache.spark.SparkException(((java.lang.String) references[2] /* errMsg */), e); /* 148 */ } /* 149 */ /* 150 */ boolean project_isNull_1 = project_result_0 == null; /* 151 */ InternalRow project_value_1 = null; /* 152 */ if (!project_isNull_1) { /* 153 */ project_value_1 = project_result_0; /* 154 */ } /* 155 */ /* 156 */ agg_doConsume_0(rdd_value_1, rdd_isNull_1, project_value_1, project_isNull_1); /* 157 */ // shouldStop check is eliminated /* 158 */ } /* 159 */ /* 160 */ } /* 161 */ /* 162 */ private void project_writeFields_0_1(InternalRow project_tmpInput_0) { /* 163 */ if ((project_tmpInput_0.isNullAt(3))) { /* 164 */ rdd_mutableStateArray_0[3].setNullAt(3); /* 165 */ } else { /* 166 */ // Remember the current cursor so that we can calculate how many bytes are /* 167 */ // written later. /* 168 */ final int project_previousCursor_2 = rdd_mutableStateArray_0[3].cursor(); /* 169 */ /* 170 */ final ArrayData project_tmpInput_2 = (project_tmpInput_0.getArray(3)); /* 171 */ if (project_tmpInput_2 instanceof UnsafeArrayData) { /* 172 */ rdd_mutableStateArray_0[3].write((UnsafeArrayData) project_tmpInput_2); /* 173 */ } else { /* 174 */ final int project_numElements_1 = project_tmpInput_2.numElements(); /* 175 */ rdd_mutableStateArray_1[3].initialize(project_numElements_1); /* 176 */ /* 177 */ for (int project_index_1 = 0; project_index_1 < project_numElements_1; project_index_1++) { /* 178 */ rdd_mutableStateArray_1[3].write(project_index_1, project_tmpInput_2.getDouble(project_index_1)); /* 179 */ } /* 180 */ } /* 181 */ /* 182 */ rdd_mutableStateArray_0[3].setOffsetAndSizeFromPreviousCursor(3, project_previousCursor_2); /* 183 */ } /* 184 */ /* 185 */ } /* 186 */ /* 187 */ private void wholestagecodegen_init_0_1() { /* 188 */ rdd_mutableStateArray_0[4] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(2, 32); /* 189 */ rdd_mutableStateArray_0[5] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(rdd_mutableStateArray_0[4], 4); /* 190 */ rdd_mutableStateArray_1[4] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[5], 4); /* 191 */ rdd_mutableStateArray_1[5] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[5], 8); /* 192 */ rdd_mutableStateArray_0[6] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(2, 0); /* 193 */ /* 194 */ } /* 195 */ /* 196 */ private void agg_doConsume_0(double agg_expr_0_0, boolean agg_exprIsNull_0_0, InternalRow agg_expr_1_0, boolean agg_exprIsNull_1_0) throws java.io.IOException { /* 197 */ // do aggregate /* 198 */ // common sub-expressions /* 199 */ /* 200 */ // evaluate aggregate functions and update aggregation buffers /* 201 */ agg_doAggregate_sum_0(agg_exprIsNull_1_0, agg_expr_1_0, agg_expr_0_0, agg_exprIsNull_0_0); /* 202 */ agg_doAggregate_sum_1(); /* 203 */ /* 204 */ } /* 205 */ /* 206 */ private void agg_doAggregate_sum_1() throws java.io.IOException { /* 207 */ agg_agg_isNull_22_0 = true; /* 208 */ double agg_value_22 = -1.0; /* 209 */ do { /* 210 */ if (!agg_bufIsNull_1) { /* 211 */ agg_agg_isNull_22_0 = false; /* 212 */ agg_value_22 = agg_bufValue_1; /* 213 */ continue; /* 214 */ } /* 215 */ /* 216 */ if (!false) { /* 217 */ agg_agg_isNull_22_0 = false; /* 218 */ agg_value_22 = 0.0D; /* 219 */ continue; /* 220 */ } /* 221 */ /* 222 */ } while (false); /* 223 */ /* 224 */ double agg_value_21 = -1.0; /* 225 */ /* 226 */ agg_value_21 = agg_value_22 + 1.0D; /* 227 */ /* 228 */ agg_bufIsNull_1 = false; /* 229 */ agg_bufValue_1 = agg_value_21; /* 230 */ } /* 231 */ /* 232 */ private void project_writeFields_0_0(InternalRow project_tmpInput_0) { /* 233 */ rdd_mutableStateArray_0[3].write(0, (project_tmpInput_0.getByte(0))); /* 234 */ /* 235 */ if ((project_tmpInput_0.isNullAt(1))) { /* 236 */ rdd_mutableStateArray_0[3].setNullAt(1); /* 237 */ } else { /* 238 */ rdd_mutableStateArray_0[3].write(1, (project_tmpInput_0.getInt(1))); /* 239 */ } /* 240 */ /* 241 */ if ((project_tmpInput_0.isNullAt(2))) { /* 242 */ rdd_mutableStateArray_0[3].setNullAt(2); /* 243 */ } else { /* 244 */ // Remember the current cursor so that we can calculate how many bytes are /* 245 */ // written later. /* 246 */ final int project_previousCursor_1 = rdd_mutableStateArray_0[3].cursor(); /* 247 */ /* 248 */ final ArrayData project_tmpInput_1 = (project_tmpInput_0.getArray(2)); /* 249 */ if (project_tmpInput_1 instanceof UnsafeArrayData) { /* 250 */ rdd_mutableStateArray_0[3].write((UnsafeArrayData) project_tmpInput_1); /* 251 */ } else { /* 252 */ final int project_numElements_0 = project_tmpInput_1.numElements(); /* 253 */ rdd_mutableStateArray_1[2].initialize(project_numElements_0); /* 254 */ /* 255 */ for (int project_index_0 = 0; project_index_0 < project_numElements_0; project_index_0++) { /* 256 */ rdd_mutableStateArray_1[2].write(project_index_0, project_tmpInput_1.getInt(project_index_0)); /* 257 */ } /* 258 */ } /* 259 */ /* 260 */ rdd_mutableStateArray_0[3].setOffsetAndSizeFromPreviousCursor(2, project_previousCursor_1); /* 261 */ } /* 262 */ /* 263 */ } /* 264 */ /* 265 */ protected void processNext() throws java.io.IOException { /* 266 */ while (!agg_initAgg_0) { /* 267 */ agg_initAgg_0 = true; /* 268 */ long agg_beforeAgg_0 = System.nanoTime(); /* 269 */ agg_doAggregateWithoutKey_0(); /* 270 */ ((org.apache.spark.sql.execution.metric.SQLMetric) references[8] /* aggTime */).add((System.nanoTime() - agg_beforeAgg_0) / 1000000); /* 271 */ /* 272 */ // output the result /* 273 */ /* 274 */ ((org.apache.spark.sql.execution.metric.SQLMetric) references[7] /* numOutputRows */).add(1); /* 275 */ rdd_mutableStateArray_0[6].reset(); /* 276 */ /* 277 */ rdd_mutableStateArray_0[6].zeroOutNullBytes(); /* 278 */ /* 279 */ if (agg_bufIsNull_0) { /* 280 */ rdd_mutableStateArray_0[6].setNullAt(0); /* 281 */ } else { /* 282 */ rdd_mutableStateArray_0[6].write(0, agg_bufValue_0); /* 283 */ } /* 284 */ /* 285 */ if (agg_bufIsNull_1) { /* 286 */ rdd_mutableStateArray_0[6].setNullAt(1); /* 287 */ } else { /* 288 */ rdd_mutableStateArray_0[6].write(1, agg_bufValue_1); /* 289 */ } /* 290 */ append((rdd_mutableStateArray_0[6].getRow())); /* 291 */ } /* 292 */ } /* 293 */ /* 294 */ private void wholestagecodegen_init_0_0() { /* 295 */ rdd_input_0 = inputs[0]; /* 296 */ rdd_mutableStateArray_0[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(2, 32); /* 297 */ rdd_mutableStateArray_0[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(rdd_mutableStateArray_0[0], 4); /* 298 */ rdd_mutableStateArray_1[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[1], 4); /* 299 */ rdd_mutableStateArray_1[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[1], 8); /* 300 */ rdd_mutableStateArray_0[2] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(2, 32); /* 301 */ rdd_mutableStateArray_0[3] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(rdd_mutableStateArray_0[2], 4); /* 302 */ rdd_mutableStateArray_1[2] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[3], 4); /* 303 */ rdd_mutableStateArray_1[3] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(rdd_mutableStateArray_0[3], 8); /* 304 */ /* 305 */ } /* 306 */ /* 307 */ } 21/01/17 17:43:08.829 block-manager-storage-async-thread-pool-12 DEBUG BlockManagerStorageEndpoint: Sent response: 0 to 192.168.1.225:41897 21/01/17 17:43:08.830 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_5 stored as values in memory (estimated size 4.9 KiB, free 366.2 MiB) 21/01/17 17:43:08.830 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_5 locally took 1 ms 21/01/17 17:43:08.831 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_5 without replication took 1 ms 21/01/17 17:43:08.832 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_5_piece0 stored as bytes in memory (estimated size 2.7 KiB, free 366.2 MiB) 21/01/17 17:43:08.832 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master broadcast_5_piece0 for BlockManagerId(driver, 192.168.1.225, 46295, None) 21/01/17 17:43:08.832 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added broadcast_5_piece0 in memory on 192.168.1.225:46295 (size: 2.7 KiB, free: 366.3 MiB) 21/01/17 17:43:08.833 dag-scheduler-event-loop DEBUG BlockManagerMaster: Updated info of block broadcast_5_piece0 21/01/17 17:43:08.833 dag-scheduler-event-loop DEBUG BlockManager: Told master about block broadcast_5_piece0 21/01/17 17:43:08.833 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_5_piece0 locally took 1 ms 21/01/17 17:43:08.833 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_5_piece0 without replication took 1 ms 21/01/17 17:43:08.833 dag-scheduler-event-loop INFO SparkContext: Created broadcast 5 from broadcast at DAGScheduler.scala:1383 21/01/17 17:43:08.834 dag-scheduler-event-loop INFO DAGScheduler: Submitting 4 missing tasks from ResultStage 3 (MapPartitionsRDD[21] at map at CountVectorizer.scala:230) (first 15 tasks are for partitions Vector(0, 1, 2, 3)) 21/01/17 17:43:08.834 dag-scheduler-event-loop INFO TaskSchedulerImpl: Adding task set 3.0 with 4 tasks resource profile 0 21/01/17 17:43:08.835 dag-scheduler-event-loop DEBUG TaskSetManager: Epoch for TaskSet 3.0: 1 21/01/17 17:43:08.836 dag-scheduler-event-loop DEBUG TaskSetManager: Adding pending tasks took 1 ms 21/01/17 17:43:08.837 dag-scheduler-event-loop DEBUG TaskSetManager: Valid locality levels for TaskSet 3.0: NODE_LOCAL, NO_PREF, ANY 21/01/17 17:43:08.838 dispatcher-event-loop-2 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_3.0, runningTasks: 0 21/01/17 17:43:08.840 dispatcher-event-loop-2 INFO TaskSetManager: Starting task 1.0 in stage 3.0 (TID 12) (192.168.1.225, executor driver, partition 1, NODE_LOCAL, 7143 bytes) taskResourceAssignments Map() 21/01/17 17:43:08.841 dispatcher-event-loop-2 INFO TaskSetManager: Starting task 2.0 in stage 3.0 (TID 13) (192.168.1.225, executor driver, partition 2, NODE_LOCAL, 7143 bytes) taskResourceAssignments Map() 21/01/17 17:43:08.841 dispatcher-event-loop-2 INFO TaskSetManager: Starting task 3.0 in stage 3.0 (TID 14) (192.168.1.225, executor driver, partition 3, NODE_LOCAL, 7143 bytes) taskResourceAssignments Map() 21/01/17 17:43:08.841 dispatcher-event-loop-2 DEBUG TaskSetManager: No tasks for locality level NODE_LOCAL, so moving to locality level NO_PREF 21/01/17 17:43:08.842 dispatcher-event-loop-2 DEBUG TaskSetManager: Moving to ANY after waiting for 0ms 21/01/17 17:43:08.842 dispatcher-event-loop-2 INFO TaskSetManager: Starting task 0.0 in stage 3.0 (TID 15) (192.168.1.225, executor driver, partition 0, PROCESS_LOCAL, 7143 bytes) taskResourceAssignments Map() 21/01/17 17:43:08.842 Executor task launch worker for task 2.0 in stage 3.0 (TID 13) INFO Executor: Running task 2.0 in stage 3.0 (TID 13) 21/01/17 17:43:08.842 Executor task launch worker for task 3.0 in stage 3.0 (TID 14) INFO Executor: Running task 3.0 in stage 3.0 (TID 14) 21/01/17 17:43:08.843 Executor task launch worker for task 2.0 in stage 3.0 (TID 13) DEBUG ExecutorMetricsPoller: stageTCMP: (3, 0) -> 1 21/01/17 17:43:08.842 Executor task launch worker for task 0.0 in stage 3.0 (TID 15) INFO Executor: Running task 0.0 in stage 3.0 (TID 15) 21/01/17 17:43:08.842 Executor task launch worker for task 1.0 in stage 3.0 (TID 12) INFO Executor: Running task 1.0 in stage 3.0 (TID 12) 21/01/17 17:43:08.843 Executor task launch worker for task 3.0 in stage 3.0 (TID 14) DEBUG ExecutorMetricsPoller: stageTCMP: (3, 0) -> 2 21/01/17 17:43:08.844 Executor task launch worker for task 0.0 in stage 3.0 (TID 15) DEBUG ExecutorMetricsPoller: stageTCMP: (3, 0) -> 3 21/01/17 17:43:08.844 Executor task launch worker for task 2.0 in stage 3.0 (TID 13) DEBUG BlockManager: Getting local block broadcast_5 21/01/17 17:43:08.844 Executor task launch worker for task 1.0 in stage 3.0 (TID 12) DEBUG ExecutorMetricsPoller: stageTCMP: (3, 0) -> 4 21/01/17 17:43:08.844 Executor task launch worker for task 2.0 in stage 3.0 (TID 13) DEBUG BlockManager: Level for block broadcast_5 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:08.846 block-manager-storage-async-thread-pool-14 DEBUG BlockManagerStorageEndpoint: Done removing broadcast 5, response is 0 21/01/17 17:43:08.847 Executor task launch worker for task 1.0 in stage 3.0 (TID 12) DEBUG BlockManager: Getting local block rdd_21_1 21/01/17 17:43:08.848 Executor task launch worker for task 2.0 in stage 3.0 (TID 13) DEBUG BlockManager: Getting local block rdd_21_2 21/01/17 17:43:08.848 Executor task launch worker for task 0.0 in stage 3.0 (TID 15) DEBUG BlockManager: Getting local block rdd_21_0 21/01/17 17:43:08.848 Executor task launch worker for task 1.0 in stage 3.0 (TID 12) DEBUG BlockManager: Block rdd_21_1 was not found 21/01/17 17:43:08.847 Executor task launch worker for task 3.0 in stage 3.0 (TID 14) DEBUG BlockManager: Getting local block rdd_21_3 21/01/17 17:43:08.848 Executor task launch worker for task 2.0 in stage 3.0 (TID 13) DEBUG BlockManager: Block rdd_21_2 was not found 21/01/17 17:43:08.848 Executor task launch worker for task 0.0 in stage 3.0 (TID 15) DEBUG BlockManager: Block rdd_21_0 was not found 21/01/17 17:43:08.848 Executor task launch worker for task 3.0 in stage 3.0 (TID 14) DEBUG BlockManager: Block rdd_21_3 was not found 21/01/17 17:43:08.849 Executor task launch worker for task 2.0 in stage 3.0 (TID 13) DEBUG BlockManager: Getting remote block rdd_21_2 21/01/17 17:43:08.849 Executor task launch worker for task 3.0 in stage 3.0 (TID 14) DEBUG BlockManager: Getting remote block rdd_21_3 21/01/17 17:43:08.849 Executor task launch worker for task 0.0 in stage 3.0 (TID 15) DEBUG BlockManager: Getting remote block rdd_21_0 21/01/17 17:43:08.849 Executor task launch worker for task 1.0 in stage 3.0 (TID 12) DEBUG BlockManager: Getting remote block rdd_21_1 21/01/17 17:43:08.852 Executor task launch worker for task 0.0 in stage 3.0 (TID 15) DEBUG BlockManager: Block rdd_21_0 is unknown by block manager master 21/01/17 17:43:08.852 Executor task launch worker for task 3.0 in stage 3.0 (TID 14) DEBUG BlockManager: Block rdd_21_3 is unknown by block manager master 21/01/17 17:43:08.852 Executor task launch worker for task 1.0 in stage 3.0 (TID 12) DEBUG BlockManager: Block rdd_21_1 is unknown by block manager master 21/01/17 17:43:08.853 dispatcher-event-loop-3 INFO TaskSetManager: Starting task 2.0 in stage 17.0 (TID 65) (192.168.1.225, executor driver, partition 2, PROCESS_LOCAL, 7433 bytes) taskResourceAssignments Map() 21/01/17 17:43:08.853 dispatcher-event-loop-3 INFO TaskSetManager: Starting task 3.0 in stage 17.0 (TID 66) (192.168.1.225, executor driver, partition 3, PROCESS_LOCAL, 7433 bytes) taskResourceAssignments Map() 21/01/17 17:43:08.854 Executor task launch worker for task 0.0 in stage 17.0 (TID 63) INFO Executor: Running task 0.0 in stage 17.0 (TID 63) 21/01/17 17:43:08.855 Executor task launch worker for task 3.0 in stage 17.0 (TID 66) INFO Executor: Running task 3.0 in stage 17.0 (TID 66) 21/01/17 17:43:08.855 Executor task launch worker for task 1.0 in stage 17.0 (TID 64) INFO Executor: Running task 1.0 in stage 17.0 (TID 64) 21/01/17 17:43:08.855 Executor task launch worker for task 2.0 in stage 17.0 (TID 65) INFO Executor: Running task 2.0 in stage 17.0 (TID 65) 21/01/17 17:43:08.855 Executor task launch worker for task 1.0 in stage 17.0 (TID 64) DEBUG ExecutorMetricsPoller: stageTCMP: (17, 0) -> 1 21/01/17 17:43:08.856 Executor task launch worker for task 1.0 in stage 17.0 (TID 64) DEBUG BlockManager: Getting local block broadcast_9 21/01/17 17:43:08.856 Executor task launch worker for task 1.0 in stage 17.0 (TID 64) DEBUG BlockManager: Level for block broadcast_9 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:08.857 Executor task launch worker for task 2.0 in stage 17.0 (TID 65) DEBUG ExecutorMetricsPoller: stageTCMP: (17, 0) -> 2 21/01/17 17:43:08.857 Executor task launch worker for task 3.0 in stage 17.0 (TID 66) DEBUG ExecutorMetricsPoller: stageTCMP: (17, 0) -> 3 21/01/17 17:43:08.857 Executor task launch worker for task 0.0 in stage 17.0 (TID 63) DEBUG ExecutorMetricsPoller: stageTCMP: (17, 0) -> 4 21/01/17 17:43:08.857 Executor task launch worker for task 3.0 in stage 3.0 (TID 14) DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 0 21/01/17 17:43:08.853 Executor task launch worker for task 0.0 in stage 3.0 (TID 15) DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 0 21/01/17 17:43:08.860 Executor task launch worker for task 3.0 in stage 3.0 (TID 14) DEBUG MapOutputTrackerMaster: Convert map statuses for shuffle 0, mappers 0-4, partitions 3-4 21/01/17 17:43:08.861 Executor task launch worker for task 0.0 in stage 3.0 (TID 15) DEBUG MapOutputTrackerMaster: Convert map statuses for shuffle 0, mappers 0-4, partitions 0-1 21/01/17 17:43:08.853 Executor task launch worker for task 1.0 in stage 3.0 (TID 12) DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 0 21/01/17 17:43:08.861 Executor task launch worker for task 2.0 in stage 3.0 (TID 13) DEBUG BlockManager: Block rdd_21_2 is unknown by block manager master 21/01/17 17:43:08.861 Executor task launch worker for task 1.0 in stage 3.0 (TID 12) DEBUG MapOutputTrackerMaster: Convert map statuses for shuffle 0, mappers 0-4, partitions 1-2 21/01/17 17:43:08.861 Executor task launch worker for task 2.0 in stage 3.0 (TID 13) DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 0 21/01/17 17:43:08.864 Executor task launch worker for task 2.0 in stage 3.0 (TID 13) DEBUG MapOutputTrackerMaster: Convert map statuses for shuffle 0, mappers 0-4, partitions 2-3 21/01/17 17:43:08.875 Executor task launch worker for task 1.0 in stage 3.0 (TID 12) DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 21/01/17 17:43:08.877 block-manager-storage-async-thread-pool-14 DEBUG BlockManagerStorageEndpoint: Sent response: 0 to 192.168.1.225:41897 21/01/17 17:43:08.879 Executor task launch worker for task 3.0 in stage 3.0 (TID 14) DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 21/01/17 17:43:08.879 Spark Context Cleaner DEBUG ContextCleaner: Cleaned broadcast 5 21/01/17 17:43:08.879 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(107) 21/01/17 17:43:08.879 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 107 21/01/17 17:43:08.880 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 107 21/01/17 17:43:08.880 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(31) 21/01/17 17:43:08.880 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 31 21/01/17 17:43:08.880 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 31 21/01/17 17:43:08.880 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(159) 21/01/17 17:43:08.880 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 159 21/01/17 17:43:08.880 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 159 21/01/17 17:43:08.880 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(145) 21/01/17 17:43:08.880 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 145 21/01/17 17:43:08.880 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 145 21/01/17 17:43:08.880 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(106) 21/01/17 17:43:08.880 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 106 21/01/17 17:43:08.880 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 106 21/01/17 17:43:08.880 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(56) 21/01/17 17:43:08.880 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 56 21/01/17 17:43:08.880 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 56 21/01/17 17:43:08.880 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(214) 21/01/17 17:43:08.880 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 214 21/01/17 17:43:08.880 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 214 21/01/17 17:43:08.880 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(171) 21/01/17 17:43:08.880 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 171 21/01/17 17:43:08.880 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 171 21/01/17 17:43:08.880 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(180) 21/01/17 17:43:08.880 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 180 21/01/17 17:43:08.880 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 180 21/01/17 17:43:08.880 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanBroadcast(6) 21/01/17 17:43:08.880 Spark Context Cleaner DEBUG ContextCleaner: Cleaning broadcast 6 21/01/17 17:43:08.880 Spark Context Cleaner DEBUG TorrentBroadcast: Unpersisting TorrentBroadcast 6 21/01/17 17:43:08.881 Executor task launch worker for task 1.0 in stage 17.0 (TID 64) DEBUG BlockManager: Getting local block rdd_55_1 21/01/17 17:43:08.881 Executor task launch worker for task 1.0 in stage 17.0 (TID 64) DEBUG BlockManager: Block rdd_55_1 was not found 21/01/17 17:43:08.881 Executor task launch worker for task 1.0 in stage 17.0 (TID 64) DEBUG BlockManager: Getting remote block rdd_55_1 21/01/17 17:43:08.883 Executor task launch worker for task 0.0 in stage 3.0 (TID 15) DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 21/01/17 17:43:08.883 Executor task launch worker for task 2.0 in stage 17.0 (TID 65) DEBUG BlockManager: Getting local block rdd_55_2 21/01/17 17:43:08.883 Executor task launch worker for task 2.0 in stage 17.0 (TID 65) DEBUG BlockManager: Block rdd_55_2 was not found 21/01/17 17:43:08.883 Executor task launch worker for task 2.0 in stage 17.0 (TID 65) DEBUG BlockManager: Getting remote block rdd_55_2 21/01/17 17:43:08.884 Executor task launch worker for task 0.0 in stage 17.0 (TID 63) DEBUG BlockManager: Getting local block rdd_55_0 21/01/17 17:43:08.884 Executor task launch worker for task 0.0 in stage 17.0 (TID 63) DEBUG BlockManager: Block rdd_55_0 was not found 21/01/17 17:43:08.884 Executor task launch worker for task 0.0 in stage 17.0 (TID 63) DEBUG BlockManager: Getting remote block rdd_55_0 21/01/17 17:43:08.884 Executor task launch worker for task 2.0 in stage 17.0 (TID 65) DEBUG BlockManager: Block rdd_55_2 is unknown by block manager master 21/01/17 17:43:08.884 Executor task launch worker for task 2.0 in stage 17.0 (TID 65) DEBUG BlockManager: Getting local block rdd_24_2 21/01/17 17:43:08.884 Executor task launch worker for task 2.0 in stage 17.0 (TID 65) DEBUG BlockManager: Level for block rdd_24_2 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:08.884 Executor task launch worker for task 2.0 in stage 17.0 (TID 65) INFO BlockManager: Found block rdd_24_2 locally 21/01/17 17:43:08.885 Executor task launch worker for task 2.0 in stage 17.0 (TID 65) DEBUG BlockManager: Getting local block rdd_24_2 21/01/17 17:43:08.885 Executor task launch worker for task 2.0 in stage 17.0 (TID 65) DEBUG BlockManager: Level for block rdd_24_2 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:08.885 Executor task launch worker for task 2.0 in stage 17.0 (TID 65) INFO BlockManager: Found block rdd_24_2 locally 21/01/17 17:43:08.885 Executor task launch worker for task 2.0 in stage 17.0 (TID 65) DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 4 21/01/17 17:43:08.885 Executor task launch worker for task 2.0 in stage 17.0 (TID 65) DEBUG MapOutputTrackerMaster: Convert map statuses for shuffle 4, mappers 0-10, partitions 2-3 21/01/17 17:43:08.885 Executor task launch worker for task 0.0 in stage 17.0 (TID 63) DEBUG BlockManager: Block rdd_55_0 is unknown by block manager master 21/01/17 17:43:08.885 Executor task launch worker for task 0.0 in stage 17.0 (TID 63) DEBUG BlockManager: Getting local block rdd_24_0 21/01/17 17:43:08.885 Executor task launch worker for task 0.0 in stage 17.0 (TID 63) DEBUG BlockManager: Level for block rdd_24_0 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:08.885 Executor task launch worker for task 0.0 in stage 17.0 (TID 63) INFO BlockManager: Found block rdd_24_0 locally 21/01/17 17:43:08.885 Executor task launch worker for task 0.0 in stage 17.0 (TID 63) DEBUG BlockManager: Getting local block rdd_24_0 21/01/17 17:43:08.885 Executor task launch worker for task 0.0 in stage 17.0 (TID 63) DEBUG BlockManager: Level for block rdd_24_0 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:08.885 Executor task launch worker for task 0.0 in stage 17.0 (TID 63) INFO BlockManager: Found block rdd_24_0 locally 21/01/17 17:43:08.885 Executor task launch worker for task 0.0 in stage 17.0 (TID 63) DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 4 21/01/17 17:43:08.885 Executor task launch worker for task 0.0 in stage 17.0 (TID 63) DEBUG MapOutputTrackerMaster: Convert map statuses for shuffle 4, mappers 0-10, partitions 0-1 21/01/17 17:43:08.886 Executor task launch worker for task 0.0 in stage 17.0 (TID 63) DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 21/01/17 17:43:08.886 Executor task launch worker for task 0.0 in stage 17.0 (TID 63) INFO ShuffleBlockFetcherIterator: Getting 3 (684.0 B) non-empty blocks including 3 (684.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 21/01/17 17:43:08.886 Executor task launch worker for task 0.0 in stage 17.0 (TID 63) INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 21/01/17 17:43:08.886 Executor task launch worker for task 0.0 in stage 17.0 (TID 63) DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_4_33_0,0), (shuffle_4_34_0,1), (shuffle_4_35_0,2) 21/01/17 17:43:08.886 Executor task launch worker for task 0.0 in stage 17.0 (TID 63) DEBUG BlockManager: Getting local shuffle block shuffle_4_33_0 21/01/17 17:43:08.886 Executor task launch worker for task 0.0 in stage 3.0 (TID 15) INFO ShuffleBlockFetcherIterator: Getting 0 (0.0 B) non-empty blocks including 0 (0.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 21/01/17 17:43:08.886 Executor task launch worker for task 0.0 in stage 17.0 (TID 63) DEBUG BlockManager: Getting local shuffle block shuffle_4_34_0 21/01/17 17:43:08.887 Executor task launch worker for task 0.0 in stage 17.0 (TID 63) DEBUG BlockManager: Getting local shuffle block shuffle_4_35_0 21/01/17 17:43:08.887 Executor task launch worker for task 0.0 in stage 17.0 (TID 63) DEBUG ShuffleBlockFetcherIterator: Got local blocks in 1 ms 21/01/17 17:43:08.887 Executor task launch worker for task 2.0 in stage 17.0 (TID 65) DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 21/01/17 17:43:08.888 Executor task launch worker for task 0.0 in stage 3.0 (TID 15) INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 9 ms 21/01/17 17:43:08.888 Executor task launch worker for task 2.0 in stage 17.0 (TID 65) INFO ShuffleBlockFetcherIterator: Getting 3 (707.0 B) non-empty blocks including 3 (707.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 21/01/17 17:43:08.888 Executor task launch worker for task 2.0 in stage 17.0 (TID 65) INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 21/01/17 17:43:08.888 Executor task launch worker for task 2.0 in stage 17.0 (TID 65) DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_4_33_2,0), (shuffle_4_34_2,1), (shuffle_4_35_2,2) 21/01/17 17:43:08.888 Executor task launch worker for task 2.0 in stage 17.0 (TID 65) DEBUG BlockManager: Getting local shuffle block shuffle_4_33_2 21/01/17 17:43:08.888 Executor task launch worker for task 0.0 in stage 3.0 (TID 15) DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: 21/01/17 17:43:08.888 Executor task launch worker for task 2.0 in stage 17.0 (TID 65) DEBUG BlockManager: Getting local shuffle block shuffle_4_34_2 21/01/17 17:43:08.888 Executor task launch worker for task 2.0 in stage 17.0 (TID 65) DEBUG BlockManager: Getting local shuffle block shuffle_4_35_2 21/01/17 17:43:08.888 Executor task launch worker for task 2.0 in stage 17.0 (TID 65) DEBUG ShuffleBlockFetcherIterator: Got local blocks in 0 ms 21/01/17 17:43:08.888 Executor task launch worker for task 0.0 in stage 3.0 (TID 15) DEBUG ShuffleBlockFetcherIterator: Got local blocks in 10 ms 21/01/17 17:43:08.889 Thread-3 INFO CodeGenerator: Code generated in 65.083199 ms 21/01/17 17:43:08.889 Thread-3 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$doExecute$4$adapted 21/01/17 17:43:08.890 Executor task launch worker for task 3.0 in stage 17.0 (TID 66) DEBUG BlockManager: Getting local block rdd_55_3 21/01/17 17:43:08.890 Executor task launch worker for task 3.0 in stage 17.0 (TID 66) DEBUG BlockManager: Block rdd_55_3 was not found 21/01/17 17:43:08.890 Executor task launch worker for task 3.0 in stage 17.0 (TID 66) DEBUG BlockManager: Getting remote block rdd_55_3 21/01/17 17:43:08.890 Executor task launch worker for task 3.0 in stage 17.0 (TID 66) DEBUG BlockManager: Block rdd_55_3 is unknown by block manager master 21/01/17 17:43:08.890 Executor task launch worker for task 2.0 in stage 17.0 (TID 65) DEBUG TaskMemoryManager: Task 65 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@4cc1db24 21/01/17 17:43:08.890 Executor task launch worker for task 3.0 in stage 17.0 (TID 66) DEBUG BlockManager: Getting local block rdd_24_3 21/01/17 17:43:08.890 Thread-3 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$doExecute$4$adapted) is now cleaned +++ 21/01/17 17:43:08.890 Executor task launch worker for task 3.0 in stage 17.0 (TID 66) DEBUG BlockManager: Level for block rdd_24_3 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:08.890 Executor task launch worker for task 3.0 in stage 17.0 (TID 66) INFO BlockManager: Found block rdd_24_3 locally 21/01/17 17:43:08.890 Executor task launch worker for task 3.0 in stage 17.0 (TID 66) DEBUG BlockManager: Getting local block rdd_24_3 21/01/17 17:43:08.890 Executor task launch worker for task 1.0 in stage 17.0 (TID 64) DEBUG BlockManager: Block rdd_55_1 is unknown by block manager master 21/01/17 17:43:08.891 Executor task launch worker for task 3.0 in stage 17.0 (TID 66) DEBUG BlockManager: Level for block rdd_24_3 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:08.891 Executor task launch worker for task 3.0 in stage 17.0 (TID 66) INFO BlockManager: Found block rdd_24_3 locally 21/01/17 17:43:08.891 Executor task launch worker for task 3.0 in stage 17.0 (TID 66) DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 4 21/01/17 17:43:08.891 Executor task launch worker for task 3.0 in stage 17.0 (TID 66) DEBUG MapOutputTrackerMaster: Convert map statuses for shuffle 4, mappers 0-10, partitions 3-4 21/01/17 17:43:08.891 Executor task launch worker for task 3.0 in stage 17.0 (TID 66) DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 21/01/17 17:43:08.891 Executor task launch worker for task 3.0 in stage 17.0 (TID 66) INFO ShuffleBlockFetcherIterator: Getting 3 (684.0 B) non-empty blocks including 3 (684.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 21/01/17 17:43:08.892 Executor task launch worker for task 3.0 in stage 17.0 (TID 66) INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 21/01/17 17:43:08.892 Executor task launch worker for task 3.0 in stage 17.0 (TID 66) DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_4_33_3,0), (shuffle_4_34_3,1), (shuffle_4_35_3,2) 21/01/17 17:43:08.892 Executor task launch worker for task 3.0 in stage 17.0 (TID 66) DEBUG BlockManager: Getting local shuffle block shuffle_4_33_3 21/01/17 17:43:08.892 Executor task launch worker for task 3.0 in stage 17.0 (TID 66) DEBUG BlockManager: Getting local shuffle block shuffle_4_34_3 21/01/17 17:43:08.892 block-manager-storage-async-thread-pool-15 DEBUG BlockManagerStorageEndpoint: removing broadcast 6 21/01/17 17:43:08.892 block-manager-storage-async-thread-pool-15 DEBUG BlockManager: Removing broadcast 6 21/01/17 17:43:08.892 block-manager-storage-async-thread-pool-15 DEBUG BlockManager: Removing block broadcast_6_piece0 21/01/17 17:43:08.878 Executor task launch worker for task 2.0 in stage 3.0 (TID 13) DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 21/01/17 17:43:08.900 Executor task launch worker for task 3.0 in stage 3.0 (TID 14) INFO ShuffleBlockFetcherIterator: Getting 2 (378.0 B) non-empty blocks including 2 (378.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 21/01/17 17:43:08.900 Executor task launch worker for task 3.0 in stage 3.0 (TID 14) INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 21 ms 21/01/17 17:43:08.900 Executor task launch worker for task 3.0 in stage 3.0 (TID 14) DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_0_8_3,0), (shuffle_0_11_3,3) 21/01/17 17:43:08.901 Executor task launch worker for task 3.0 in stage 3.0 (TID 14) DEBUG BlockManager: Getting local shuffle block shuffle_0_8_3 21/01/17 17:43:08.903 Executor task launch worker for task 2.0 in stage 3.0 (TID 13) INFO ShuffleBlockFetcherIterator: Getting 2 (378.0 B) non-empty blocks including 2 (378.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 21/01/17 17:43:08.903 Executor task launch worker for task 2.0 in stage 3.0 (TID 13) INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 25 ms 21/01/17 17:43:08.903 Executor task launch worker for task 2.0 in stage 3.0 (TID 13) DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_0_8_2,0), (shuffle_0_10_2,2) 21/01/17 17:43:08.903 Executor task launch worker for task 2.0 in stage 3.0 (TID 13) DEBUG BlockManager: Getting local shuffle block shuffle_0_8_2 21/01/17 17:43:08.891 Executor task launch worker for task 1.0 in stage 17.0 (TID 64) DEBUG BlockManager: Getting local block rdd_24_1 21/01/17 17:43:08.905 Executor task launch worker for task 1.0 in stage 17.0 (TID 64) DEBUG BlockManager: Level for block rdd_24_1 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:08.905 Executor task launch worker for task 1.0 in stage 17.0 (TID 64) INFO BlockManager: Found block rdd_24_1 locally 21/01/17 17:43:08.905 Executor task launch worker for task 1.0 in stage 17.0 (TID 64) DEBUG BlockManager: Getting local block rdd_24_1 21/01/17 17:43:08.893 block-manager-storage-async-thread-pool-15 DEBUG MemoryStore: Block broadcast_6_piece0 of size 16900 dropped from memory (free 383636935) 21/01/17 17:43:08.892 Executor task launch worker for task 3.0 in stage 17.0 (TID 66) DEBUG BlockManager: Getting local shuffle block shuffle_4_35_3 21/01/17 17:43:08.905 Executor task launch worker for task 2.0 in stage 17.0 (TID 65) DEBUG TaskMemoryManager: Task 65 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@162e1fc8 21/01/17 17:43:08.906 Executor task launch worker for task 3.0 in stage 17.0 (TID 66) DEBUG ShuffleBlockFetcherIterator: Got local blocks in 14 ms 21/01/17 17:43:08.892 Executor task launch worker for task 0.0 in stage 17.0 (TID 63) DEBUG TaskMemoryManager: Task 63 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@479fa26b 21/01/17 17:43:08.906 Executor task launch worker for task 1.0 in stage 3.0 (TID 12) INFO ShuffleBlockFetcherIterator: Getting 3 (567.0 B) non-empty blocks including 3 (567.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 21/01/17 17:43:08.906 Executor task launch worker for task 1.0 in stage 3.0 (TID 12) INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 32 ms 21/01/17 17:43:08.907 Executor task launch worker for task 1.0 in stage 3.0 (TID 12) DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_0_8_1,0), (shuffle_0_9_1,1), (shuffle_0_10_1,2) 21/01/17 17:43:08.907 Executor task launch worker for task 1.0 in stage 3.0 (TID 12) DEBUG BlockManager: Getting local shuffle block shuffle_0_8_1 21/01/17 17:43:08.907 Executor task launch worker for task 0.0 in stage 17.0 (TID 63) DEBUG TaskMemoryManager: Task 63 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@4c69e6d7 21/01/17 17:43:08.907 Executor task launch worker for task 3.0 in stage 17.0 (TID 66) DEBUG TaskMemoryManager: Task 66 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@739e90f4 21/01/17 17:43:08.908 Executor task launch worker for task 0.0 in stage 17.0 (TID 63) DEBUG TaskMemoryManager: Task 63 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@bae43f3 21/01/17 17:43:08.908 Executor task launch worker for task 3.0 in stage 17.0 (TID 66) DEBUG TaskMemoryManager: Task 66 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@f638a11 21/01/17 17:43:08.908 Executor task launch worker for task 0.0 in stage 17.0 (TID 63) INFO MemoryStore: Block rdd_55_0 stored as values in memory (estimated size 88.0 B, free 365.9 MiB) 21/01/17 17:43:08.908 Executor task launch worker for task 3.0 in stage 17.0 (TID 66) DEBUG TaskMemoryManager: Task 66 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@2c8d0751 21/01/17 17:43:08.908 Executor task launch worker for task 3.0 in stage 17.0 (TID 66) INFO MemoryStore: Block rdd_55_3 stored as values in memory (estimated size 16.0 B, free 365.9 MiB) 21/01/17 17:43:08.908 Executor task launch worker for task 2.0 in stage 17.0 (TID 65) DEBUG TaskMemoryManager: Task 65 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@14adbc40 21/01/17 17:43:08.909 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master broadcast_6_piece0 for BlockManagerId(driver, 192.168.1.225, 40985, None) 21/01/17 17:43:08.909 Executor task launch worker for task 2.0 in stage 17.0 (TID 65) INFO MemoryStore: Block rdd_55_2 stored as values in memory (estimated size 88.0 B, free 365.9 MiB) 21/01/17 17:43:08.905 Executor task launch worker for task 1.0 in stage 17.0 (TID 64) DEBUG BlockManager: Level for block rdd_24_1 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:08.909 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Removed broadcast_6_piece0 on 192.168.1.225:40985 in memory (size: 16.5 KiB, free: 366.2 MiB) 21/01/17 17:43:08.909 Executor task launch worker for task 1.0 in stage 17.0 (TID 64) INFO BlockManager: Found block rdd_24_1 locally 21/01/17 17:43:08.909 Executor task launch worker for task 1.0 in stage 17.0 (TID 64) DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 4 21/01/17 17:43:08.909 Executor task launch worker for task 1.0 in stage 17.0 (TID 64) DEBUG MapOutputTrackerMaster: Convert map statuses for shuffle 4, mappers 0-10, partitions 1-2 21/01/17 17:43:08.909 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_55_0 for BlockManagerId(driver, 192.168.1.225, 40985, None) 21/01/17 17:43:08.910 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_55_0 in memory on 192.168.1.225:40985 (size: 88.0 B, free: 366.2 MiB) 21/01/17 17:43:08.910 Executor task launch worker for task 1.0 in stage 17.0 (TID 64) DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 21/01/17 17:43:08.910 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_55_3 for BlockManagerId(driver, 192.168.1.225, 40985, None) 21/01/17 17:43:08.910 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_55_3 in memory on 192.168.1.225:40985 (size: 16.0 B, free: 366.2 MiB) 21/01/17 17:43:08.910 Executor task launch worker for task 0.0 in stage 17.0 (TID 63) DEBUG BlockManagerMaster: Updated info of block rdd_55_0 21/01/17 17:43:08.910 Executor task launch worker for task 0.0 in stage 17.0 (TID 63) DEBUG BlockManager: Told master about block rdd_55_0 21/01/17 17:43:08.910 Executor task launch worker for task 0.0 in stage 17.0 (TID 63) DEBUG BlockManager: Put block rdd_55_0 locally took 25 ms 21/01/17 17:43:08.910 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_55_2 for BlockManagerId(driver, 192.168.1.225, 40985, None) 21/01/17 17:43:08.910 Executor task launch worker for task 0.0 in stage 17.0 (TID 63) DEBUG BlockManager: Putting block rdd_55_0 without replication took 25 ms 21/01/17 17:43:08.910 Executor task launch worker for task 0.0 in stage 17.0 (TID 63) DEBUG BlockManager: Getting local block rdd_55_0 21/01/17 17:43:08.910 Executor task launch worker for task 0.0 in stage 17.0 (TID 63) DEBUG BlockManager: Level for block rdd_55_0 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:08.910 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_55_2 in memory on 192.168.1.225:40985 (size: 88.0 B, free: 366.2 MiB) 21/01/17 17:43:08.911 Executor task launch worker for task 2.0 in stage 17.0 (TID 65) DEBUG BlockManagerMaster: Updated info of block rdd_55_2 21/01/17 17:43:08.911 Executor task launch worker for task 2.0 in stage 17.0 (TID 65) DEBUG BlockManager: Told master about block rdd_55_2 21/01/17 17:43:08.911 Executor task launch worker for task 2.0 in stage 17.0 (TID 65) DEBUG BlockManager: Put block rdd_55_2 locally took 26 ms 21/01/17 17:43:08.911 Executor task launch worker for task 2.0 in stage 17.0 (TID 65) DEBUG BlockManager: Putting block rdd_55_2 without replication took 26 ms 21/01/17 17:43:08.911 Executor task launch worker for task 2.0 in stage 17.0 (TID 65) DEBUG BlockManager: Getting local block rdd_55_2 21/01/17 17:43:08.911 Executor task launch worker for task 2.0 in stage 17.0 (TID 65) DEBUG BlockManager: Level for block rdd_55_2 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:08.911 Executor task launch worker for task 3.0 in stage 17.0 (TID 66) DEBUG BlockManagerMaster: Updated info of block rdd_55_3 21/01/17 17:43:08.911 Executor task launch worker for task 3.0 in stage 17.0 (TID 66) DEBUG BlockManager: Told master about block rdd_55_3 21/01/17 17:43:08.911 Executor task launch worker for task 3.0 in stage 17.0 (TID 66) DEBUG BlockManager: Put block rdd_55_3 locally took 20 ms 21/01/17 17:43:08.911 Executor task launch worker for task 3.0 in stage 17.0 (TID 66) DEBUG BlockManager: Putting block rdd_55_3 without replication took 20 ms 21/01/17 17:43:08.911 Executor task launch worker for task 3.0 in stage 17.0 (TID 66) DEBUG BlockManager: Getting local block rdd_55_3 21/01/17 17:43:08.911 Executor task launch worker for task 3.0 in stage 17.0 (TID 66) DEBUG BlockManager: Level for block rdd_55_3 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:08.911 Executor task launch worker for task 0.0 in stage 3.0 (TID 15) DEBUG TaskMemoryManager: Task 15 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@2c95c53f 21/01/17 17:43:08.912 Executor task launch worker for task 0.0 in stage 17.0 (TID 63) INFO Executor: Finished task 0.0 in stage 17.0 (TID 63). 2256 bytes result sent to driver 21/01/17 17:43:08.912 Executor task launch worker for task 0.0 in stage 17.0 (TID 63) DEBUG ExecutorMetricsPoller: stageTCMP: (17, 0) -> 3 21/01/17 17:43:08.912 Executor task launch worker for task 0.0 in stage 3.0 (TID 15) INFO MemoryStore: Block rdd_21_0 stored as values in memory (estimated size 16.0 B, free 366.2 MiB) 21/01/17 17:43:08.910 Executor task launch worker for task 1.0 in stage 17.0 (TID 64) INFO ShuffleBlockFetcherIterator: Getting 3 (730.0 B) non-empty blocks including 3 (730.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 21/01/17 17:43:08.912 Executor task launch worker for task 1.0 in stage 17.0 (TID 64) INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 2 ms 21/01/17 17:43:08.912 Executor task launch worker for task 1.0 in stage 17.0 (TID 64) DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_4_33_1,0), (shuffle_4_34_1,1), (shuffle_4_35_1,2) 21/01/17 17:43:08.912 Executor task launch worker for task 1.0 in stage 17.0 (TID 64) DEBUG BlockManager: Getting local shuffle block shuffle_4_33_1 21/01/17 17:43:08.912 Executor task launch worker for task 3.0 in stage 17.0 (TID 66) INFO Executor: Finished task 3.0 in stage 17.0 (TID 66). 2213 bytes result sent to driver 21/01/17 17:43:08.912 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_21_0 for BlockManagerId(driver, 192.168.1.225, 46295, None) 21/01/17 17:43:08.912 dispatcher-event-loop-1 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_17.0, runningTasks: 3 21/01/17 17:43:08.912 Executor task launch worker for task 3.0 in stage 17.0 (TID 66) DEBUG ExecutorMetricsPoller: stageTCMP: (17, 0) -> 2 21/01/17 17:43:08.913 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_21_0 in memory on 192.168.1.225:46295 (size: 16.0 B, free: 366.3 MiB) 21/01/17 17:43:08.913 dispatcher-event-loop-1 INFO TaskSetManager: Starting task 4.0 in stage 17.0 (TID 67) (192.168.1.225, executor driver, partition 4, PROCESS_LOCAL, 7433 bytes) taskResourceAssignments Map() 21/01/17 17:43:08.913 block-manager-storage-async-thread-pool-15 DEBUG BlockManagerMaster: Updated info of block broadcast_6_piece0 21/01/17 17:43:08.913 block-manager-storage-async-thread-pool-15 DEBUG BlockManager: Told master about block broadcast_6_piece0 21/01/17 17:43:08.913 block-manager-storage-async-thread-pool-15 DEBUG BlockManager: Removing block broadcast_6 21/01/17 17:43:08.913 block-manager-storage-async-thread-pool-15 DEBUG MemoryStore: Block broadcast_6 of size 36400 dropped from memory (free 383673143) 21/01/17 17:43:08.913 Executor task launch worker for task 4.0 in stage 17.0 (TID 67) INFO Executor: Running task 4.0 in stage 17.0 (TID 67) 21/01/17 17:43:08.913 dispatcher-event-loop-1 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_17.0, runningTasks: 3 21/01/17 17:43:08.914 Executor task launch worker for task 4.0 in stage 17.0 (TID 67) DEBUG ExecutorMetricsPoller: stageTCMP: (17, 0) -> 3 21/01/17 17:43:08.912 Executor task launch worker for task 1.0 in stage 17.0 (TID 64) DEBUG BlockManager: Getting local shuffle block shuffle_4_34_1 21/01/17 17:43:08.914 Executor task launch worker for task 1.0 in stage 17.0 (TID 64) DEBUG BlockManager: Getting local shuffle block shuffle_4_35_1 21/01/17 17:43:08.914 Executor task launch worker for task 1.0 in stage 17.0 (TID 64) DEBUG ShuffleBlockFetcherIterator: Got local blocks in 4 ms 21/01/17 17:43:08.914 dispatcher-event-loop-1 INFO TaskSetManager: Starting task 5.0 in stage 17.0 (TID 68) (192.168.1.225, executor driver, partition 5, PROCESS_LOCAL, 7433 bytes) taskResourceAssignments Map() 21/01/17 17:43:08.905 dag-scheduler-event-loop DEBUG DAGScheduler: Merging stage rdd profiles: Set() 21/01/17 17:43:08.915 task-result-getter-1 INFO TaskSetManager: Finished task 3.0 in stage 17.0 (TID 66) in 62 ms on 192.168.1.225 (executor driver) (1/10) 21/01/17 17:43:08.915 dag-scheduler-event-loop INFO DAGScheduler: Registering RDD 14 (collect at ClusteringMetrics.scala:97) as input to shuffle 1 21/01/17 17:43:08.915 task-result-getter-0 INFO TaskSetManager: Finished task 0.0 in stage 17.0 (TID 63) in 123 ms on 192.168.1.225 (executor driver) (2/10) 21/01/17 17:43:08.914 Executor task launch worker for task 2.0 in stage 17.0 (TID 65) INFO Executor: Finished task 2.0 in stage 17.0 (TID 65). 2256 bytes result sent to driver 21/01/17 17:43:08.916 Executor task launch worker for task 2.0 in stage 17.0 (TID 65) DEBUG ExecutorMetricsPoller: stageTCMP: (17, 0) -> 2 21/01/17 17:43:08.916 Executor task launch worker for task 1.0 in stage 17.0 (TID 64) DEBUG TaskMemoryManager: Task 64 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@3cda1c 21/01/17 17:43:08.916 dag-scheduler-event-loop INFO DAGScheduler: Got map stage job 2 (collect at ClusteringMetrics.scala:97) with 4 output partitions 21/01/17 17:43:08.916 dispatcher-event-loop-2 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_17.0, runningTasks: 3 21/01/17 17:43:08.916 Executor task launch worker for task 1.0 in stage 17.0 (TID 64) DEBUG TaskMemoryManager: Task 64 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@403f5f51 21/01/17 17:43:08.917 dag-scheduler-event-loop INFO DAGScheduler: Final stage: ShuffleMapStage 3 (collect at ClusteringMetrics.scala:97) 21/01/17 17:43:08.916 dispatcher-event-loop-2 INFO TaskSetManager: Starting task 6.0 in stage 17.0 (TID 69) (192.168.1.225, executor driver, partition 6, PROCESS_LOCAL, 7433 bytes) taskResourceAssignments Map() 21/01/17 17:43:08.917 Executor task launch worker for task 1.0 in stage 17.0 (TID 64) DEBUG TaskMemoryManager: Task 64 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@6eba71a5 21/01/17 17:43:08.917 dag-scheduler-event-loop INFO DAGScheduler: Parents of final stage: List() 21/01/17 17:43:08.917 task-result-getter-2 INFO TaskSetManager: Finished task 2.0 in stage 17.0 (TID 65) in 87 ms on 192.168.1.225 (executor driver) (3/10) 21/01/17 17:43:08.917 Executor task launch worker for task 6.0 in stage 17.0 (TID 69) INFO Executor: Running task 6.0 in stage 17.0 (TID 69) 21/01/17 17:43:08.918 Executor task launch worker for task 6.0 in stage 17.0 (TID 69) DEBUG ExecutorMetricsPoller: stageTCMP: (17, 0) -> 3 21/01/17 17:43:08.918 dag-scheduler-event-loop INFO DAGScheduler: Missing parents: List() 21/01/17 17:43:08.919 Executor task launch worker for task 5.0 in stage 17.0 (TID 68) INFO Executor: Running task 5.0 in stage 17.0 (TID 68) 21/01/17 17:43:08.919 Executor task launch worker for task 5.0 in stage 17.0 (TID 68) DEBUG ExecutorMetricsPoller: stageTCMP: (17, 0) -> 4 21/01/17 17:43:08.917 Executor task launch worker for task 1.0 in stage 17.0 (TID 64) INFO MemoryStore: Block rdd_55_1 stored as values in memory (estimated size 88.0 B, free 365.9 MiB) 21/01/17 17:43:08.921 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_55_1 for BlockManagerId(driver, 192.168.1.225, 40985, None) 21/01/17 17:43:08.921 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_55_1 in memory on 192.168.1.225:40985 (size: 88.0 B, free: 366.2 MiB) 21/01/17 17:43:08.923 dag-scheduler-event-loop DEBUG DAGScheduler: submitStage(ShuffleMapStage 3 (name=collect at ClusteringMetrics.scala:97;jobs=2)) 21/01/17 17:43:08.923 dag-scheduler-event-loop DEBUG DAGScheduler: missing: List() 21/01/17 17:43:08.923 Executor task launch worker for task 4.0 in stage 17.0 (TID 67) DEBUG BlockManager: Getting local block rdd_55_4 21/01/17 17:43:08.923 dag-scheduler-event-loop INFO DAGScheduler: Submitting ShuffleMapStage 3 (MapPartitionsRDD[14] at collect at ClusteringMetrics.scala:97), which has no missing parents 21/01/17 17:43:08.923 Executor task launch worker for task 4.0 in stage 17.0 (TID 67) DEBUG BlockManager: Block rdd_55_4 was not found 21/01/17 17:43:08.923 Executor task launch worker for task 4.0 in stage 17.0 (TID 67) DEBUG BlockManager: Getting remote block rdd_55_4 21/01/17 17:43:08.923 Executor task launch worker for task 2.0 in stage 3.0 (TID 13) DEBUG BlockManager: Getting local shuffle block shuffle_0_10_2 21/01/17 17:43:08.923 Executor task launch worker for task 6.0 in stage 17.0 (TID 69) DEBUG BlockManager: Getting local block rdd_55_6 21/01/17 17:43:08.923 Executor task launch worker for task 6.0 in stage 17.0 (TID 69) DEBUG BlockManager: Block rdd_55_6 was not found 21/01/17 17:43:08.923 Executor task launch worker for task 3.0 in stage 3.0 (TID 14) DEBUG BlockManager: Getting local shuffle block shuffle_0_11_3 21/01/17 17:43:08.923 Executor task launch worker for task 6.0 in stage 17.0 (TID 69) DEBUG BlockManager: Getting remote block rdd_55_6 21/01/17 17:43:08.923 Executor task launch worker for task 1.0 in stage 17.0 (TID 64) DEBUG BlockManagerMaster: Updated info of block rdd_55_1 21/01/17 17:43:08.923 Executor task launch worker for task 1.0 in stage 17.0 (TID 64) DEBUG BlockManager: Told master about block rdd_55_1 21/01/17 17:43:08.923 Executor task launch worker for task 4.0 in stage 17.0 (TID 67) DEBUG BlockManager: Block rdd_55_4 is unknown by block manager master 21/01/17 17:43:08.924 Executor task launch worker for task 2.0 in stage 3.0 (TID 13) DEBUG ShuffleBlockFetcherIterator: Got local blocks in 45 ms 21/01/17 17:43:08.924 Executor task launch worker for task 3.0 in stage 3.0 (TID 14) DEBUG ShuffleBlockFetcherIterator: Got local blocks in 44 ms 21/01/17 17:43:08.924 Executor task launch worker for task 4.0 in stage 17.0 (TID 67) DEBUG BlockManager: Getting local block rdd_24_4 21/01/17 17:43:08.924 Executor task launch worker for task 4.0 in stage 17.0 (TID 67) DEBUG BlockManager: Level for block rdd_24_4 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:08.924 Executor task launch worker for task 4.0 in stage 17.0 (TID 67) INFO BlockManager: Found block rdd_24_4 locally 21/01/17 17:43:08.924 Executor task launch worker for task 4.0 in stage 17.0 (TID 67) DEBUG BlockManager: Getting local block rdd_24_4 21/01/17 17:43:08.924 Executor task launch worker for task 4.0 in stage 17.0 (TID 67) DEBUG BlockManager: Level for block rdd_24_4 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:08.924 Executor task launch worker for task 4.0 in stage 17.0 (TID 67) INFO BlockManager: Found block rdd_24_4 locally 21/01/17 17:43:08.924 Executor task launch worker for task 4.0 in stage 17.0 (TID 67) DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 4 21/01/17 17:43:08.924 Executor task launch worker for task 4.0 in stage 17.0 (TID 67) DEBUG MapOutputTrackerMaster: Convert map statuses for shuffle 4, mappers 0-10, partitions 4-5 21/01/17 17:43:08.923 Executor task launch worker for task 1.0 in stage 3.0 (TID 12) DEBUG BlockManager: Getting local shuffle block shuffle_0_9_1 21/01/17 17:43:08.924 Executor task launch worker for task 4.0 in stage 17.0 (TID 67) DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 21/01/17 17:43:08.925 Executor task launch worker for task 1.0 in stage 3.0 (TID 12) DEBUG BlockManager: Getting local shuffle block shuffle_0_10_1 21/01/17 17:43:08.923 dag-scheduler-event-loop DEBUG DAGScheduler: submitMissingTasks(ShuffleMapStage 3) 21/01/17 17:43:08.925 Executor task launch worker for task 1.0 in stage 3.0 (TID 12) DEBUG ShuffleBlockFetcherIterator: Got local blocks in 50 ms 21/01/17 17:43:08.925 Executor task launch worker for task 0.0 in stage 3.0 (TID 15) DEBUG BlockManagerMaster: Updated info of block rdd_21_0 21/01/17 17:43:08.925 Executor task launch worker for task 0.0 in stage 3.0 (TID 15) DEBUG BlockManager: Told master about block rdd_21_0 21/01/17 17:43:08.925 Executor task launch worker for task 4.0 in stage 17.0 (TID 67) INFO ShuffleBlockFetcherIterator: Getting 3 (684.0 B) non-empty blocks including 3 (684.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 21/01/17 17:43:08.925 Executor task launch worker for task 0.0 in stage 3.0 (TID 15) DEBUG BlockManager: Put block rdd_21_0 locally took 73 ms 21/01/17 17:43:08.925 Executor task launch worker for task 4.0 in stage 17.0 (TID 67) INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 21/01/17 17:43:08.925 Executor task launch worker for task 4.0 in stage 17.0 (TID 67) DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_4_33_4,0), (shuffle_4_34_4,1), (shuffle_4_35_4,2) 21/01/17 17:43:08.925 Executor task launch worker for task 4.0 in stage 17.0 (TID 67) DEBUG BlockManager: Getting local shuffle block shuffle_4_33_4 21/01/17 17:43:08.925 Executor task launch worker for task 4.0 in stage 17.0 (TID 67) DEBUG BlockManager: Getting local shuffle block shuffle_4_34_4 21/01/17 17:43:08.925 Executor task launch worker for task 4.0 in stage 17.0 (TID 67) DEBUG BlockManager: Getting local shuffle block shuffle_4_35_4 21/01/17 17:43:08.926 Executor task launch worker for task 4.0 in stage 17.0 (TID 67) DEBUG ShuffleBlockFetcherIterator: Got local blocks in 1 ms 21/01/17 17:43:08.926 Executor task launch worker for task 0.0 in stage 3.0 (TID 15) DEBUG BlockManager: Putting block rdd_21_0 without replication took 74 ms 21/01/17 17:43:08.926 Executor task launch worker for task 0.0 in stage 3.0 (TID 15) DEBUG BlockManager: Getting local block rdd_21_0 21/01/17 17:43:08.924 Executor task launch worker for task 1.0 in stage 17.0 (TID 64) DEBUG BlockManager: Put block rdd_55_1 locally took 32 ms 21/01/17 17:43:08.927 Executor task launch worker for task 1.0 in stage 17.0 (TID 64) DEBUG BlockManager: Putting block rdd_55_1 without replication took 36 ms 21/01/17 17:43:08.927 Executor task launch worker for task 1.0 in stage 17.0 (TID 64) DEBUG BlockManager: Getting local block rdd_55_1 21/01/17 17:43:08.927 Executor task launch worker for task 1.0 in stage 17.0 (TID 64) DEBUG BlockManager: Level for block rdd_55_1 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:08.927 Executor task launch worker for task 4.0 in stage 17.0 (TID 67) DEBUG TaskMemoryManager: Task 67 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@300faf6a 21/01/17 17:43:08.928 Executor task launch worker for task 4.0 in stage 17.0 (TID 67) DEBUG TaskMemoryManager: Task 67 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@49cd77d5 21/01/17 17:43:08.928 Executor task launch worker for task 1.0 in stage 17.0 (TID 64) INFO Executor: Finished task 1.0 in stage 17.0 (TID 64). 2256 bytes result sent to driver 21/01/17 17:43:08.928 Executor task launch worker for task 4.0 in stage 17.0 (TID 67) DEBUG TaskMemoryManager: Task 67 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@44bc4cde 21/01/17 17:43:08.928 Executor task launch worker for task 6.0 in stage 17.0 (TID 69) DEBUG BlockManager: Block rdd_55_6 is unknown by block manager master 21/01/17 17:43:08.928 Executor task launch worker for task 4.0 in stage 17.0 (TID 67) INFO MemoryStore: Block rdd_55_4 stored as values in memory (estimated size 16.0 B, free 365.9 MiB) 21/01/17 17:43:08.928 Executor task launch worker for task 0.0 in stage 3.0 (TID 15) DEBUG BlockManager: Level for block rdd_21_0 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:08.929 dispatcher-event-loop-1 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_17.0, runningTasks: 3 21/01/17 17:43:08.929 dispatcher-event-loop-1 INFO TaskSetManager: Starting task 7.0 in stage 17.0 (TID 70) (192.168.1.225, executor driver, partition 7, PROCESS_LOCAL, 7433 bytes) taskResourceAssignments Map() 21/01/17 17:43:08.929 Executor task launch worker for task 1.0 in stage 17.0 (TID 64) DEBUG ExecutorMetricsPoller: stageTCMP: (17, 0) -> 3 21/01/17 17:43:08.928 Executor task launch worker for task 6.0 in stage 17.0 (TID 69) DEBUG BlockManager: Getting local block rdd_24_6 21/01/17 17:43:08.929 Executor task launch worker for task 7.0 in stage 17.0 (TID 70) INFO Executor: Running task 7.0 in stage 17.0 (TID 70) 21/01/17 17:43:08.929 Executor task launch worker for task 6.0 in stage 17.0 (TID 69) DEBUG BlockManager: Level for block rdd_24_6 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:08.930 Executor task launch worker for task 6.0 in stage 17.0 (TID 69) INFO BlockManager: Found block rdd_24_6 locally 21/01/17 17:43:08.930 Executor task launch worker for task 6.0 in stage 17.0 (TID 69) DEBUG BlockManager: Getting local block rdd_24_6 21/01/17 17:43:08.930 task-result-getter-3 INFO TaskSetManager: Finished task 1.0 in stage 17.0 (TID 64) in 137 ms on 192.168.1.225 (executor driver) (4/10) 21/01/17 17:43:08.930 Executor task launch worker for task 6.0 in stage 17.0 (TID 69) DEBUG BlockManager: Level for block rdd_24_6 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:08.930 Executor task launch worker for task 6.0 in stage 17.0 (TID 69) INFO BlockManager: Found block rdd_24_6 locally 21/01/17 17:43:08.930 Executor task launch worker for task 7.0 in stage 17.0 (TID 70) DEBUG ExecutorMetricsPoller: stageTCMP: (17, 0) -> 4 21/01/17 17:43:08.931 Executor task launch worker for task 5.0 in stage 17.0 (TID 68) DEBUG BlockManager: Getting local block rdd_55_5 21/01/17 17:43:08.932 Executor task launch worker for task 5.0 in stage 17.0 (TID 68) DEBUG BlockManager: Block rdd_55_5 was not found 21/01/17 17:43:08.932 Executor task launch worker for task 5.0 in stage 17.0 (TID 68) DEBUG BlockManager: Getting remote block rdd_55_5 21/01/17 17:43:08.930 Executor task launch worker for task 6.0 in stage 17.0 (TID 69) DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 4 21/01/17 17:43:08.932 Executor task launch worker for task 6.0 in stage 17.0 (TID 69) DEBUG MapOutputTrackerMaster: Convert map statuses for shuffle 4, mappers 0-10, partitions 6-7 21/01/17 17:43:08.932 Executor task launch worker for task 5.0 in stage 17.0 (TID 68) DEBUG BlockManager: Block rdd_55_5 is unknown by block manager master 21/01/17 17:43:08.932 Executor task launch worker for task 5.0 in stage 17.0 (TID 68) DEBUG BlockManager: Getting local block rdd_24_5 21/01/17 17:43:08.932 Executor task launch worker for task 5.0 in stage 17.0 (TID 68) DEBUG BlockManager: Level for block rdd_24_5 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:08.932 Executor task launch worker for task 5.0 in stage 17.0 (TID 68) INFO BlockManager: Found block rdd_24_5 locally 21/01/17 17:43:08.932 Executor task launch worker for task 5.0 in stage 17.0 (TID 68) DEBUG BlockManager: Getting local block rdd_24_5 21/01/17 17:43:08.932 Executor task launch worker for task 6.0 in stage 17.0 (TID 69) DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 21/01/17 17:43:08.932 Executor task launch worker for task 5.0 in stage 17.0 (TID 68) DEBUG BlockManager: Level for block rdd_24_5 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:08.932 Executor task launch worker for task 5.0 in stage 17.0 (TID 68) INFO BlockManager: Found block rdd_24_5 locally 21/01/17 17:43:08.933 Executor task launch worker for task 5.0 in stage 17.0 (TID 68) DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 4 21/01/17 17:43:08.933 Executor task launch worker for task 5.0 in stage 17.0 (TID 68) DEBUG MapOutputTrackerMaster: Convert map statuses for shuffle 4, mappers 0-10, partitions 5-6 21/01/17 17:43:08.933 Executor task launch worker for task 6.0 in stage 17.0 (TID 69) INFO ShuffleBlockFetcherIterator: Getting 3 (684.0 B) non-empty blocks including 3 (684.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 21/01/17 17:43:08.933 Executor task launch worker for task 6.0 in stage 17.0 (TID 69) INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 21/01/17 17:43:08.933 Executor task launch worker for task 6.0 in stage 17.0 (TID 69) DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_4_33_6,0), (shuffle_4_34_6,1), (shuffle_4_35_6,2) 21/01/17 17:43:08.933 Executor task launch worker for task 6.0 in stage 17.0 (TID 69) DEBUG BlockManager: Getting local shuffle block shuffle_4_33_6 21/01/17 17:43:08.933 Executor task launch worker for task 5.0 in stage 17.0 (TID 68) DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 21/01/17 17:43:08.933 Executor task launch worker for task 6.0 in stage 17.0 (TID 69) DEBUG BlockManager: Getting local shuffle block shuffle_4_34_6 21/01/17 17:43:08.933 Executor task launch worker for task 6.0 in stage 17.0 (TID 69) DEBUG BlockManager: Getting local shuffle block shuffle_4_35_6 21/01/17 17:43:08.933 Executor task launch worker for task 5.0 in stage 17.0 (TID 68) INFO ShuffleBlockFetcherIterator: Getting 3 (684.0 B) non-empty blocks including 3 (684.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 21/01/17 17:43:08.933 Executor task launch worker for task 5.0 in stage 17.0 (TID 68) INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 21/01/17 17:43:08.933 Executor task launch worker for task 5.0 in stage 17.0 (TID 68) DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_4_33_5,0), (shuffle_4_34_5,1), (shuffle_4_35_5,2) 21/01/17 17:43:08.934 Executor task launch worker for task 5.0 in stage 17.0 (TID 68) DEBUG BlockManager: Getting local shuffle block shuffle_4_33_5 21/01/17 17:43:08.934 Executor task launch worker for task 5.0 in stage 17.0 (TID 68) DEBUG BlockManager: Getting local shuffle block shuffle_4_34_5 21/01/17 17:43:08.934 Executor task launch worker for task 5.0 in stage 17.0 (TID 68) DEBUG BlockManager: Getting local shuffle block shuffle_4_35_5 21/01/17 17:43:08.934 Executor task launch worker for task 5.0 in stage 17.0 (TID 68) DEBUG ShuffleBlockFetcherIterator: Got local blocks in 1 ms 21/01/17 17:43:08.935 Executor task launch worker for task 7.0 in stage 17.0 (TID 70) DEBUG BlockManager: Getting local block rdd_55_7 21/01/17 17:43:08.935 Executor task launch worker for task 7.0 in stage 17.0 (TID 70) DEBUG BlockManager: Block rdd_55_7 was not found 21/01/17 17:43:08.935 Executor task launch worker for task 7.0 in stage 17.0 (TID 70) DEBUG BlockManager: Getting remote block rdd_55_7 21/01/17 17:43:08.935 Executor task launch worker for task 7.0 in stage 17.0 (TID 70) DEBUG BlockManager: Block rdd_55_7 is unknown by block manager master 21/01/17 17:43:08.935 Executor task launch worker for task 7.0 in stage 17.0 (TID 70) DEBUG BlockManager: Getting local block rdd_24_7 21/01/17 17:43:08.935 Executor task launch worker for task 7.0 in stage 17.0 (TID 70) DEBUG BlockManager: Level for block rdd_24_7 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:08.935 Executor task launch worker for task 7.0 in stage 17.0 (TID 70) INFO BlockManager: Found block rdd_24_7 locally 21/01/17 17:43:08.935 Executor task launch worker for task 7.0 in stage 17.0 (TID 70) DEBUG BlockManager: Getting local block rdd_24_7 21/01/17 17:43:08.935 Executor task launch worker for task 7.0 in stage 17.0 (TID 70) DEBUG BlockManager: Level for block rdd_24_7 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:08.935 Executor task launch worker for task 7.0 in stage 17.0 (TID 70) INFO BlockManager: Found block rdd_24_7 locally 21/01/17 17:43:08.935 Executor task launch worker for task 7.0 in stage 17.0 (TID 70) DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 4 21/01/17 17:43:08.935 Executor task launch worker for task 5.0 in stage 17.0 (TID 68) DEBUG TaskMemoryManager: Task 68 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@385f918c 21/01/17 17:43:08.936 Executor task launch worker for task 5.0 in stage 17.0 (TID 68) DEBUG TaskMemoryManager: Task 68 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@4e8a7b4a 21/01/17 17:43:08.936 Executor task launch worker for task 5.0 in stage 17.0 (TID 68) DEBUG TaskMemoryManager: Task 68 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@42ada548 21/01/17 17:43:08.933 Executor task launch worker for task 6.0 in stage 17.0 (TID 69) DEBUG ShuffleBlockFetcherIterator: Got local blocks in 1 ms 21/01/17 17:43:08.935 Executor task launch worker for task 7.0 in stage 17.0 (TID 70) DEBUG MapOutputTrackerMaster: Convert map statuses for shuffle 4, mappers 0-10, partitions 7-8 21/01/17 17:43:08.936 Executor task launch worker for task 5.0 in stage 17.0 (TID 68) INFO MemoryStore: Block rdd_55_5 stored as values in memory (estimated size 16.0 B, free 365.9 MiB) 21/01/17 17:43:08.936 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_55_4 for BlockManagerId(driver, 192.168.1.225, 40985, None) 21/01/17 17:43:08.937 Executor task launch worker for task 7.0 in stage 17.0 (TID 70) DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 21/01/17 17:43:08.937 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_55_4 in memory on 192.168.1.225:40985 (size: 16.0 B, free: 366.2 MiB) 21/01/17 17:43:08.937 Executor task launch worker for task 7.0 in stage 17.0 (TID 70) INFO ShuffleBlockFetcherIterator: Getting 3 (684.0 B) non-empty blocks including 3 (684.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 21/01/17 17:43:08.938 Executor task launch worker for task 4.0 in stage 17.0 (TID 67) DEBUG BlockManagerMaster: Updated info of block rdd_55_4 21/01/17 17:43:08.938 Executor task launch worker for task 7.0 in stage 17.0 (TID 70) INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 21/01/17 17:43:08.938 Executor task launch worker for task 7.0 in stage 17.0 (TID 70) DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_4_33_7,0), (shuffle_4_34_7,1), (shuffle_4_35_7,2) 21/01/17 17:43:08.938 block-manager-storage-async-thread-pool-17 DEBUG BlockManagerStorageEndpoint: Done removing broadcast 6, response is 0 21/01/17 17:43:08.938 Executor task launch worker for task 6.0 in stage 17.0 (TID 69) DEBUG TaskMemoryManager: Task 69 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@58718eaa 21/01/17 17:43:08.938 Executor task launch worker for task 4.0 in stage 17.0 (TID 67) DEBUG BlockManager: Told master about block rdd_55_4 21/01/17 17:43:08.938 Executor task launch worker for task 7.0 in stage 17.0 (TID 70) DEBUG BlockManager: Getting local shuffle block shuffle_4_33_7 21/01/17 17:43:08.938 Executor task launch worker for task 4.0 in stage 17.0 (TID 67) DEBUG BlockManager: Put block rdd_55_4 locally took 14 ms 21/01/17 17:43:08.938 Executor task launch worker for task 4.0 in stage 17.0 (TID 67) DEBUG BlockManager: Putting block rdd_55_4 without replication took 14 ms 21/01/17 17:43:08.938 Executor task launch worker for task 7.0 in stage 17.0 (TID 70) DEBUG BlockManager: Getting local shuffle block shuffle_4_34_7 21/01/17 17:43:08.938 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_55_5 for BlockManagerId(driver, 192.168.1.225, 40985, None) 21/01/17 17:43:08.938 Executor task launch worker for task 7.0 in stage 17.0 (TID 70) DEBUG BlockManager: Getting local shuffle block shuffle_4_35_7 21/01/17 17:43:08.938 Executor task launch worker for task 6.0 in stage 17.0 (TID 69) DEBUG TaskMemoryManager: Task 69 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@6b327b09 21/01/17 17:43:08.938 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_55_5 in memory on 192.168.1.225:40985 (size: 16.0 B, free: 366.2 MiB) 21/01/17 17:43:08.939 Executor task launch worker for task 6.0 in stage 17.0 (TID 69) DEBUG TaskMemoryManager: Task 69 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@2221a7cc 21/01/17 17:43:08.939 Executor task launch worker for task 5.0 in stage 17.0 (TID 68) DEBUG BlockManagerMaster: Updated info of block rdd_55_5 21/01/17 17:43:08.939 Executor task launch worker for task 5.0 in stage 17.0 (TID 68) DEBUG BlockManager: Told master about block rdd_55_5 21/01/17 17:43:08.939 Executor task launch worker for task 5.0 in stage 17.0 (TID 68) DEBUG BlockManager: Put block rdd_55_5 locally took 6 ms 21/01/17 17:43:08.939 Executor task launch worker for task 5.0 in stage 17.0 (TID 68) DEBUG BlockManager: Putting block rdd_55_5 without replication took 7 ms 21/01/17 17:43:08.939 Executor task launch worker for task 5.0 in stage 17.0 (TID 68) DEBUG BlockManager: Getting local block rdd_55_5 21/01/17 17:43:08.938 Executor task launch worker for task 4.0 in stage 17.0 (TID 67) DEBUG BlockManager: Getting local block rdd_55_4 21/01/17 17:43:08.939 block-manager-storage-async-thread-pool-17 DEBUG BlockManagerStorageEndpoint: Sent response: 0 to 192.168.1.225:41897 21/01/17 17:43:08.939 Executor task launch worker for task 5.0 in stage 17.0 (TID 68) DEBUG BlockManager: Level for block rdd_55_5 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:08.939 Executor task launch worker for task 6.0 in stage 17.0 (TID 69) INFO MemoryStore: Block rdd_55_6 stored as values in memory (estimated size 16.0 B, free 365.9 MiB) 21/01/17 17:43:08.938 Executor task launch worker for task 7.0 in stage 17.0 (TID 70) DEBUG ShuffleBlockFetcherIterator: Got local blocks in 1 ms 21/01/17 17:43:08.940 Spark Context Cleaner DEBUG ContextCleaner: Cleaned broadcast 6 21/01/17 17:43:08.940 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(184) 21/01/17 17:43:08.940 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 184 21/01/17 17:43:08.940 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 184 21/01/17 17:43:08.940 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(197) 21/01/17 17:43:08.940 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 197 21/01/17 17:43:08.940 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 197 21/01/17 17:43:08.940 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(200) 21/01/17 17:43:08.940 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 200 21/01/17 17:43:08.940 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 200 21/01/17 17:43:08.940 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(84) 21/01/17 17:43:08.940 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 84 21/01/17 17:43:08.940 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 84 21/01/17 17:43:08.940 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(129) 21/01/17 17:43:08.940 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 129 21/01/17 17:43:08.940 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 129 21/01/17 17:43:08.940 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(147) 21/01/17 17:43:08.940 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 147 21/01/17 17:43:08.940 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 147 21/01/17 17:43:08.940 Executor task launch worker for task 5.0 in stage 17.0 (TID 68) INFO Executor: Finished task 5.0 in stage 17.0 (TID 68). 2213 bytes result sent to driver 21/01/17 17:43:08.941 dispatcher-event-loop-0 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_17.0, runningTasks: 3 21/01/17 17:43:08.941 dispatcher-event-loop-0 INFO TaskSetManager: Starting task 8.0 in stage 17.0 (TID 71) (192.168.1.225, executor driver, partition 8, PROCESS_LOCAL, 7433 bytes) taskResourceAssignments Map() 21/01/17 17:43:08.941 Executor task launch worker for task 8.0 in stage 17.0 (TID 71) INFO Executor: Running task 8.0 in stage 17.0 (TID 71) 21/01/17 17:43:08.942 Executor task launch worker for task 7.0 in stage 17.0 (TID 70) DEBUG TaskMemoryManager: Task 70 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@5324bff3 21/01/17 17:43:08.942 Executor task launch worker for task 8.0 in stage 17.0 (TID 71) DEBUG ExecutorMetricsPoller: stageTCMP: (17, 0) -> 5 21/01/17 17:43:08.942 Executor task launch worker for task 7.0 in stage 17.0 (TID 70) DEBUG TaskMemoryManager: Task 70 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@2562ac04 21/01/17 17:43:08.942 Executor task launch worker for task 7.0 in stage 17.0 (TID 70) DEBUG TaskMemoryManager: Task 70 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@7df73779 21/01/17 17:43:08.943 Executor task launch worker for task 7.0 in stage 17.0 (TID 70) INFO MemoryStore: Block rdd_55_7 stored as values in memory (estimated size 16.0 B, free 365.9 MiB) 21/01/17 17:43:08.943 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_55_7 for BlockManagerId(driver, 192.168.1.225, 40985, None) 21/01/17 17:43:08.943 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_55_7 in memory on 192.168.1.225:40985 (size: 16.0 B, free: 366.2 MiB) 21/01/17 17:43:08.943 Executor task launch worker for task 7.0 in stage 17.0 (TID 70) DEBUG BlockManagerMaster: Updated info of block rdd_55_7 21/01/17 17:43:08.943 Executor task launch worker for task 7.0 in stage 17.0 (TID 70) DEBUG BlockManager: Told master about block rdd_55_7 21/01/17 17:43:08.943 Executor task launch worker for task 7.0 in stage 17.0 (TID 70) DEBUG BlockManager: Put block rdd_55_7 locally took 8 ms 21/01/17 17:43:08.943 Executor task launch worker for task 7.0 in stage 17.0 (TID 70) DEBUG BlockManager: Putting block rdd_55_7 without replication took 8 ms 21/01/17 17:43:08.943 Executor task launch worker for task 7.0 in stage 17.0 (TID 70) DEBUG BlockManager: Getting local block rdd_55_7 21/01/17 17:43:08.943 Executor task launch worker for task 7.0 in stage 17.0 (TID 70) DEBUG BlockManager: Level for block rdd_55_7 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:08.944 Executor task launch worker for task 7.0 in stage 17.0 (TID 70) INFO Executor: Finished task 7.0 in stage 17.0 (TID 70). 2213 bytes result sent to driver 21/01/17 17:43:08.944 Executor task launch worker for task 7.0 in stage 17.0 (TID 70) DEBUG ExecutorMetricsPoller: stageTCMP: (17, 0) -> 4 21/01/17 17:43:08.939 Executor task launch worker for task 4.0 in stage 17.0 (TID 67) DEBUG BlockManager: Level for block rdd_55_4 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:08.942 task-result-getter-1 INFO TaskSetManager: Finished task 5.0 in stage 17.0 (TID 68) in 29 ms on 192.168.1.225 (executor driver) (5/10) 21/01/17 17:43:08.940 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(54) 21/01/17 17:43:08.944 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 54 21/01/17 17:43:08.945 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 54 21/01/17 17:43:08.945 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(153) 21/01/17 17:43:08.945 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 153 21/01/17 17:43:08.945 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 153 21/01/17 17:43:08.945 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(154) 21/01/17 17:43:08.945 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 154 21/01/17 17:43:08.945 Executor task launch worker for task 0.0 in stage 3.0 (TID 15) INFO Executor: Finished task 0.0 in stage 3.0 (TID 15). 1305 bytes result sent to driver 21/01/17 17:43:08.945 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 154 21/01/17 17:43:08.945 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(96) 21/01/17 17:43:08.945 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 96 21/01/17 17:43:08.945 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 96 21/01/17 17:43:08.945 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(97) 21/01/17 17:43:08.945 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 97 21/01/17 17:43:08.945 Executor task launch worker for task 0.0 in stage 3.0 (TID 15) DEBUG ExecutorMetricsPoller: stageTCMP: (3, 0) -> 3 21/01/17 17:43:08.945 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 97 21/01/17 17:43:08.945 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(91) 21/01/17 17:43:08.945 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 91 21/01/17 17:43:08.945 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 91 21/01/17 17:43:08.945 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(215) 21/01/17 17:43:08.945 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 215 21/01/17 17:43:08.945 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 215 21/01/17 17:43:08.945 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(141) 21/01/17 17:43:08.945 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 141 21/01/17 17:43:08.945 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 141 21/01/17 17:43:08.945 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(50) 21/01/17 17:43:08.945 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 50 21/01/17 17:43:08.945 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 50 21/01/17 17:43:08.945 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(40) 21/01/17 17:43:08.945 Executor task launch worker for task 4.0 in stage 17.0 (TID 67) INFO Executor: Finished task 4.0 in stage 17.0 (TID 67). 2213 bytes result sent to driver 21/01/17 17:43:08.945 dispatcher-event-loop-3 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_17.0, runningTasks: 3 21/01/17 17:43:08.945 Executor task launch worker for task 4.0 in stage 17.0 (TID 67) DEBUG ExecutorMetricsPoller: stageTCMP: (17, 0) -> 3 21/01/17 17:43:08.945 dispatcher-event-loop-1 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_3.0, runningTasks: 3 21/01/17 17:43:08.945 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_55_6 for BlockManagerId(driver, 192.168.1.225, 40985, None) 21/01/17 17:43:08.945 dispatcher-event-loop-1 DEBUG TaskSetManager: No tasks for locality level NODE_LOCAL, so moving to locality level NO_PREF 21/01/17 17:43:08.946 dispatcher-event-loop-3 INFO TaskSetManager: Starting task 9.0 in stage 17.0 (TID 72) (192.168.1.225, executor driver, partition 9, PROCESS_LOCAL, 7433 bytes) taskResourceAssignments Map() 21/01/17 17:43:08.946 dispatcher-event-loop-1 DEBUG TaskSetManager: No tasks for locality level NO_PREF, so moving to locality level ANY 21/01/17 17:43:08.946 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_55_6 in memory on 192.168.1.225:40985 (size: 16.0 B, free: 366.2 MiB) 21/01/17 17:43:08.946 Executor task launch worker for task 9.0 in stage 17.0 (TID 72) INFO Executor: Running task 9.0 in stage 17.0 (TID 72) 21/01/17 17:43:08.945 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 40 21/01/17 17:43:08.946 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 40 21/01/17 17:43:08.946 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(144) 21/01/17 17:43:08.946 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 144 21/01/17 17:43:08.946 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 144 21/01/17 17:43:08.946 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(126) 21/01/17 17:43:08.946 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 126 21/01/17 17:43:08.946 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 126 21/01/17 17:43:08.946 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(134) 21/01/17 17:43:08.946 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 134 21/01/17 17:43:08.946 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 134 21/01/17 17:43:08.946 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(123) 21/01/17 17:43:08.946 Executor task launch worker for task 9.0 in stage 17.0 (TID 72) DEBUG ExecutorMetricsPoller: stageTCMP: (17, 0) -> 4 21/01/17 17:43:08.946 task-result-getter-0 INFO TaskSetManager: Finished task 7.0 in stage 17.0 (TID 70) in 17 ms on 192.168.1.225 (executor driver) (6/10) 21/01/17 17:43:08.946 task-result-getter-0 INFO TaskSetManager: Finished task 0.0 in stage 3.0 (TID 15) in 104 ms on 192.168.1.225 (executor driver) (1/4) 21/01/17 17:43:08.947 dispatcher-event-loop-3 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_17.0, runningTasks: 3 21/01/17 17:43:08.947 dispatcher-event-loop-3 DEBUG TaskSetManager: No tasks for locality level PROCESS_LOCAL, so moving to locality level NODE_LOCAL 21/01/17 17:43:08.947 dispatcher-event-loop-3 DEBUG TaskSetManager: No tasks for locality level NODE_LOCAL, so moving to locality level ANY 21/01/17 17:43:08.948 Executor task launch worker for task 8.0 in stage 17.0 (TID 71) DEBUG BlockManager: Getting local block rdd_55_8 21/01/17 17:43:08.946 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 123 21/01/17 17:43:08.948 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 123 21/01/17 17:43:08.948 Executor task launch worker for task 5.0 in stage 17.0 (TID 68) DEBUG ExecutorMetricsPoller: stageTCMP: (17, 0) -> 3 21/01/17 17:43:08.948 Executor task launch worker for task 8.0 in stage 17.0 (TID 71) DEBUG BlockManager: Block rdd_55_8 was not found 21/01/17 17:43:08.948 task-result-getter-2 INFO TaskSetManager: Finished task 4.0 in stage 17.0 (TID 67) in 35 ms on 192.168.1.225 (executor driver) (7/10) 21/01/17 17:43:08.951 Executor task launch worker for task 9.0 in stage 17.0 (TID 72) DEBUG BlockManager: Getting local block rdd_55_9 21/01/17 17:43:08.951 Executor task launch worker for task 9.0 in stage 17.0 (TID 72) DEBUG BlockManager: Block rdd_55_9 was not found 21/01/17 17:43:08.951 Executor task launch worker for task 9.0 in stage 17.0 (TID 72) DEBUG BlockManager: Getting remote block rdd_55_9 21/01/17 17:43:08.951 Executor task launch worker for task 6.0 in stage 17.0 (TID 69) DEBUG BlockManagerMaster: Updated info of block rdd_55_6 21/01/17 17:43:08.951 Executor task launch worker for task 6.0 in stage 17.0 (TID 69) DEBUG BlockManager: Told master about block rdd_55_6 21/01/17 17:43:08.951 Executor task launch worker for task 6.0 in stage 17.0 (TID 69) DEBUG BlockManager: Put block rdd_55_6 locally took 23 ms 21/01/17 17:43:08.951 Executor task launch worker for task 6.0 in stage 17.0 (TID 69) DEBUG BlockManager: Putting block rdd_55_6 without replication took 23 ms 21/01/17 17:43:08.951 Executor task launch worker for task 6.0 in stage 17.0 (TID 69) DEBUG BlockManager: Getting local block rdd_55_6 21/01/17 17:43:08.951 Executor task launch worker for task 6.0 in stage 17.0 (TID 69) DEBUG BlockManager: Level for block rdd_55_6 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:08.952 Executor task launch worker for task 6.0 in stage 17.0 (TID 69) INFO Executor: Finished task 6.0 in stage 17.0 (TID 69). 2213 bytes result sent to driver 21/01/17 17:43:08.952 Executor task launch worker for task 6.0 in stage 17.0 (TID 69) DEBUG ExecutorMetricsPoller: stageTCMP: (17, 0) -> 2 21/01/17 17:43:08.952 Executor task launch worker for task 1.0 in stage 3.0 (TID 12) DEBUG TaskMemoryManager: Task 12 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@2ceb5337 21/01/17 17:43:08.948 Executor task launch worker for task 8.0 in stage 17.0 (TID 71) DEBUG BlockManager: Getting remote block rdd_55_8 21/01/17 17:43:08.948 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(227) 21/01/17 17:43:08.953 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 227 21/01/17 17:43:08.953 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 227 21/01/17 17:43:08.953 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(166) 21/01/17 17:43:08.953 Executor task launch worker for task 8.0 in stage 17.0 (TID 71) DEBUG BlockManager: Block rdd_55_8 is unknown by block manager master 21/01/17 17:43:08.953 task-result-getter-3 INFO TaskSetManager: Finished task 6.0 in stage 17.0 (TID 69) in 37 ms on 192.168.1.225 (executor driver) (8/10) 21/01/17 17:43:08.953 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 166 21/01/17 17:43:08.953 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 166 21/01/17 17:43:08.953 Executor task launch worker for task 8.0 in stage 17.0 (TID 71) DEBUG BlockManager: Getting local block rdd_24_8 21/01/17 17:43:08.953 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(142) 21/01/17 17:43:08.953 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 142 21/01/17 17:43:08.953 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 142 21/01/17 17:43:08.953 Executor task launch worker for task 8.0 in stage 17.0 (TID 71) DEBUG BlockManager: Level for block rdd_24_8 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:08.953 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(173) 21/01/17 17:43:08.953 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 173 21/01/17 17:43:08.953 Executor task launch worker for task 8.0 in stage 17.0 (TID 71) INFO BlockManager: Found block rdd_24_8 locally 21/01/17 17:43:08.953 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 173 21/01/17 17:43:08.953 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(41) 21/01/17 17:43:08.953 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 41 21/01/17 17:43:08.953 Executor task launch worker for task 8.0 in stage 17.0 (TID 71) DEBUG BlockManager: Getting local block rdd_24_8 21/01/17 17:43:08.953 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 41 21/01/17 17:43:08.953 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(135) 21/01/17 17:43:08.953 Executor task launch worker for task 8.0 in stage 17.0 (TID 71) DEBUG BlockManager: Level for block rdd_24_8 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:08.953 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 135 21/01/17 17:43:08.953 Executor task launch worker for task 8.0 in stage 17.0 (TID 71) INFO BlockManager: Found block rdd_24_8 locally 21/01/17 17:43:08.953 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 135 21/01/17 17:43:08.953 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(67) 21/01/17 17:43:08.953 Executor task launch worker for task 8.0 in stage 17.0 (TID 71) DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 4 21/01/17 17:43:08.953 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 67 21/01/17 17:43:08.953 Executor task launch worker for task 8.0 in stage 17.0 (TID 71) DEBUG MapOutputTrackerMaster: Convert map statuses for shuffle 4, mappers 0-10, partitions 8-9 21/01/17 17:43:08.953 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 67 21/01/17 17:43:08.954 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(51) 21/01/17 17:43:08.954 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 51 21/01/17 17:43:08.954 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 51 21/01/17 17:43:08.954 Executor task launch worker for task 9.0 in stage 17.0 (TID 72) DEBUG BlockManager: Block rdd_55_9 is unknown by block manager master 21/01/17 17:43:08.954 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(53) 21/01/17 17:43:08.954 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 53 21/01/17 17:43:08.954 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 53 21/01/17 17:43:08.954 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(33) 21/01/17 17:43:08.954 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 33 21/01/17 17:43:08.954 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 33 21/01/17 17:43:08.954 Executor task launch worker for task 8.0 in stage 17.0 (TID 71) DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 21/01/17 17:43:08.954 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(110) 21/01/17 17:43:08.954 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 110 21/01/17 17:43:08.954 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 110 21/01/17 17:43:08.954 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(93) 21/01/17 17:43:08.954 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 93 21/01/17 17:43:08.954 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 93 21/01/17 17:43:08.954 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(156) 21/01/17 17:43:08.954 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 156 21/01/17 17:43:08.954 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 156 21/01/17 17:43:08.954 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(92) 21/01/17 17:43:08.954 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 92 21/01/17 17:43:08.954 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 92 21/01/17 17:43:08.954 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(34) 21/01/17 17:43:08.954 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 34 21/01/17 17:43:08.954 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 34 21/01/17 17:43:08.954 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(46) 21/01/17 17:43:08.954 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 46 21/01/17 17:43:08.954 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 46 21/01/17 17:43:08.954 Executor task launch worker for task 8.0 in stage 17.0 (TID 71) INFO ShuffleBlockFetcherIterator: Getting 3 (684.0 B) non-empty blocks including 3 (684.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 21/01/17 17:43:08.954 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(161) 21/01/17 17:43:08.954 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 161 21/01/17 17:43:08.954 Executor task launch worker for task 8.0 in stage 17.0 (TID 71) INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 21/01/17 17:43:08.954 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 161 21/01/17 17:43:08.954 Executor task launch worker for task 8.0 in stage 17.0 (TID 71) DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_4_33_8,0), (shuffle_4_34_8,1), (shuffle_4_35_8,2) 21/01/17 17:43:08.954 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(95) 21/01/17 17:43:08.955 Executor task launch worker for task 8.0 in stage 17.0 (TID 71) DEBUG BlockManager: Getting local shuffle block shuffle_4_33_8 21/01/17 17:43:08.955 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 95 21/01/17 17:43:08.955 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 95 21/01/17 17:43:08.955 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(127) 21/01/17 17:43:08.955 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 127 21/01/17 17:43:08.955 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 127 21/01/17 17:43:08.955 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(137) 21/01/17 17:43:08.955 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 137 21/01/17 17:43:08.955 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 137 21/01/17 17:43:08.955 Executor task launch worker for task 8.0 in stage 17.0 (TID 71) DEBUG BlockManager: Getting local shuffle block shuffle_4_34_8 21/01/17 17:43:08.955 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(186) 21/01/17 17:43:08.955 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 186 21/01/17 17:43:08.955 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 186 21/01/17 17:43:08.955 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(140) 21/01/17 17:43:08.955 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 140 21/01/17 17:43:08.955 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 140 21/01/17 17:43:08.955 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(192) 21/01/17 17:43:08.955 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 192 21/01/17 17:43:08.955 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 192 21/01/17 17:43:08.955 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(195) 21/01/17 17:43:08.955 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 195 21/01/17 17:43:08.955 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 195 21/01/17 17:43:08.955 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(207) 21/01/17 17:43:08.955 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 207 21/01/17 17:43:08.955 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 207 21/01/17 17:43:08.955 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(49) 21/01/17 17:43:08.955 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 49 21/01/17 17:43:08.955 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 49 21/01/17 17:43:08.955 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanBroadcast(4) 21/01/17 17:43:08.955 Spark Context Cleaner DEBUG ContextCleaner: Cleaning broadcast 4 21/01/17 17:43:08.955 Spark Context Cleaner DEBUG TorrentBroadcast: Unpersisting TorrentBroadcast 4 21/01/17 17:43:08.955 Executor task launch worker for task 9.0 in stage 17.0 (TID 72) DEBUG BlockManager: Getting local block rdd_24_9 21/01/17 17:43:08.955 Executor task launch worker for task 9.0 in stage 17.0 (TID 72) DEBUG BlockManager: Level for block rdd_24_9 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:08.956 Executor task launch worker for task 9.0 in stage 17.0 (TID 72) INFO BlockManager: Found block rdd_24_9 locally 21/01/17 17:43:08.956 Executor task launch worker for task 9.0 in stage 17.0 (TID 72) DEBUG BlockManager: Getting local block rdd_24_9 21/01/17 17:43:08.956 Executor task launch worker for task 9.0 in stage 17.0 (TID 72) DEBUG BlockManager: Level for block rdd_24_9 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:08.956 Executor task launch worker for task 9.0 in stage 17.0 (TID 72) INFO BlockManager: Found block rdd_24_9 locally 21/01/17 17:43:08.956 Executor task launch worker for task 9.0 in stage 17.0 (TID 72) DEBUG MapOutputTrackerMaster: Fetching outputs for shuffle 4 21/01/17 17:43:08.956 Executor task launch worker for task 9.0 in stage 17.0 (TID 72) DEBUG MapOutputTrackerMaster: Convert map statuses for shuffle 4, mappers 0-10, partitions 9-10 21/01/17 17:43:08.956 Executor task launch worker for task 9.0 in stage 17.0 (TID 72) DEBUG ShuffleBlockFetcherIterator: maxBytesInFlight: 50331648, targetRemoteRequestSize: 10066329, maxBlocksInFlightPerAddress: 2147483647 21/01/17 17:43:08.957 Executor task launch worker for task 9.0 in stage 17.0 (TID 72) INFO ShuffleBlockFetcherIterator: Getting 3 (684.0 B) non-empty blocks including 3 (684.0 B) local and 0 (0.0 B) host-local and 0 (0.0 B) remote blocks 21/01/17 17:43:08.957 Executor task launch worker for task 9.0 in stage 17.0 (TID 72) INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 21/01/17 17:43:08.957 Executor task launch worker for task 9.0 in stage 17.0 (TID 72) DEBUG ShuffleBlockFetcherIterator: Start fetching local blocks: (shuffle_4_33_9,0), (shuffle_4_34_9,1), (shuffle_4_35_9,2) 21/01/17 17:43:08.957 Executor task launch worker for task 9.0 in stage 17.0 (TID 72) DEBUG BlockManager: Getting local shuffle block shuffle_4_33_9 21/01/17 17:43:08.957 Executor task launch worker for task 9.0 in stage 17.0 (TID 72) DEBUG BlockManager: Getting local shuffle block shuffle_4_34_9 21/01/17 17:43:08.957 Executor task launch worker for task 9.0 in stage 17.0 (TID 72) DEBUG BlockManager: Getting local shuffle block shuffle_4_35_9 21/01/17 17:43:08.957 Executor task launch worker for task 9.0 in stage 17.0 (TID 72) DEBUG ShuffleBlockFetcherIterator: Got local blocks in 1 ms 21/01/17 17:43:08.955 Executor task launch worker for task 8.0 in stage 17.0 (TID 71) DEBUG BlockManager: Getting local shuffle block shuffle_4_35_8 21/01/17 17:43:08.958 Executor task launch worker for task 8.0 in stage 17.0 (TID 71) DEBUG ShuffleBlockFetcherIterator: Got local blocks in 4 ms 21/01/17 17:43:08.959 Executor task launch worker for task 9.0 in stage 17.0 (TID 72) DEBUG TaskMemoryManager: Task 72 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@158b95b4 21/01/17 17:43:08.959 Executor task launch worker for task 9.0 in stage 17.0 (TID 72) DEBUG TaskMemoryManager: Task 72 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@19ec5449 21/01/17 17:43:08.960 Executor task launch worker for task 8.0 in stage 17.0 (TID 71) DEBUG TaskMemoryManager: Task 71 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@4d639b00 21/01/17 17:43:08.960 Executor task launch worker for task 9.0 in stage 17.0 (TID 72) DEBUG TaskMemoryManager: Task 72 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@7c2222fc 21/01/17 17:43:08.961 Executor task launch worker for task 8.0 in stage 17.0 (TID 71) DEBUG TaskMemoryManager: Task 71 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@3c273742 21/01/17 17:43:08.961 Executor task launch worker for task 9.0 in stage 17.0 (TID 72) INFO MemoryStore: Block rdd_55_9 stored as values in memory (estimated size 16.0 B, free 365.9 MiB) 21/01/17 17:43:08.961 Executor task launch worker for task 8.0 in stage 17.0 (TID 71) DEBUG TaskMemoryManager: Task 71 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@3c36d959 21/01/17 17:43:08.961 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_55_9 for BlockManagerId(driver, 192.168.1.225, 40985, None) 21/01/17 17:43:08.961 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_55_9 in memory on 192.168.1.225:40985 (size: 16.0 B, free: 366.2 MiB) 21/01/17 17:43:08.961 Executor task launch worker for task 8.0 in stage 17.0 (TID 71) INFO MemoryStore: Block rdd_55_8 stored as values in memory (estimated size 16.0 B, free 365.9 MiB) 21/01/17 17:43:08.961 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_55_8 for BlockManagerId(driver, 192.168.1.225, 40985, None) 21/01/17 17:43:08.962 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_55_8 in memory on 192.168.1.225:40985 (size: 16.0 B, free: 366.2 MiB) 21/01/17 17:43:08.962 Executor task launch worker for task 8.0 in stage 17.0 (TID 71) DEBUG BlockManagerMaster: Updated info of block rdd_55_8 21/01/17 17:43:08.962 Executor task launch worker for task 8.0 in stage 17.0 (TID 71) DEBUG BlockManager: Told master about block rdd_55_8 21/01/17 17:43:08.962 Executor task launch worker for task 8.0 in stage 17.0 (TID 71) DEBUG BlockManager: Put block rdd_55_8 locally took 8 ms 21/01/17 17:43:08.962 Executor task launch worker for task 9.0 in stage 17.0 (TID 72) DEBUG BlockManagerMaster: Updated info of block rdd_55_9 21/01/17 17:43:08.962 Executor task launch worker for task 9.0 in stage 17.0 (TID 72) DEBUG BlockManager: Told master about block rdd_55_9 21/01/17 17:43:08.962 Executor task launch worker for task 9.0 in stage 17.0 (TID 72) DEBUG BlockManager: Put block rdd_55_9 locally took 7 ms 21/01/17 17:43:08.962 Executor task launch worker for task 8.0 in stage 17.0 (TID 71) DEBUG BlockManager: Putting block rdd_55_8 without replication took 8 ms 21/01/17 17:43:08.962 Executor task launch worker for task 8.0 in stage 17.0 (TID 71) DEBUG BlockManager: Getting local block rdd_55_8 21/01/17 17:43:08.962 Executor task launch worker for task 9.0 in stage 17.0 (TID 72) DEBUG BlockManager: Putting block rdd_55_9 without replication took 7 ms 21/01/17 17:43:08.962 Executor task launch worker for task 9.0 in stage 17.0 (TID 72) DEBUG BlockManager: Getting local block rdd_55_9 21/01/17 17:43:08.962 Executor task launch worker for task 8.0 in stage 17.0 (TID 71) DEBUG BlockManager: Level for block rdd_55_8 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:08.962 Executor task launch worker for task 9.0 in stage 17.0 (TID 72) DEBUG BlockManager: Level for block rdd_55_9 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:08.963 Executor task launch worker for task 8.0 in stage 17.0 (TID 71) INFO Executor: Finished task 8.0 in stage 17.0 (TID 71). 2213 bytes result sent to driver 21/01/17 17:43:08.963 Executor task launch worker for task 8.0 in stage 17.0 (TID 71) DEBUG ExecutorMetricsPoller: stageTCMP: (17, 0) -> 1 21/01/17 17:43:08.963 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_4 stored as values in memory (estimated size 35.1 KiB, free 366.2 MiB) 21/01/17 17:43:08.964 task-result-getter-1 INFO TaskSetManager: Finished task 8.0 in stage 17.0 (TID 71) in 23 ms on 192.168.1.225 (executor driver) (9/10) 21/01/17 17:43:08.964 Executor task launch worker for task 9.0 in stage 17.0 (TID 72) INFO Executor: Finished task 9.0 in stage 17.0 (TID 72). 2213 bytes result sent to driver 21/01/17 17:43:08.964 Executor task launch worker for task 9.0 in stage 17.0 (TID 72) DEBUG ExecutorMetricsPoller: removing (17, 0) from stageTCMP 21/01/17 17:43:08.964 task-result-getter-0 INFO TaskSetManager: Finished task 9.0 in stage 17.0 (TID 72) in 19 ms on 192.168.1.225 (executor driver) (10/10) 21/01/17 17:43:08.964 task-result-getter-0 INFO TaskSchedulerImpl: Removed TaskSet 17.0, whose tasks have all completed, from pool 21/01/17 17:43:08.965 dag-scheduler-event-loop INFO DAGScheduler: ResultStage 17 (count at ALS.scala:1080) finished in 0.282 s 21/01/17 17:43:08.965 Executor task launch worker for task 2.0 in stage 3.0 (TID 13) DEBUG TaskMemoryManager: Task 13 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@67d27cc6 21/01/17 17:43:08.966 dag-scheduler-event-loop DEBUG DAGScheduler: After removal of stage 17, remaining stages = 4 21/01/17 17:43:08.966 dag-scheduler-event-loop DEBUG DAGScheduler: After removal of stage 14, remaining stages = 3 21/01/17 17:43:08.966 dag-scheduler-event-loop DEBUG DAGScheduler: After removal of stage 13, remaining stages = 2 21/01/17 17:43:08.966 Executor task launch worker for task 1.0 in stage 3.0 (TID 12) INFO MemoryStore: Block rdd_21_1 stored as values in memory (estimated size 120.0 B, free 366.2 MiB) 21/01/17 17:43:08.966 dag-scheduler-event-loop DEBUG DAGScheduler: After removal of stage 16, remaining stages = 1 21/01/17 17:43:08.966 dag-scheduler-event-loop DEBUG DAGScheduler: After removal of stage 15, remaining stages = 0 21/01/17 17:43:08.966 dag-scheduler-event-loop INFO DAGScheduler: Job 4 is finished. Cancelling potential speculative or zombie tasks for this job 21/01/17 17:43:08.966 Executor task launch worker for task 2.0 in stage 3.0 (TID 13) INFO MemoryStore: Block rdd_21_2 stored as values in memory (estimated size 120.0 B, free 366.2 MiB) 21/01/17 17:43:08.966 dag-scheduler-event-loop INFO TaskSchedulerImpl: Killing all running tasks in stage 17: Stage finished 21/01/17 17:43:08.965 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_4 locally took 2 ms 21/01/17 17:43:08.966 Thread-3 INFO DAGScheduler: Job 4 finished: count at ALS.scala:1080, took 0.291473 s 21/01/17 17:43:08.966 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_4 without replication took 3 ms 21/01/17 17:43:08.966 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_21_2 for BlockManagerId(driver, 192.168.1.225, 46295, None) 21/01/17 17:43:08.967 block-manager-storage-async-thread-pool-18 DEBUG BlockManagerStorageEndpoint: removing broadcast 4 21/01/17 17:43:08.967 block-manager-storage-async-thread-pool-18 DEBUG BlockManager: Removing broadcast 4 21/01/17 17:43:08.967 block-manager-storage-async-thread-pool-18 DEBUG BlockManager: Removing block broadcast_4_piece0 21/01/17 17:43:08.967 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_21_2 in memory on 192.168.1.225:46295 (size: 120.0 B, free: 366.3 MiB) 21/01/17 17:43:08.967 Executor task launch worker for task 2.0 in stage 3.0 (TID 13) DEBUG BlockManagerMaster: Updated info of block rdd_21_2 21/01/17 17:43:08.967 Executor task launch worker for task 2.0 in stage 3.0 (TID 13) DEBUG BlockManager: Told master about block rdd_21_2 21/01/17 17:43:08.967 Executor task launch worker for task 2.0 in stage 3.0 (TID 13) DEBUG BlockManager: Put block rdd_21_2 locally took 106 ms 21/01/17 17:43:08.967 Executor task launch worker for task 2.0 in stage 3.0 (TID 13) DEBUG BlockManager: Putting block rdd_21_2 without replication took 106 ms 21/01/17 17:43:08.967 Executor task launch worker for task 2.0 in stage 3.0 (TID 13) DEBUG BlockManager: Getting local block rdd_21_2 21/01/17 17:43:08.967 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_21_1 for BlockManagerId(driver, 192.168.1.225, 46295, None) 21/01/17 17:43:08.967 Executor task launch worker for task 2.0 in stage 3.0 (TID 13) DEBUG BlockManager: Level for block rdd_21_2 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:08.968 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_21_1 in memory on 192.168.1.225:46295 (size: 120.0 B, free: 366.3 MiB) 21/01/17 17:43:08.968 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_4_piece0 stored as bytes in memory (estimated size 14.9 KiB, free 366.2 MiB) 21/01/17 17:43:08.968 Executor task launch worker for task 1.0 in stage 3.0 (TID 12) DEBUG BlockManagerMaster: Updated info of block rdd_21_1 21/01/17 17:43:08.968 Executor task launch worker for task 1.0 in stage 3.0 (TID 12) DEBUG BlockManager: Told master about block rdd_21_1 21/01/17 17:43:08.968 Executor task launch worker for task 1.0 in stage 3.0 (TID 12) DEBUG BlockManager: Put block rdd_21_1 locally took 116 ms 21/01/17 17:43:08.969 Executor task launch worker for task 1.0 in stage 3.0 (TID 12) DEBUG BlockManager: Putting block rdd_21_1 without replication took 116 ms 21/01/17 17:43:08.969 Executor task launch worker for task 1.0 in stage 3.0 (TID 12) DEBUG BlockManager: Getting local block rdd_21_1 21/01/17 17:43:08.969 Executor task launch worker for task 1.0 in stage 3.0 (TID 12) DEBUG BlockManager: Level for block rdd_21_1 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:08.969 Thread-3 INFO MapPartitionsRDD: Removing RDD 36 from persistence list 21/01/17 17:43:08.970 Executor task launch worker for task 1.0 in stage 3.0 (TID 12) INFO Executor: Finished task 1.0 in stage 3.0 (TID 12). 1305 bytes result sent to driver 21/01/17 17:43:08.970 Thread-3 INFO MapPartitionsRDD: Removing RDD 24 from persistence list 21/01/17 17:43:08.970 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master broadcast_4_piece0 for BlockManagerId(driver, 192.168.1.225, 33787, None) 21/01/17 17:43:08.970 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added broadcast_4_piece0 in memory on 192.168.1.225:33787 (size: 14.9 KiB, free: 366.3 MiB) 21/01/17 17:43:08.971 Executor task launch worker for task 1.0 in stage 3.0 (TID 12) DEBUG ExecutorMetricsPoller: stageTCMP: (3, 0) -> 2 21/01/17 17:43:08.971 dag-scheduler-event-loop DEBUG BlockManagerMaster: Updated info of block broadcast_4_piece0 21/01/17 17:43:08.971 dag-scheduler-event-loop DEBUG BlockManager: Told master about block broadcast_4_piece0 21/01/17 17:43:08.971 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_4_piece0 locally took 3 ms 21/01/17 17:43:08.971 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_4_piece0 without replication took 3 ms 21/01/17 17:43:08.971 dag-scheduler-event-loop INFO SparkContext: Created broadcast 4 from broadcast at DAGScheduler.scala:1383 21/01/17 17:43:08.972 task-result-getter-2 INFO TaskSetManager: Finished task 1.0 in stage 3.0 (TID 12) in 134 ms on 192.168.1.225 (executor driver) (2/4) 21/01/17 17:43:08.972 dag-scheduler-event-loop INFO DAGScheduler: Submitting 4 missing tasks from ShuffleMapStage 3 (MapPartitionsRDD[14] at collect at ClusteringMetrics.scala:97) (first 15 tasks are for partitions Vector(0, 1, 2, 3)) 21/01/17 17:43:08.972 dag-scheduler-event-loop INFO TaskSchedulerImpl: Adding task set 3.0 with 4 tasks resource profile 0 21/01/17 17:43:08.973 dag-scheduler-event-loop DEBUG TaskSetManager: Epoch for TaskSet 3.0: 1 21/01/17 17:43:08.973 dag-scheduler-event-loop DEBUG TaskSetManager: Adding pending tasks took 0 ms 21/01/17 17:43:08.973 dag-scheduler-event-loop DEBUG TaskSetManager: Valid locality levels for TaskSet 3.0: NO_PREF, ANY 21/01/17 17:43:08.973 dispatcher-event-loop-3 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_3.0, runningTasks: 0 21/01/17 17:43:08.974 dispatcher-event-loop-3 INFO TaskSetManager: Starting task 0.0 in stage 3.0 (TID 9) (192.168.1.225, executor driver, partition 0, PROCESS_LOCAL, 7387 bytes) taskResourceAssignments Map() 21/01/17 17:43:08.969 Executor task launch worker for task 2.0 in stage 3.0 (TID 13) INFO Executor: Finished task 2.0 in stage 3.0 (TID 13). 1305 bytes result sent to driver 21/01/17 17:43:08.974 Executor task launch worker for task 2.0 in stage 3.0 (TID 13) DEBUG ExecutorMetricsPoller: stageTCMP: (3, 0) -> 1 21/01/17 17:43:08.974 dispatcher-event-loop-3 INFO TaskSetManager: Starting task 1.0 in stage 3.0 (TID 10) (192.168.1.225, executor driver, partition 1, PROCESS_LOCAL, 7452 bytes) taskResourceAssignments Map() 21/01/17 17:43:08.975 dispatcher-event-loop-3 INFO TaskSetManager: Starting task 2.0 in stage 3.0 (TID 11) (192.168.1.225, executor driver, partition 2, PROCESS_LOCAL, 7387 bytes) taskResourceAssignments Map() 21/01/17 17:43:08.975 dispatcher-event-loop-3 INFO TaskSetManager: Starting task 3.0 in stage 3.0 (TID 12) (192.168.1.225, executor driver, partition 3, PROCESS_LOCAL, 7452 bytes) taskResourceAssignments Map() 21/01/17 17:43:08.975 task-result-getter-1 INFO TaskSetManager: Finished task 2.0 in stage 3.0 (TID 13) in 135 ms on 192.168.1.225 (executor driver) (3/4) 21/01/17 17:43:08.976 Executor task launch worker for task 0.0 in stage 3.0 (TID 9) INFO Executor: Running task 0.0 in stage 3.0 (TID 9) 21/01/17 17:43:08.976 Executor task launch worker for task 1.0 in stage 3.0 (TID 10) INFO Executor: Running task 1.0 in stage 3.0 (TID 10) 21/01/17 17:43:08.977 Executor task launch worker for task 3.0 in stage 3.0 (TID 14) DEBUG TaskMemoryManager: Task 14 release 0.0 B from org.apache.spark.util.collection.ExternalAppendOnlyMap@70a0918f 21/01/17 17:43:08.977 Executor task launch worker for task 1.0 in stage 3.0 (TID 10) DEBUG ExecutorMetricsPoller: stageTCMP: (3, 0) -> 1 21/01/17 17:43:08.978 Executor task launch worker for task 3.0 in stage 3.0 (TID 14) INFO MemoryStore: Block rdd_21_3 stored as values in memory (estimated size 120.0 B, free 366.2 MiB) 21/01/17 17:43:08.978 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master rdd_21_3 for BlockManagerId(driver, 192.168.1.225, 46295, None) 21/01/17 17:43:08.978 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added rdd_21_3 in memory on 192.168.1.225:46295 (size: 120.0 B, free: 366.3 MiB) 21/01/17 17:43:08.978 Executor task launch worker for task 1.0 in stage 3.0 (TID 10) DEBUG BlockManager: Getting local block broadcast_4 21/01/17 17:43:08.979 Executor task launch worker for task 1.0 in stage 3.0 (TID 10) DEBUG BlockManager: Level for block broadcast_4 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:08.979 Executor task launch worker for task 3.0 in stage 3.0 (TID 14) DEBUG BlockManagerMaster: Updated info of block rdd_21_3 21/01/17 17:43:08.979 Executor task launch worker for task 3.0 in stage 3.0 (TID 14) DEBUG BlockManager: Told master about block rdd_21_3 21/01/17 17:43:08.979 Executor task launch worker for task 3.0 in stage 3.0 (TID 14) DEBUG BlockManager: Put block rdd_21_3 locally took 127 ms 21/01/17 17:43:08.979 Executor task launch worker for task 3.0 in stage 3.0 (TID 14) DEBUG BlockManager: Putting block rdd_21_3 without replication took 127 ms 21/01/17 17:43:08.979 Executor task launch worker for task 3.0 in stage 3.0 (TID 14) DEBUG BlockManager: Getting local block rdd_21_3 21/01/17 17:43:08.979 Executor task launch worker for task 3.0 in stage 3.0 (TID 14) DEBUG BlockManager: Level for block rdd_21_3 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:08.980 Executor task launch worker for task 0.0 in stage 3.0 (TID 9) DEBUG ExecutorMetricsPoller: stageTCMP: (3, 0) -> 2 21/01/17 17:43:08.983 Executor task launch worker for task 3.0 in stage 3.0 (TID 12) INFO Executor: Running task 3.0 in stage 3.0 (TID 12) 21/01/17 17:43:08.985 Executor task launch worker for task 2.0 in stage 3.0 (TID 11) INFO Executor: Running task 2.0 in stage 3.0 (TID 11) 21/01/17 17:43:08.985 Executor task launch worker for task 3.0 in stage 3.0 (TID 14) INFO Executor: Finished task 3.0 in stage 3.0 (TID 14). 1305 bytes result sent to driver 21/01/17 17:43:08.985 Executor task launch worker for task 3.0 in stage 3.0 (TID 14) DEBUG ExecutorMetricsPoller: removing (3, 0) from stageTCMP 21/01/17 17:43:08.987 task-result-getter-3 INFO TaskSetManager: Finished task 3.0 in stage 3.0 (TID 14) in 146 ms on 192.168.1.225 (executor driver) (4/4) 21/01/17 17:43:08.987 task-result-getter-3 INFO TaskSchedulerImpl: Removed TaskSet 3.0, whose tasks have all completed, from pool 21/01/17 17:43:08.988 dag-scheduler-event-loop INFO DAGScheduler: ResultStage 3 (count at CountVectorizer.scala:233) finished in 0.160 s 21/01/17 17:43:08.988 dag-scheduler-event-loop DEBUG DAGScheduler: After removal of stage 2, remaining stages = 1 21/01/17 17:43:08.988 dag-scheduler-event-loop DEBUG DAGScheduler: After removal of stage 3, remaining stages = 0 21/01/17 17:43:08.988 dag-scheduler-event-loop INFO DAGScheduler: Job 2 is finished. Cancelling potential speculative or zombie tasks for this job 21/01/17 17:43:08.989 dag-scheduler-event-loop INFO TaskSchedulerImpl: Killing all running tasks in stage 3: Stage finished 21/01/17 17:43:08.989 Thread-3 INFO DAGScheduler: Job 2 finished: count at CountVectorizer.scala:233, took 0.792404 s 21/01/17 17:43:08.995 Thread-3 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$takeOrdered$2 21/01/17 17:43:08.998 Thread-3 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$takeOrdered$2) is now cleaned +++ 21/01/17 17:43:09.001 block-manager-storage-async-thread-pool-19 DEBUG BlockManagerStorageEndpoint: removing RDD 36 21/01/17 17:43:09.001 block-manager-storage-async-thread-pool-19 INFO BlockManager: Removing RDD 36 21/01/17 17:43:09.002 block-manager-storage-async-thread-pool-18 DEBUG MemoryStore: Block broadcast_4_piece0 of size 16238 dropped from memory (free 383689197) 21/01/17 17:43:09.003 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master broadcast_4_piece0 for BlockManagerId(driver, 192.168.1.225, 40985, None) 21/01/17 17:43:09.003 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Removed broadcast_4_piece0 on 192.168.1.225:40985 in memory (size: 15.9 KiB, free: 366.2 MiB) 21/01/17 17:43:09.004 Executor task launch worker for task 2.0 in stage 3.0 (TID 11) DEBUG ExecutorMetricsPoller: stageTCMP: (3, 0) -> 3 21/01/17 17:43:09.008 Executor task launch worker for task 3.0 in stage 3.0 (TID 12) DEBUG ExecutorMetricsPoller: stageTCMP: (3, 0) -> 4 21/01/17 17:43:09.022 Thread-3 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$takeOrdered$3 21/01/17 17:43:09.025 Thread-3 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$takeOrdered$3) is now cleaned +++ 21/01/17 17:43:09.029 Thread-3 DEBUG ClosureCleaner: Cleaning indylambda closure: $anonfun$runJob$6 21/01/17 17:43:09.032 Thread-3 DEBUG ClosureCleaner: +++ indylambda closure ($anonfun$runJob$6) is now cleaned +++ 21/01/17 17:43:09.035 Thread-3 INFO SparkContext: Starting job: top at CountVectorizer.scala:236 21/01/17 17:43:09.036 dag-scheduler-event-loop DEBUG DAGScheduler: Merging stage rdd profiles: Set() 21/01/17 17:43:09.036 dag-scheduler-event-loop DEBUG DAGScheduler: Merging stage rdd profiles: Set() 21/01/17 17:43:09.037 dag-scheduler-event-loop INFO DAGScheduler: Got job 3 (top at CountVectorizer.scala:236) with 4 output partitions 21/01/17 17:43:09.037 dag-scheduler-event-loop INFO DAGScheduler: Final stage: ResultStage 5 (top at CountVectorizer.scala:236) 21/01/17 17:43:09.037 dag-scheduler-event-loop INFO DAGScheduler: Parents of final stage: List(ShuffleMapStage 4) 21/01/17 17:43:09.038 dag-scheduler-event-loop INFO DAGScheduler: Missing parents: List() 21/01/17 17:43:09.039 dag-scheduler-event-loop DEBUG DAGScheduler: submitStage(ResultStage 5 (name=top at CountVectorizer.scala:236;jobs=3)) 21/01/17 17:43:09.039 dag-scheduler-event-loop DEBUG DAGScheduler: missing: List() 21/01/17 17:43:09.039 dag-scheduler-event-loop INFO DAGScheduler: Submitting ResultStage 5 (MapPartitionsRDD[22] at top at CountVectorizer.scala:236), which has no missing parents 21/01/17 17:43:09.039 dag-scheduler-event-loop DEBUG DAGScheduler: submitMissingTasks(ResultStage 5) 21/01/17 17:43:09.017 block-manager-storage-async-thread-pool-22 DEBUG BlockManagerStorageEndpoint: Done removing RDD 36, response is 0 21/01/17 17:43:09.018 block-manager-storage-async-thread-pool-18 DEBUG BlockManagerMaster: Updated info of block broadcast_4_piece0 21/01/17 17:43:09.047 block-manager-storage-async-thread-pool-18 DEBUG BlockManager: Told master about block broadcast_4_piece0 21/01/17 17:43:09.047 block-manager-storage-async-thread-pool-18 DEBUG BlockManager: Removing block broadcast_4 21/01/17 17:43:09.048 block-manager-storage-async-thread-pool-18 DEBUG MemoryStore: Block broadcast_4 of size 34328 dropped from memory (free 383723525) 21/01/17 17:43:09.049 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_6 stored as values in memory (estimated size 6.0 KiB, free 366.2 MiB) 21/01/17 17:43:09.049 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_6 locally took 4 ms 21/01/17 17:43:09.049 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_6 without replication took 4 ms 21/01/17 17:43:09.050 dag-scheduler-event-loop INFO MemoryStore: Block broadcast_6_piece0 stored as bytes in memory (estimated size 3.2 KiB, free 366.2 MiB) 21/01/17 17:43:09.051 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master broadcast_6_piece0 for BlockManagerId(driver, 192.168.1.225, 46295, None) 21/01/17 17:43:09.051 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Added broadcast_6_piece0 in memory on 192.168.1.225:46295 (size: 3.2 KiB, free: 366.3 MiB) 21/01/17 17:43:09.051 dag-scheduler-event-loop DEBUG BlockManagerMaster: Updated info of block broadcast_6_piece0 21/01/17 17:43:09.052 dag-scheduler-event-loop DEBUG BlockManager: Told master about block broadcast_6_piece0 21/01/17 17:43:09.052 dag-scheduler-event-loop DEBUG BlockManager: Put block broadcast_6_piece0 locally took 1 ms 21/01/17 17:43:09.052 dag-scheduler-event-loop DEBUG BlockManager: Putting block broadcast_6_piece0 without replication took 1 ms 21/01/17 17:43:09.052 dag-scheduler-event-loop INFO SparkContext: Created broadcast 6 from broadcast at DAGScheduler.scala:1383 21/01/17 17:43:09.055 block-manager-storage-async-thread-pool-22 DEBUG BlockManagerStorageEndpoint: Sent response: 0 to 192.168.1.225:41897 21/01/17 17:43:09.055 block-manager-storage-async-thread-pool-20 DEBUG BlockManagerStorageEndpoint: removing RDD 24 21/01/17 17:43:09.055 block-manager-storage-async-thread-pool-20 INFO BlockManager: Removing RDD 24 21/01/17 17:43:09.055 block-manager-storage-async-thread-pool-20 DEBUG BlockManager: Removing block rdd_24_0 21/01/17 17:43:09.055 block-manager-storage-async-thread-pool-20 DEBUG MemoryStore: Block rdd_24_0 of size 192 dropped from memory (free 383723717) 21/01/17 17:43:09.056 block-manager-storage-async-thread-pool-20 DEBUG BlockManager: Removing block rdd_24_4 21/01/17 17:43:09.056 block-manager-storage-async-thread-pool-20 DEBUG MemoryStore: Block rdd_24_4 of size 16 dropped from memory (free 383723733) 21/01/17 17:43:09.056 block-manager-storage-async-thread-pool-20 DEBUG BlockManager: Removing block rdd_24_6 21/01/17 17:43:09.056 block-manager-storage-async-thread-pool-20 DEBUG MemoryStore: Block rdd_24_6 of size 16 dropped from memory (free 383723749) 21/01/17 17:43:09.056 block-manager-storage-async-thread-pool-20 DEBUG BlockManager: Removing block rdd_24_5 21/01/17 17:43:09.056 block-manager-storage-async-thread-pool-20 DEBUG MemoryStore: Block rdd_24_5 of size 16 dropped from memory (free 383723765) 21/01/17 17:43:09.056 block-manager-storage-async-thread-pool-20 DEBUG BlockManager: Removing block rdd_24_7 21/01/17 17:43:09.056 block-manager-storage-async-thread-pool-20 DEBUG MemoryStore: Block rdd_24_7 of size 16 dropped from memory (free 383723781) 21/01/17 17:43:09.056 block-manager-storage-async-thread-pool-20 DEBUG BlockManager: Removing block rdd_24_8 21/01/17 17:43:09.056 block-manager-storage-async-thread-pool-20 DEBUG MemoryStore: Block rdd_24_8 of size 16 dropped from memory (free 383723797) 21/01/17 17:43:09.057 block-manager-storage-async-thread-pool-20 DEBUG BlockManager: Removing block rdd_24_3 21/01/17 17:43:09.057 block-manager-storage-async-thread-pool-20 DEBUG MemoryStore: Block rdd_24_3 of size 16 dropped from memory (free 383723813) 21/01/17 17:43:09.057 block-manager-storage-async-thread-pool-20 DEBUG BlockManager: Removing block rdd_24_2 21/01/17 17:43:09.057 block-manager-storage-async-thread-pool-20 DEBUG MemoryStore: Block rdd_24_2 of size 192 dropped from memory (free 383724005) 21/01/17 17:43:09.057 block-manager-storage-async-thread-pool-20 DEBUG BlockManager: Removing block rdd_24_9 21/01/17 17:43:09.057 block-manager-storage-async-thread-pool-20 DEBUG MemoryStore: Block rdd_24_9 of size 16 dropped from memory (free 383724021) 21/01/17 17:43:09.057 block-manager-storage-async-thread-pool-20 DEBUG BlockManager: Removing block rdd_24_1 21/01/17 17:43:09.057 block-manager-storage-async-thread-pool-20 DEBUG MemoryStore: Block rdd_24_1 of size 208 dropped from memory (free 383724229) 21/01/17 17:43:09.061 dag-scheduler-event-loop INFO DAGScheduler: Submitting 4 missing tasks from ResultStage 5 (MapPartitionsRDD[22] at top at CountVectorizer.scala:236) (first 15 tasks are for partitions Vector(0, 1, 2, 3)) 21/01/17 17:43:09.061 dag-scheduler-event-loop INFO TaskSchedulerImpl: Adding task set 5.0 with 4 tasks resource profile 0 21/01/17 17:43:09.061 dag-scheduler-event-loop DEBUG TaskSetManager: Epoch for TaskSet 5.0: 1 21/01/17 17:43:09.062 dag-scheduler-event-loop DEBUG TaskSetManager: Adding pending tasks took 0 ms 21/01/17 17:43:09.063 dag-scheduler-event-loop DEBUG TaskSetManager: Valid locality levels for TaskSet 5.0: PROCESS_LOCAL, NODE_LOCAL, ANY 21/01/17 17:43:09.063 dispatcher-event-loop-1 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_5.0, runningTasks: 0 21/01/17 17:43:09.063 block-manager-storage-async-thread-pool-24 DEBUG BlockManagerStorageEndpoint: Done removing RDD 24, response is 0 21/01/17 17:43:09.064 dispatcher-event-loop-1 INFO TaskSetManager: Starting task 0.0 in stage 5.0 (TID 16) (192.168.1.225, executor driver, partition 0, PROCESS_LOCAL, 7143 bytes) taskResourceAssignments Map() 21/01/17 17:43:09.064 dispatcher-event-loop-1 INFO TaskSetManager: Starting task 1.0 in stage 5.0 (TID 17) (192.168.1.225, executor driver, partition 1, PROCESS_LOCAL, 7143 bytes) taskResourceAssignments Map() 21/01/17 17:43:09.065 dispatcher-event-loop-1 INFO TaskSetManager: Starting task 2.0 in stage 5.0 (TID 18) (192.168.1.225, executor driver, partition 2, PROCESS_LOCAL, 7143 bytes) taskResourceAssignments Map() 21/01/17 17:43:09.065 dispatcher-event-loop-1 INFO TaskSetManager: Starting task 3.0 in stage 5.0 (TID 19) (192.168.1.225, executor driver, partition 3, PROCESS_LOCAL, 7143 bytes) taskResourceAssignments Map() 21/01/17 17:43:09.066 Executor task launch worker for task 0.0 in stage 5.0 (TID 16) INFO Executor: Running task 0.0 in stage 5.0 (TID 16) 21/01/17 17:43:09.066 Executor task launch worker for task 3.0 in stage 5.0 (TID 19) INFO Executor: Running task 3.0 in stage 5.0 (TID 19) 21/01/17 17:43:09.066 Executor task launch worker for task 1.0 in stage 5.0 (TID 17) INFO Executor: Running task 1.0 in stage 5.0 (TID 17) 21/01/17 17:43:09.066 Executor task launch worker for task 2.0 in stage 5.0 (TID 18) INFO Executor: Running task 2.0 in stage 5.0 (TID 18) 21/01/17 17:43:09.066 Executor task launch worker for task 0.0 in stage 5.0 (TID 16) DEBUG ExecutorMetricsPoller: stageTCMP: (5, 0) -> 1 21/01/17 17:43:09.067 Executor task launch worker for task 2.0 in stage 5.0 (TID 18) DEBUG ExecutorMetricsPoller: stageTCMP: (5, 0) -> 2 21/01/17 17:43:09.067 Executor task launch worker for task 3.0 in stage 5.0 (TID 19) DEBUG ExecutorMetricsPoller: stageTCMP: (5, 0) -> 3 21/01/17 17:43:09.067 Executor task launch worker for task 1.0 in stage 5.0 (TID 17) DEBUG ExecutorMetricsPoller: stageTCMP: (5, 0) -> 4 21/01/17 17:43:09.067 Executor task launch worker for task 2.0 in stage 5.0 (TID 18) DEBUG BlockManager: Getting local block broadcast_6 21/01/17 17:43:09.068 Executor task launch worker for task 2.0 in stage 5.0 (TID 18) DEBUG BlockManager: Level for block broadcast_6 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:09.073 Executor task launch worker for task 1.0 in stage 5.0 (TID 17) DEBUG BlockManager: Getting local block rdd_21_1 21/01/17 17:43:09.073 Executor task launch worker for task 2.0 in stage 5.0 (TID 18) DEBUG BlockManager: Getting local block rdd_21_2 21/01/17 17:43:09.073 Executor task launch worker for task 1.0 in stage 5.0 (TID 17) DEBUG BlockManager: Level for block rdd_21_1 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:09.073 Executor task launch worker for task 2.0 in stage 5.0 (TID 18) DEBUG BlockManager: Level for block rdd_21_2 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:09.074 Executor task launch worker for task 3.0 in stage 5.0 (TID 19) DEBUG BlockManager: Getting local block rdd_21_3 21/01/17 17:43:09.074 Executor task launch worker for task 3.0 in stage 5.0 (TID 19) DEBUG BlockManager: Level for block rdd_21_3 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:09.074 Executor task launch worker for task 1.0 in stage 5.0 (TID 17) INFO BlockManager: Found block rdd_21_1 locally 21/01/17 17:43:09.074 Executor task launch worker for task 2.0 in stage 5.0 (TID 18) INFO BlockManager: Found block rdd_21_2 locally 21/01/17 17:43:09.074 Executor task launch worker for task 3.0 in stage 5.0 (TID 19) INFO BlockManager: Found block rdd_21_3 locally 21/01/17 17:43:09.074 Executor task launch worker for task 0.0 in stage 5.0 (TID 16) DEBUG BlockManager: Getting local block rdd_21_0 21/01/17 17:43:09.074 Executor task launch worker for task 0.0 in stage 5.0 (TID 16) DEBUG BlockManager: Level for block rdd_21_0 is StorageLevel(disk, memory, deserialized, 1 replicas) 21/01/17 17:43:09.074 Executor task launch worker for task 0.0 in stage 5.0 (TID 16) INFO BlockManager: Found block rdd_21_0 locally 21/01/17 17:43:09.096 block-manager-storage-async-thread-pool-26 DEBUG BlockManagerStorageEndpoint: Done removing broadcast 4, response is 0 21/01/17 17:43:09.101 Executor task launch worker for task 1.0 in stage 5.0 (TID 17) INFO Executor: Finished task 1.0 in stage 5.0 (TID 17). 2643 bytes result sent to driver 21/01/17 17:43:09.101 dispatcher-event-loop-3 DEBUG TaskSchedulerImpl: parentName: , name: TaskSet_5.0, runningTasks: 3 21/01/17 17:43:09.102 dispatcher-event-loop-3 DEBUG TaskSetManager: No tasks for locality level PROCESS_LOCAL, so moving to locality level NODE_LOCAL 21/01/17 17:43:09.102 dispatcher-event-loop-3 DEBUG TaskSetManager: No tasks for locality level NODE_LOCAL, so moving to locality level ANY 21/01/17 17:43:09.102 block-manager-storage-async-thread-pool-24 DEBUG BlockManagerStorageEndpoint: Sent response: 0 to 192.168.1.225:41897 21/01/17 17:43:09.103 task-result-getter-0 INFO TaskSetManager: Finished task 1.0 in stage 5.0 (TID 17) in 39 ms on 192.168.1.225 (executor driver) (1/4) 21/01/17 17:43:09.105 Executor task launch worker for task 2.0 in stage 5.0 (TID 18) INFO Executor: Finished task 2.0 in stage 5.0 (TID 18). 2600 bytes result sent to driver 21/01/17 17:43:09.105 Executor task launch worker for task 2.0 in stage 5.0 (TID 18) DEBUG ExecutorMetricsPoller: stageTCMP: (5, 0) -> 3 21/01/17 17:43:09.106 task-result-getter-2 INFO TaskSetManager: Finished task 2.0 in stage 5.0 (TID 18) in 41 ms on 192.168.1.225 (executor driver) (2/4) 21/01/17 17:43:09.115 block-manager-storage-async-thread-pool-26 DEBUG BlockManagerStorageEndpoint: Sent response: 0 to 192.168.1.225:41897 21/01/17 17:43:09.117 Executor task launch worker for task 1.0 in stage 5.0 (TID 17) DEBUG ExecutorMetricsPoller: stageTCMP: (5, 0) -> 2 21/01/17 17:43:09.119 Executor task launch worker for task 3.0 in stage 5.0 (TID 19) INFO Executor: Finished task 3.0 in stage 5.0 (TID 19). 2600 bytes result sent to driver 21/01/17 17:43:09.119 Executor task launch worker for task 3.0 in stage 5.0 (TID 19) DEBUG ExecutorMetricsPoller: stageTCMP: (5, 0) -> 1 21/01/17 17:43:09.119 Spark Context Cleaner DEBUG ContextCleaner: Cleaned broadcast 4 21/01/17 17:43:09.119 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(175) 21/01/17 17:43:09.119 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 175 21/01/17 17:43:09.119 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 175 21/01/17 17:43:09.119 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(185) 21/01/17 17:43:09.119 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 185 21/01/17 17:43:09.119 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 185 21/01/17 17:43:09.120 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(71) 21/01/17 17:43:09.120 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 71 21/01/17 17:43:09.120 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 71 21/01/17 17:43:09.120 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(35) 21/01/17 17:43:09.120 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 35 21/01/17 17:43:09.120 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 35 21/01/17 17:43:09.120 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(162) 21/01/17 17:43:09.120 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 162 21/01/17 17:43:09.120 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 162 21/01/17 17:43:09.120 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(228) 21/01/17 17:43:09.120 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 228 21/01/17 17:43:09.120 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 228 21/01/17 17:43:09.120 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(189) 21/01/17 17:43:09.120 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 189 21/01/17 17:43:09.120 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 189 21/01/17 17:43:09.120 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(120) 21/01/17 17:43:09.120 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 120 21/01/17 17:43:09.120 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 120 21/01/17 17:43:09.120 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(103) 21/01/17 17:43:09.120 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 103 21/01/17 17:43:09.120 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 103 21/01/17 17:43:09.120 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(83) 21/01/17 17:43:09.120 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 83 21/01/17 17:43:09.120 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 83 21/01/17 17:43:09.120 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(181) 21/01/17 17:43:09.120 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 181 21/01/17 17:43:09.120 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 181 21/01/17 17:43:09.120 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(48) 21/01/17 17:43:09.120 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 48 21/01/17 17:43:09.120 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 48 21/01/17 17:43:09.120 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(121) 21/01/17 17:43:09.120 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 121 21/01/17 17:43:09.120 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 121 21/01/17 17:43:09.120 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(85) 21/01/17 17:43:09.120 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 85 21/01/17 17:43:09.120 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 85 21/01/17 17:43:09.120 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(63) 21/01/17 17:43:09.120 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 63 21/01/17 17:43:09.120 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 63 21/01/17 17:43:09.120 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(169) 21/01/17 17:43:09.120 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 169 21/01/17 17:43:09.120 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 169 21/01/17 17:43:09.120 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(72) 21/01/17 17:43:09.120 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 72 21/01/17 17:43:09.120 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 72 21/01/17 17:43:09.120 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(113) 21/01/17 17:43:09.120 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 113 21/01/17 17:43:09.120 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 113 21/01/17 17:43:09.120 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(203) 21/01/17 17:43:09.120 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 203 21/01/17 17:43:09.120 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 203 21/01/17 17:43:09.120 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(116) 21/01/17 17:43:09.120 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 116 21/01/17 17:43:09.120 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 116 21/01/17 17:43:09.121 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(202) 21/01/17 17:43:09.121 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 202 21/01/17 17:43:09.121 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 202 21/01/17 17:43:09.121 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(105) 21/01/17 17:43:09.121 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 105 21/01/17 17:43:09.121 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 105 21/01/17 17:43:09.121 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(170) 21/01/17 17:43:09.121 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 170 21/01/17 17:43:09.121 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 170 21/01/17 17:43:09.121 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(80) 21/01/17 17:43:09.121 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 80 21/01/17 17:43:09.121 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 80 21/01/17 17:43:09.121 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(43) 21/01/17 17:43:09.121 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 43 21/01/17 17:43:09.121 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 43 21/01/17 17:43:09.121 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(37) 21/01/17 17:43:09.121 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 37 21/01/17 17:43:09.121 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 37 21/01/17 17:43:09.121 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(146) 21/01/17 17:43:09.121 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 146 21/01/17 17:43:09.121 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 146 21/01/17 17:43:09.121 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(198) 21/01/17 17:43:09.121 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 198 21/01/17 17:43:09.121 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 198 21/01/17 17:43:09.121 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(60) 21/01/17 17:43:09.121 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 60 21/01/17 17:43:09.121 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 60 21/01/17 17:43:09.121 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(124) 21/01/17 17:43:09.121 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 124 21/01/17 17:43:09.121 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 124 21/01/17 17:43:09.121 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(205) 21/01/17 17:43:09.121 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 205 21/01/17 17:43:09.121 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 205 21/01/17 17:43:09.121 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(108) 21/01/17 17:43:09.121 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 108 21/01/17 17:43:09.121 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 108 21/01/17 17:43:09.121 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(119) 21/01/17 17:43:09.121 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 119 21/01/17 17:43:09.121 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 119 21/01/17 17:43:09.121 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(90) 21/01/17 17:43:09.121 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 90 21/01/17 17:43:09.121 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 90 21/01/17 17:43:09.121 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(131) 21/01/17 17:43:09.121 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 131 21/01/17 17:43:09.121 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 131 21/01/17 17:43:09.121 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(36) 21/01/17 17:43:09.121 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 36 21/01/17 17:43:09.121 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 36 21/01/17 17:43:09.121 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(61) 21/01/17 17:43:09.121 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 61 21/01/17 17:43:09.121 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 61 21/01/17 17:43:09.121 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(70) 21/01/17 17:43:09.121 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 70 21/01/17 17:43:09.122 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 70 21/01/17 17:43:09.122 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(167) 21/01/17 17:43:09.122 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 167 21/01/17 17:43:09.122 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 167 21/01/17 17:43:09.122 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(157) 21/01/17 17:43:09.122 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 157 21/01/17 17:43:09.122 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 157 21/01/17 17:43:09.122 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(128) 21/01/17 17:43:09.122 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 128 21/01/17 17:43:09.122 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 128 21/01/17 17:43:09.122 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(38) 21/01/17 17:43:09.122 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 38 21/01/17 17:43:09.122 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 38 21/01/17 17:43:09.122 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(79) 21/01/17 17:43:09.122 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 79 21/01/17 17:43:09.122 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 79 21/01/17 17:43:09.122 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(98) 21/01/17 17:43:09.122 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 98 21/01/17 17:43:09.122 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 98 21/01/17 17:43:09.122 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(62) 21/01/17 17:43:09.122 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 62 21/01/17 17:43:09.122 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 62 21/01/17 17:43:09.122 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(87) 21/01/17 17:43:09.122 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 87 21/01/17 17:43:09.122 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 87 21/01/17 17:43:09.122 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(158) 21/01/17 17:43:09.122 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 158 21/01/17 17:43:09.122 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 158 21/01/17 17:43:09.122 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(86) 21/01/17 17:43:09.122 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 86 21/01/17 17:43:09.122 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 86 21/01/17 17:43:09.122 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanBroadcast(7) 21/01/17 17:43:09.122 Spark Context Cleaner DEBUG ContextCleaner: Cleaning broadcast 7 21/01/17 17:43:09.122 Spark Context Cleaner DEBUG TorrentBroadcast: Unpersisting TorrentBroadcast 7 21/01/17 17:43:09.124 task-result-getter-1 INFO TaskSetManager: Finished task 3.0 in stage 5.0 (TID 19) in 59 ms on 192.168.1.225 (executor driver) (3/4) 21/01/17 17:43:09.140 block-manager-storage-async-thread-pool-27 DEBUG BlockManagerStorageEndpoint: removing broadcast 7 21/01/17 17:43:09.140 block-manager-storage-async-thread-pool-27 DEBUG BlockManager: Removing broadcast 7 21/01/17 17:43:09.140 block-manager-storage-async-thread-pool-27 DEBUG BlockManager: Removing block broadcast_7 21/01/17 17:43:09.140 block-manager-storage-async-thread-pool-27 DEBUG MemoryStore: Block broadcast_7 of size 39472 dropped from memory (free 383763701) 21/01/17 17:43:09.140 block-manager-storage-async-thread-pool-27 DEBUG BlockManager: Removing block broadcast_7_piece0 21/01/17 17:43:09.147 block-manager-storage-async-thread-pool-27 DEBUG MemoryStore: Block broadcast_7_piece0 of size 17805 dropped from memory (free 383781506) 21/01/17 17:43:09.147 dispatcher-BlockManagerMaster DEBUG BlockManagerMasterEndpoint: Updating block info on master broadcast_7_piece0 for BlockManagerId(driver, 192.168.1.225, 40985, None) 21/01/17 17:43:09.147 dispatcher-BlockManagerMaster INFO BlockManagerInfo: Removed broadcast_7_piece0 on 192.168.1.225:40985 in memory (size: 17.4 KiB, free: 366.2 MiB) 21/01/17 17:43:09.148 block-manager-storage-async-thread-pool-27 DEBUG BlockManagerMaster: Updated info of block broadcast_7_piece0 21/01/17 17:43:09.148 block-manager-storage-async-thread-pool-27 DEBUG BlockManager: Told master about block broadcast_7_piece0 21/01/17 17:43:09.163 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(53) 21/01/17 17:43:09.163 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 53 21/01/17 17:43:09.165 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 53 21/01/17 17:43:09.165 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(81) 21/01/17 17:43:09.165 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 81 21/01/17 17:43:09.165 Executor task launch worker for task 0.0 in stage 5.0 (TID 16) INFO Executor: Finished task 0.0 in stage 5.0 (TID 16). 2469 bytes result sent to driver 21/01/17 17:43:09.165 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 81 21/01/17 17:43:09.165 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(84) 21/01/17 17:43:09.165 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 84 21/01/17 17:43:09.165 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 84 21/01/17 17:43:09.165 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(49) 21/01/17 17:43:09.165 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 49 21/01/17 17:43:09.165 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 49 21/01/17 17:43:09.165 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(105) 21/01/17 17:43:09.165 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 105 21/01/17 17:43:09.165 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 105 21/01/17 17:43:09.165 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(45) 21/01/17 17:43:09.165 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 45 21/01/17 17:43:09.165 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 45 21/01/17 17:43:09.165 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(91) 21/01/17 17:43:09.165 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 91 21/01/17 17:43:09.166 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 91 21/01/17 17:43:09.166 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(106) 21/01/17 17:43:09.166 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 106 21/01/17 17:43:09.166 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 106 21/01/17 17:43:09.166 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(66) 21/01/17 17:43:09.166 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 66 21/01/17 17:43:09.166 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 66 21/01/17 17:43:09.166 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(101) 21/01/17 17:43:09.166 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 101 21/01/17 17:43:09.166 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 101 21/01/17 17:43:09.166 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(57) 21/01/17 17:43:09.166 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 57 21/01/17 17:43:09.166 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 57 21/01/17 17:43:09.166 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(90) 21/01/17 17:43:09.166 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 90 21/01/17 17:43:09.166 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 90 21/01/17 17:43:09.166 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(54) 21/01/17 17:43:09.166 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 54 21/01/17 17:43:09.166 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 54 21/01/17 17:43:09.166 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(36) 21/01/17 17:43:09.166 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 36 21/01/17 17:43:09.166 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 36 21/01/17 17:43:09.166 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(46) 21/01/17 17:43:09.166 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 46 21/01/17 17:43:09.166 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 46 21/01/17 17:43:09.166 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(72) 21/01/17 17:43:09.166 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 72 21/01/17 17:43:09.166 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 72 21/01/17 17:43:09.166 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(35) 21/01/17 17:43:09.166 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 35 21/01/17 17:43:09.166 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 35 21/01/17 17:43:09.166 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(42) 21/01/17 17:43:09.166 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 42 21/01/17 17:43:09.166 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 42 21/01/17 17:43:09.167 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(85) 21/01/17 17:43:09.167 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 85 21/01/17 17:43:09.167 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 85 21/01/17 17:43:09.167 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(32) 21/01/17 17:43:09.168 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 32 21/01/17 17:43:09.168 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 32 21/01/17 17:43:09.168 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanAccum(87) 21/01/17 17:43:09.168 Spark Context Cleaner DEBUG ContextCleaner: Cleaning accumulator 87 21/01/17 17:43:09.168 Spark Context Cleaner DEBUG ContextCleaner: Cleaned accumulator 87 21/01/17 17:43:09.168 Spark Context Cleaner DEBUG ContextCleaner: Got cleaning task CleanBroadcast(4) 21/01/17 17:43:09.168 Executor task launch worker for task 0.0 in stage 3.0 (TID 9) DEBUG GenerateSafeProjection: code for newInstance(class org.apache.spark.ml.linalg.VectorUDT).deserialize: /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificSafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificSafeProjection extends org.apache.spark.sql.catalyst.expressions.codegen.BaseProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private InternalRow mutableRow; /* 009 */ private boolean resultIsNull_0; /* 010 */ private InternalRow[] mutableStateArray_0 = new InternalRow[1]; /* 011 */ /* 012 */ public SpecificSafeProjection(Object[] references) { /* 013 */ this.references = references; /* 014 */ mutableRow = (InternalRow) references[references.length - 1]; /* 015 */ /* 016 */ /* 017 */ } /* 018 */ /* 019 */ public void initialize(int partitionIndex) { /* 020 */ /* 021 */ } /* 022 */ /* 023 */ public java.lang.Object apply(java.lang.Object _i) { /* 024 */ InternalRow i = (InternalRow) _i; /* 025 */ final org.apache.spark.ml.linalg.VectorUDT value_1 = false ? /* 026 */ null : new org.apache.spark.ml.linalg.VectorUDT(); /* 027 */ boolean isNull_0 = true; /* 028 */ org.apache.spark.ml.linalg.Vector value_0 = null; /* 029 */ if (!false) { /* 030 */ resultIsNull_0 = false; /* 031 */ if (!resultIsNull_0) { /* 032 */ boolean isNull_2 = i.isNullAt(0); /* 033 */ InternalRow value_2 = isNull_2 ? /* 034 */ null : (i.getStruct(0, 4)); /* 035 */ resultIsNull_0 = isNull_2; /* 036 */ mutableStateArray_0[0] = value_2; /* 037 */ } /* 038 */ /* 039 */ isNull_0 = resultIsNull_0; /* 040 */ if (!isNull_0) { /* 041 */ /* 042 */ Object funcResult_0 = null; /* 043 */ funcResult_0 = value_1.deserialize(mutableStateArray_0[0]); /* 044 */ /* 045 */ if (funcResult_0 != null) { /* 046 */ value_0 = (org.apache.spark.ml.linalg.Vector) funcResult_0; /* 047 */ } else { /* 048 */ isNull_0 = true; /* 049 */ } /* 050 */ /* 051 */ /* 052 */ } /* 053 */ } /* 054 */ if (isNull_0) { /* 055 */ mutableRow.setNullAt(0); /* 056 */ } else { /* 057 */ /* 058 */ mutableRow.update(0, value_0); /* 059 */ } /* 060 */ /* 061 */ return mutableRow; /* 062 */ } /* 063 */ /* 064 */ /* 065 */ } 21/01/17 17:43:09.168 Spark Context Cleaner DEBUG ContextCleaner: Cleaning broadcast 4 21/01/17 17:43:09.169 task-result-getter-3 INFO TaskSetManager: Finished task 0.0 in stage 5.0 (TID 16) in 105 ms on 192.168.1.225 (executor driver) (4/4) 21/01/17 17:43:09.169 task-result-getter-3 INFO TaskSchedulerImpl: Removed TaskSet 5.0, whose tasks have all completed, from pool 21/01/17 17:43:09.168 Executor task launch worker for task 0.0 in stage 5.0 (TID 16) DEBUG ExecutorMetricsPoller: removing (5, 0) from stageTCMP 21/01/17 17:43:09.169 Spark Context Cleaner DEBUG TorrentBroadcast: Unpersisting TorrentBroadcast 4 21/01/17 17:43:09.170 dag-scheduler-event-loop INFO DAGScheduler: ResultStage 5 (top at CountVectorizer.scala:236) finished in 0.129 s 21/01/17 17:43:09.170 dag-scheduler-event-loop DEBUG DAGScheduler: After removal of stage 5, remaining stages = 1 21/01/17 17:43:09.170 dag-scheduler-event-loop DEBUG DAGScheduler: After removal of stage 4, remaining stages = 0 21/01/17 17:43:09.170 dag-scheduler-event-loop INFO DAGScheduler: Job 3 is finished. Cancelling potential speculative or zombie tasks for this job 21/01/17 17:43:09.170 dag-scheduler-event-loop INFO TaskSchedulerImpl: Killing all running tasks in stage 5: Stage finished 21/01/17 17:43:09.171 Executor task launch worker for task 3.0 in stage 3.0 (TID 12) DEBUG GenerateSafeProjection: code for newInstance(class org.apache.spark.ml.linalg.VectorUDT).deserialize: /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificSafeProjection(references); /* 003 */ } /* 004