package me.zrj.test.test20170731
import org.apache.spark.SparkContext
import org.apache.spark.SparkConf
import java.util.Properties
import org.apache.log4j.PropertyConfigurator
import org.apache.spark.streaming.StreamingContext
import org.apache.spark.streaming.Seconds
object SparkWordCount {
val logger = getLogger(this.getClass, properties=null)
def getLogger(clazz: Class[_], properties: Properties = null): org.apache.log4j.Logger = {
if (properties == null) {
// 配置日志选项
val prop = new Properties()
prop.setProperty("log4j.logger.me.zrj.test.test20170731", "INFO, stdout")
prop.setProperty("log4j.appender.stdout", "org.apache.log4j.ConsoleAppender")
prop.setProperty("log4j.appender.stdout.layout", "org.apache.log4j.PatternLayout")
prop.setProperty("log4j.appender.stdout.layout.ConversionPattern", "[%d][%-5p]%m -- %F:%L%n")
// 让日志配置生效
PropertyConfigurator.configure(prop)
} else {
PropertyConfigurator.configure(properties)
}
org.apache.log4j.Logger.getLogger(clazz)
}
def wordCount() {
val sc = new SparkContext(new SparkConf().setMaster("local[2]").setAppName("Spark Count"))
val textFile = sc.textFile("file:///D://downloads//hdfs-site.xml")
val counts = textFile.flatMap(line => line.split(" "))
.map(word => (word, 1))
.reduceByKey(_ + _)
val res = counts.collect.sortBy( - _._2).slice(0, 10)
logger.info(s"res ${res.toList}")
}
def socketStreaming() {
val serverIP = "192.168.56.101"
val serverPort = 9999
// 创建StreamingContext，1秒一个批次
val ssc = new StreamingContext(new SparkConf().setMaster("local[2]").setAppName("Spark Count"), Seconds(1));
// 获得一个DStream负责连接 监听端口:地址
val lines = ssc.socketTextStream(serverIP, serverPort);
// 对每一行数据执行Split操作
val words = lines.flatMap(_.split(" ").toSeq);
// 统计word的数量
val pairs = words.map(word => (word, 1));
val wordCounts = pairs.reduceByKey(_ + _);
// 输出结果
wordCounts.print();
ssc.start(); // 开始
ssc.awaitTermination(); // 计算完毕退出
}
def main(args: Array[String]) {
logger.info(s"starting spark...")
wordCount()
logger.info(s"program terminated");
}
}
跑起来之后，word count 的输出是这样的
[2018-06-26 10:12:31,086][INFO ]starting spark... -- SparkWordCount.scala:62 Using Spark's default log4j profile: org/apache/spark/log4j-defaults.properties 18/06/26 10:12:31 INFO SparkContext: Running Spark version 1.6.1 18/06/26 10:12:31 INFO SecurityManager: Changing view acls to: adenzhang 18/06/26 10:12:31 INFO SecurityManager: Changing modify acls to: adenzhang 18/06/26 10:12:31 INFO SecurityManager: SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(adenzhang); users with modify permissions: Set(adenzhang) 18/06/26 10:12:32 INFO Utils: Successfully started service 'sparkDriver' on port 53504. 18/06/26 10:12:32 INFO Slf4jLogger: Slf4jLogger started 18/06/26 10:12:32 INFO Remoting: Starting remoting 18/06/26 10:12:32 INFO Remoting: Remoting started; listening on addresses :[akka.tcp://sparkDriverActorSystem@10.41.88.38:53517] 18/06/26 10:12:32 INFO Utils: Successfully started service 'sparkDriverActorSystem' on port 53517. 18/06/26 10:12:32 INFO SparkEnv: Registering MapOutputTracker 18/06/26 10:12:32 INFO SparkEnv: Registering BlockManagerMaster 18/06/26 10:12:32 INFO DiskBlockManager: Created local directory at C:\Users\adenzhang\AppData\Local\Temp\blockmgr-fa868aad-7249-4bf9-934d-84118c5ee523 18/06/26 10:12:32 INFO MemoryStore: MemoryStore started with capacity 2.4 GB 18/06/26 10:12:32 INFO SparkEnv: Registering OutputCommitCoordinator 18/06/26 10:12:32 INFO Utils: Successfully started service 'SparkUI' on port 4040. 18/06/26 10:12:32 INFO SparkUI: Started SparkUI at http://10.41.88.38:4040 18/06/26 10:12:32 INFO Executor: Starting executor ID driver on host localhost 18/06/26 10:12:32 INFO Utils: Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 53530. 18/06/26 10:12:32 INFO NettyBlockTransferService: Server created on 53530 18/06/26 10:12:32 INFO BlockManagerMaster: Trying to register BlockManager 18/06/26 10:12:32 INFO BlockManagerMasterEndpoint: Registering block manager localhost:53530 with 2.4 GB RAM, BlockManagerId(driver, localhost, 53530) 18/06/26 10:12:32 INFO BlockManagerMaster: Registered BlockManager 18/06/26 10:12:33 INFO MemoryStore: Block broadcast_0 stored as values in memory (estimated size 107.7 KB, free 107.7 KB) 18/06/26 10:12:33 INFO MemoryStore: Block broadcast_0_piece0 stored as bytes in memory (estimated size 9.8 KB, free 117.5 KB) 18/06/26 10:12:33 INFO BlockManagerInfo: Added broadcast_0_piece0 in memory on localhost:53530 (size: 9.8 KB, free: 2.4 GB) 18/06/26 10:12:33 INFO SparkContext: Created broadcast 0 from textFile at SparkWordCount.scala:30 18/06/26 10:12:33 INFO SparkContext: Starting job: collect at SparkWordCount.scala:34 18/06/26 10:12:33 INFO DAGScheduler: Registering RDD 3 (map at SparkWordCount.scala:32) 18/06/26 10:12:33 INFO DAGScheduler: Got job 0 (collect at SparkWordCount.scala:34) with 2 output partitions 18/06/26 10:12:33 INFO DAGScheduler: Final stage: ResultStage 1 (collect at SparkWordCount.scala:34) 18/06/26 10:12:33 INFO DAGScheduler: Parents of final stage: List(ShuffleMapStage 0) 18/06/26 10:12:33 INFO DAGScheduler: Missing parents: List(ShuffleMapStage 0) 18/06/26 10:12:33 INFO DAGScheduler: Submitting ShuffleMapStage 0 (MapPartitionsRDD[3] at map at SparkWordCount.scala:32), which has no missing parents 18/06/26 10:12:33 INFO MemoryStore: Block broadcast_1 stored as values in memory (estimated size 4.1 KB, free 121.6 KB) 18/06/26 10:12:33 INFO MemoryStore: Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.3 KB, free 123.9 KB) 18/06/26 10:12:33 INFO BlockManagerInfo: Added broadcast_1_piece0 in memory on localhost:53530 (size: 2.3 KB, free: 2.4 GB) 18/06/26 10:12:33 INFO SparkContext: Created broadcast 1 from broadcast at DAGScheduler.scala:1015 18/06/26 10:12:33 INFO DAGScheduler: Submitting 2 missing tasks from ShuffleMapStage 0 (MapPartitionsRDD[3] at map at SparkWordCount.scala:32) 18/06/26 10:12:33 INFO TaskSchedulerImpl: Adding task set 0.0 with 2 tasks 18/06/26 10:12:33 INFO TaskSetManager: Starting task 0.0 in stage 0.0 (TID 0, localhost, partition 0,PROCESS_LOCAL, 2123 bytes) 18/06/26 10:12:33 INFO TaskSetManager: Starting task 1.0 in stage 0.0 (TID 1, localhost, partition 1,PROCESS_LOCAL, 2123 bytes) 18/06/26 10:12:33 INFO Executor: Running task 1.0 in stage 0.0 (TID 1) 18/06/26 10:12:33 INFO Executor: Running task 0.0 in stage 0.0 (TID 0) 18/06/26 10:12:33 INFO HadoopRDD: Input split: file:/D:/downloads/hdfs-site.xml:8809+8809 18/06/26 10:12:33 INFO HadoopRDD: Input split: file:/D:/downloads/hdfs-site.xml:0+8809 18/06/26 10:12:33 INFO Executor: Finished task 1.0 in stage 0.0 (TID 1). 2254 bytes result sent to driver 18/06/26 10:12:33 INFO Executor: Finished task 0.0 in stage 0.0 (TID 0). 2254 bytes result sent to driver 18/06/26 10:12:33 INFO TaskSetManager: Finished task 1.0 in stage 0.0 (TID 1) in 141 ms on localhost (1/2) 18/06/26 10:12:33 INFO TaskSetManager: Finished task 0.0 in stage 0.0 (TID 0) in 158 ms on localhost (2/2) 18/06/26 10:12:33 INFO TaskSchedulerImpl: Removed TaskSet 0.0, whose tasks have all completed, from pool 18/06/26 10:12:33 INFO DAGScheduler: ShuffleMapStage 0 (map at SparkWordCount.scala:32) finished in 0.170 s 18/06/26 10:12:33 INFO DAGScheduler: looking for newly runnable stages 18/06/26 10:12:33 INFO DAGScheduler: running: Set() 18/06/26 10:12:33 INFO DAGScheduler: waiting: Set(ResultStage 1) 18/06/26 10:12:33 INFO DAGScheduler: failed: Set() 18/06/26 10:12:33 INFO DAGScheduler: Submitting ResultStage 1 (ShuffledRDD[4] at reduceByKey at SparkWordCount.scala:33), which has no missing parents 18/06/26 10:12:33 INFO MemoryStore: Block broadcast_2 stored as values in memory (estimated size 2.6 KB, free 126.5 KB) 18/06/26 10:12:33 INFO MemoryStore: Block broadcast_2_piece0 stored as bytes in memory (estimated size 1603.0 B, free 128.1 KB) 18/06/26 10:12:33 INFO BlockManagerInfo: Added broadcast_2_piece0 in memory on localhost:53530 (size: 1603.0 B, free: 2.4 GB) 18/06/26 10:12:33 INFO SparkContext: Created broadcast 2 from broadcast at DAGScheduler.scala:1015 18/06/26 10:12:33 INFO DAGScheduler: Submitting 2 missing tasks from ResultStage 1 (ShuffledRDD[4] at reduceByKey at SparkWordCount.scala:33) 18/06/26 10:12:33 INFO TaskSchedulerImpl: Adding task set 1.0 with 2 tasks 18/06/26 10:12:33 INFO TaskSetManager: Starting task 0.0 in stage 1.0 (TID 2, localhost, partition 0,NODE_LOCAL, 1894 bytes) 18/06/26 10:12:33 INFO TaskSetManager: Starting task 1.0 in stage 1.0 (TID 3, localhost, partition 1,NODE_LOCAL, 1894 bytes) 18/06/26 10:12:33 INFO Executor: Running task 0.0 in stage 1.0 (TID 2) 18/06/26 10:12:33 INFO Executor: Running task 1.0 in stage 1.0 (TID 3) 18/06/26 10:12:33 INFO ShuffleBlockFetcherIterator: Getting 2 non-empty blocks out of 2 blocks 18/06/26 10:12:33 INFO ShuffleBlockFetcherIterator: Getting 2 non-empty blocks out of 2 blocks 18/06/26 10:12:33 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 3 ms 18/06/26 10:12:33 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 3 ms 18/06/26 10:12:33 INFO Executor: Finished task 1.0 in stage 1.0 (TID 3). 9200 bytes result sent to driver 18/06/26 10:12:33 INFO Executor: Finished task 0.0 in stage 1.0 (TID 2). 8237 bytes result sent to driver 18/06/26 10:12:33 INFO TaskSetManager: Finished task 1.0 in stage 1.0 (TID 3) in 44 ms on localhost (1/2) 18/06/26 10:12:33 INFO TaskSetManager: Finished task 0.0 in stage 1.0 (TID 2) in 46 ms on localhost (2/2) 18/06/26 10:12:33 INFO TaskSchedulerImpl: Removed TaskSet 1.0, whose tasks have all completed, from pool 18/06/26 10:12:33 INFO DAGScheduler: ResultStage 1 (collect at SparkWordCount.scala:34) finished in 0.046 s 18/06/26 10:12:33 INFO DAGScheduler: Job 0 finished: collect at SparkWordCount.scala:34, took 0.294973 s [2018-06-26 10:12:33,654][INFO ]res List((,1926), (</property>,107), (<property>,107), (the,15), (<value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>,13), (<value>nn1,nn2,nn3</value>,9), (is,5), (<value>nn1,nn2</value>,4), (will,4), (port,4)) -- SparkWordCount.scala:35 18/06/26 10:12:33 INFO SparkWordCount$: res List((,1926), (</property>,107), (<property>,107), (the,15), (<value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>,13), (<value>nn1,nn2,nn3</value>,9), (is,5), (<value>nn1,nn2</value>,4), (will,4), (port,4)) 18/06/26 10:12:33 INFO SparkWordCount$: program terminated [2018-06-26 10:12:33,655][INFO ]program terminated -- SparkWordCount.scala:64 18/06/26 10:12:33 INFO SparkContext: Invoking stop() from shutdown hook 18/06/26 10:12:33 INFO SparkUI: Stopped Spark web UI at http://10.41.88.38:4040 18/06/26 10:12:33 INFO MapOutputTrackerMasterEndpoint: MapOutputTrackerMasterEndpoint stopped! 18/06/26 10:12:33 INFO MemoryStore: MemoryStore cleared 18/06/26 10:12:33 INFO BlockManager: BlockManager stopped 18/06/26 10:12:33 INFO BlockManagerMaster: BlockManagerMaster stopped 18/06/26 10:12:33 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint: OutputCommitCoordinator stopped! 18/06/26 10:12:33 INFO SparkContext: Successfully stopped SparkContext 18/06/26 10:12:33 INFO ShutdownHookManager: Shutdown hook called 18/06/26 10:12:33 INFO ShutdownHookManager: Deleting directory C:\Users\adenzhang\AppData\Local\Temp\spark-104c1b9e-bf57-442e-981a-9d6461567197 18/06/26 10:12:33 INFO RemoteActorRefProvider$RemotingTerminator: Shutting down remote daemon. 18/06/26 10:12:33 INFO RemoteActorRefProvider$RemotingTerminator: Remote daemon shut down; proceeding with flushing remote transports.
实时的任务跑起来的日志输出如下
[2018-06-26 10:55:44,901][INFO ]starting spark... -- SparkWordCount.scala:62 Using Spark's default log4j profile: org/apache/spark/log4j-defaults.properties 18/06/26 10:55:45 INFO SparkContext: Running Spark version 1.6.1 18/06/26 10:55:45 INFO SecurityManager: Changing view acls to: adenzhang 18/06/26 10:55:45 INFO SecurityManager: Changing modify acls to: adenzhang 18/06/26 10:55:45 INFO SecurityManager: SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(adenzhang); users with modify permissions: Set(adenzhang) 18/06/26 10:55:45 INFO Utils: Successfully started service 'sparkDriver' on port 55640. 18/06/26 10:55:46 INFO Slf4jLogger: Slf4jLogger started 18/06/26 10:55:46 INFO Remoting: Starting remoting 18/06/26 10:55:46 INFO Remoting: Remoting started; listening on addresses :[akka.tcp://sparkDriverActorSystem@10.41.88.38:55654] 18/06/26 10:55:46 INFO Utils: Successfully started service 'sparkDriverActorSystem' on port 55654. 18/06/26 10:55:46 INFO SparkEnv: Registering MapOutputTracker 18/06/26 10:55:46 INFO SparkEnv: Registering BlockManagerMaster 18/06/26 10:55:46 INFO DiskBlockManager: Created local directory at C:\Users\adenzhang\AppData\Local\Temp\blockmgr-ed31e89a-8ef4-408c-91ec-33d881b30755 18/06/26 10:55:46 INFO MemoryStore: MemoryStore started with capacity 2.4 GB 18/06/26 10:55:46 INFO SparkEnv: Registering OutputCommitCoordinator 18/06/26 10:55:46 INFO Utils: Successfully started service 'SparkUI' on port 4040. 18/06/26 10:55:46 INFO SparkUI: Started SparkUI at http://10.41.88.38:4040 18/06/26 10:55:46 INFO Executor: Starting executor ID driver on host localhost 18/06/26 10:55:46 INFO Utils: Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 55667. 18/06/26 10:55:46 INFO NettyBlockTransferService: Server created on 55667 18/06/26 10:55:46 INFO BlockManagerMaster: Trying to register BlockManager 18/06/26 10:55:46 INFO BlockManagerMasterEndpoint: Registering block manager localhost:55667 with 2.4 GB RAM, BlockManagerId(driver, localhost, 55667) 18/06/26 10:55:46 INFO BlockManagerMaster: Registered BlockManager 18/06/26 10:55:47 INFO ReceiverTracker: Starting 1 receivers 18/06/26 10:55:47 INFO ReceiverTracker: ReceiverTracker started 18/06/26 10:55:47 INFO ForEachDStream: metadataCleanupDelay = -1 18/06/26 10:55:47 INFO ShuffledDStream: metadataCleanupDelay = -1 18/06/26 10:55:47 INFO MappedDStream: metadataCleanupDelay = -1 18/06/26 10:55:47 INFO FlatMappedDStream: metadataCleanupDelay = -1 18/06/26 10:55:47 INFO SocketInputDStream: metadataCleanupDelay = -1 18/06/26 10:55:47 INFO SocketInputDStream: Slide time = 1000 ms 18/06/26 10:55:47 INFO SocketInputDStream: Storage level = StorageLevel(false, false, false, false, 1) 18/06/26 10:55:47 INFO SocketInputDStream: Checkpoint interval = null 18/06/26 10:55:47 INFO SocketInputDStream: Remember duration = 1000 ms 18/06/26 10:55:47 INFO SocketInputDStream: Initialized and validated org.apache.spark.streaming.dstream.SocketInputDStream@258773e4 18/06/26 10:55:47 INFO FlatMappedDStream: Slide time = 1000 ms 18/06/26 10:55:47 INFO FlatMappedDStream: Storage level = StorageLevel(false, false, false, false, 1) 18/06/26 10:55:47 INFO FlatMappedDStream: Checkpoint interval = null 18/06/26 10:55:47 INFO FlatMappedDStream: Remember duration = 1000 ms 18/06/26 10:55:47 INFO FlatMappedDStream: Initialized and validated org.apache.spark.streaming.dstream.FlatMappedDStream@509141bb 18/06/26 10:55:47 INFO MappedDStream: Slide time = 1000 ms 18/06/26 10:55:47 INFO MappedDStream: Storage level = StorageLevel(false, false, false, false, 1) 18/06/26 10:55:47 INFO MappedDStream: Checkpoint interval = null 18/06/26 10:55:47 INFO MappedDStream: Remember duration = 1000 ms 18/06/26 10:55:47 INFO MappedDStream: Initialized and validated org.apache.spark.streaming.dstream.MappedDStream@3a762c6a 18/06/26 10:55:47 INFO ShuffledDStream: Slide time = 1000 ms 18/06/26 10:55:47 INFO ShuffledDStream: Storage level = StorageLevel(false, false, false, false, 1) 18/06/26 10:55:47 INFO ShuffledDStream: Checkpoint interval = null 18/06/26 10:55:47 INFO ShuffledDStream: Remember duration = 1000 ms 18/06/26 10:55:47 INFO ShuffledDStream: Initialized and validated org.apache.spark.streaming.dstream.ShuffledDStream@19eb0f26 18/06/26 10:55:47 INFO ForEachDStream: Slide time = 1000 ms 18/06/26 10:55:47 INFO ForEachDStream: Storage level = StorageLevel(false, false, false, false, 1) 18/06/26 10:55:47 INFO ForEachDStream: Checkpoint interval = null 18/06/26 10:55:47 INFO ForEachDStream: Remember duration = 1000 ms 18/06/26 10:55:47 INFO ForEachDStream: Initialized and validated org.apache.spark.streaming.dstream.ForEachDStream@7ed47ac 18/06/26 10:55:47 INFO RecurringTimer: Started timer for JobGenerator at time 1529981748000 18/06/26 10:55:47 INFO JobGenerator: Started JobGenerator at 1529981748000 ms 18/06/26 10:55:47 INFO JobScheduler: Started JobScheduler 18/06/26 10:55:47 INFO StreamingContext: StreamingContext started 18/06/26 10:55:47 INFO ReceiverTracker: Receiver 0 started 18/06/26 10:55:47 INFO DAGScheduler: Got job 0 (start at SparkWordCount.scala:57) with 1 output partitions 18/06/26 10:55:47 INFO DAGScheduler: Final stage: ResultStage 0 (start at SparkWordCount.scala:57) 18/06/26 10:55:47 INFO DAGScheduler: Parents of final stage: List() 18/06/26 10:55:47 INFO DAGScheduler: Missing parents: List() 18/06/26 10:55:47 INFO DAGScheduler: Submitting ResultStage 0 (Receiver 0 ParallelCollectionRDD[0] at makeRDD at ReceiverTracker.scala:588), which has no missing parents 18/06/26 10:55:47 INFO MemoryStore: Block broadcast_0 stored as values in memory (estimated size 34.1 KB, free 34.1 KB) 18/06/26 10:55:47 INFO MemoryStore: Block broadcast_0_piece0 stored as bytes in memory (estimated size 11.0 KB, free 45.1 KB) 18/06/26 10:55:47 INFO BlockManagerInfo: Added broadcast_0_piece0 in memory on localhost:55667 (size: 11.0 KB, free: 2.4 GB) 18/06/26 10:55:47 INFO SparkContext: Created broadcast 0 from broadcast at DAGScheduler.scala:1015 18/06/26 10:55:47 INFO DAGScheduler: Submitting 1 missing tasks from ResultStage 0 (Receiver 0 ParallelCollectionRDD[0] at makeRDD at ReceiverTracker.scala:588) 18/06/26 10:55:47 INFO TaskSchedulerImpl: Adding task set 0.0 with 1 tasks 18/06/26 10:55:47 INFO TaskSetManager: Starting task 0.0 in stage 0.0 (TID 0, localhost, partition 0,PROCESS_LOCAL, 2642 bytes) 18/06/26 10:55:47 INFO Executor: Running task 0.0 in stage 0.0 (TID 0) 18/06/26 10:55:47 INFO RecurringTimer: Started timer for BlockGenerator at time 1529981747400 18/06/26 10:55:47 INFO BlockGenerator: Started BlockGenerator 18/06/26 10:55:47 INFO BlockGenerator: Started block pushing thread 18/06/26 10:55:47 INFO ReceiverTracker: Registered receiver for stream 0 from 10.41.88.38:55640 18/06/26 10:55:47 INFO ReceiverSupervisorImpl: Starting receiver 18/06/26 10:55:47 INFO ReceiverSupervisorImpl: Called receiver onStart 18/06/26 10:55:47 INFO ReceiverSupervisorImpl: Waiting for receiver to be stopped 18/06/26 10:55:47 INFO SocketReceiver: Connecting to 192.168.56.101:9999 18/06/26 10:55:48 INFO JobScheduler: Added jobs for time 1529981748000 ms 18/06/26 10:55:48 INFO JobScheduler: Starting job streaming job 1529981748000 ms.0 from job set of time 1529981748000 ms 18/06/26 10:55:48 INFO SparkContext: Starting job: print at SparkWordCount.scala:55 18/06/26 10:55:48 INFO DAGScheduler: Registering RDD 3 (map at SparkWordCount.scala:51) 18/06/26 10:55:48 INFO DAGScheduler: Got job 1 (print at SparkWordCount.scala:55) with 1 output partitions 18/06/26 10:55:48 INFO DAGScheduler: Final stage: ResultStage 2 (print at SparkWordCount.scala:55) 18/06/26 10:55:48 INFO DAGScheduler: Parents of final stage: List(ShuffleMapStage 1) 18/06/26 10:55:48 INFO DAGScheduler: Missing parents: List() 18/06/26 10:55:48 INFO DAGScheduler: Submitting ResultStage 2 (ShuffledRDD[4] at reduceByKey at SparkWordCount.scala:52), which has no missing parents 18/06/26 10:55:48 INFO MemoryStore: Block broadcast_1 stored as values in memory (estimated size 2.6 KB, free 47.7 KB) 18/06/26 10:55:48 INFO MemoryStore: Block broadcast_1_piece0 stored as bytes in memory (estimated size 1641.0 B, free 49.3 KB) 18/06/26 10:55:48 INFO BlockManagerInfo: Added broadcast_1_piece0 in memory on localhost:55667 (size: 1641.0 B, free: 2.4 GB) 18/06/26 10:55:48 INFO SparkContext: Created broadcast 1 from broadcast at DAGScheduler.scala:1015 18/06/26 10:55:48 INFO DAGScheduler: Submitting 1 missing tasks from ResultStage 2 (ShuffledRDD[4] at reduceByKey at SparkWordCount.scala:52) 18/06/26 10:55:48 INFO TaskSchedulerImpl: Adding task set 2.0 with 1 tasks 18/06/26 10:55:48 INFO TaskSetManager: Starting task 0.0 in stage 2.0 (TID 1, localhost, partition 0,PROCESS_LOCAL, 1894 bytes) 18/06/26 10:55:48 INFO Executor: Running task 0.0 in stage 2.0 (TID 1) 18/06/26 10:55:48 INFO ShuffleBlockFetcherIterator: Getting 0 non-empty blocks out of 0 blocks 18/06/26 10:55:48 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 3 ms 18/06/26 10:55:48 INFO Executor: Finished task 0.0 in stage 2.0 (TID 1). 1161 bytes result sent to driver 18/06/26 10:55:48 INFO TaskSetManager: Finished task 0.0 in stage 2.0 (TID 1) in 29 ms on localhost (1/1) 18/06/26 10:55:48 INFO TaskSchedulerImpl: Removed TaskSet 2.0, whose tasks have all completed, from pool 18/06/26 10:55:48 INFO DAGScheduler: ResultStage 2 (print at SparkWordCount.scala:55) finished in 0.031 s 18/06/26 10:55:48 INFO DAGScheduler: Job 1 finished: print at SparkWordCount.scala:55, took 0.045668 s 18/06/26 10:55:48 INFO SparkContext: Starting job: print at SparkWordCount.scala:55 18/06/26 10:55:48 INFO MapOutputTrackerMaster: Size of output statuses for shuffle 0 is 82 bytes 18/06/26 10:55:48 INFO DAGScheduler: Got job 2 (print at SparkWordCount.scala:55) with 1 output partitions 18/06/26 10:55:48 INFO DAGScheduler: Final stage: ResultStage 4 (print at SparkWordCount.scala:55) 18/06/26 10:55:48 INFO DAGScheduler: Parents of final stage: List(ShuffleMapStage 3) 18/06/26 10:55:48 INFO DAGScheduler: Missing parents: List() 18/06/26 10:55:48 INFO DAGScheduler: Submitting ResultStage 4 (ShuffledRDD[4] at reduceByKey at SparkWordCount.scala:52), which has no missing parents 18/06/26 10:55:48 INFO MemoryStore: Block broadcast_2 stored as values in memory (estimated size 2.6 KB, free 51.9 KB) 18/06/26 10:55:48 INFO MemoryStore: Block broadcast_2_piece0 stored as bytes in memory (estimated size 1641.0 B, free 53.5 KB) 18/06/26 10:55:48 INFO BlockManagerInfo: Added broadcast_2_piece0 in memory on localhost:55667 (size: 1641.0 B, free: 2.4 GB) 18/06/26 10:55:48 INFO SparkContext: Created broadcast 2 from broadcast at DAGScheduler.scala:1015 18/06/26 10:55:48 INFO DAGScheduler: Submitting 1 missing tasks from ResultStage 4 (ShuffledRDD[4] at reduceByKey at SparkWordCount.scala:52) 18/06/26 10:55:48 INFO TaskSchedulerImpl: Adding task set 4.0 with 1 tasks 18/06/26 10:55:48 INFO TaskSetManager: Starting task 0.0 in stage 4.0 (TID 2, localhost, partition 1,PROCESS_LOCAL, 1894 bytes) 18/06/26 10:55:48 INFO Executor: Running task 0.0 in stage 4.0 (TID 2) 18/06/26 10:55:48 INFO ShuffleBlockFetcherIterator: Getting 0 non-empty blocks out of 0 blocks 18/06/26 10:55:48 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 18/06/26 10:55:48 INFO Executor: Finished task 0.0 in stage 4.0 (TID 2). 1161 bytes result sent to driver 18/06/26 10:55:48 INFO TaskSetManager: Finished task 0.0 in stage 4.0 (TID 2) in 4 ms on localhost (1/1) 18/06/26 10:55:48 INFO TaskSchedulerImpl: Removed TaskSet 4.0, whose tasks have all completed, from pool 18/06/26 10:55:48 INFO DAGScheduler: ResultStage 4 (print at SparkWordCount.scala:55) finished in 0.005 s 18/06/26 10:55:48 INFO DAGScheduler: Job 2 finished: print at SparkWordCount.scala:55, took 0.013113 s ------------------------------------------- Time: 1529981748000 ms ------------------------------------------- 18/06/26 10:55:48 INFO JobScheduler: Finished job streaming job 1529981748000 ms.0 from job set of time 1529981748000 ms 18/06/26 10:55:48 INFO JobScheduler: Total delay: 0.148 s for time 1529981748000 ms (execution: 0.081 s) 18/06/26 10:55:48 INFO ReceivedBlockTracker: Deleting batches ArrayBuffer() 18/06/26 10:55:48 INFO InputInfoTracker: remove old batch metadata: 18/06/26 10:55:48 WARN ReceiverSupervisorImpl: Restarting receiver with delay 2000 ms: Error connecting to 192.168.56.101:9999 java.net.ConnectException: Connection refused: connect at java.net.DualStackPlainSocketImpl.connect0(Native Method) at java.net.DualStackPlainSocketImpl.socketConnect(DualStackPlainSocketImpl.java:79) at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350) at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206) at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188) at java.net.PlainSocketImpl.connect(PlainSocketImpl.java:172) at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392) at java.net.Socket.connect(Socket.java:589) at java.net.Socket.connect(Socket.java:538) at java.net.Socket.<init>(Socket.java:434) at java.net.Socket.<init>(Socket.java:211) at org.apache.spark.streaming.dstream.SocketReceiver.receive(SocketInputDStream.scala:73) at org.apache.spark.streaming.dstream.SocketReceiver$$anon$2.run(SocketInputDStream.scala:59) 18/06/26 10:55:48 INFO ReceiverSupervisorImpl: Stopping receiver with message: Restarting receiver with delay 2000ms: Error connecting to 192.168.56.101:9999: java.net.ConnectException: Connection refused: connect 18/06/26 10:55:48 INFO ReceiverSupervisorImpl: Called receiver onStop 18/06/26 10:55:48 INFO ReceiverSupervisorImpl: Deregistering receiver 0 18/06/26 10:55:48 ERROR ReceiverTracker: Deregistered receiver for stream 0: Restarting receiver with delay 2000ms: Error connecting to 192.168.56.101:9999 - java.net.ConnectException: Connection refused: connect at java.net.DualStackPlainSocketImpl.connect0(Native Method) at java.net.DualStackPlainSocketImpl.socketConnect(DualStackPlainSocketImpl.java:79) at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350) at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206) at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188) at java.net.PlainSocketImpl.connect(PlainSocketImpl.java:172) at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392) at java.net.Socket.connect(Socket.java:589) at java.net.Socket.connect(Socket.java:538) at java.net.Socket.<init>(Socket.java:434) at java.net.Socket.<init>(Socket.java:211) at org.apache.spark.streaming.dstream.SocketReceiver.receive(SocketInputDStream.scala:73) at org.apache.spark.streaming.dstream.SocketReceiver$$anon$2.run(SocketInputDStream.scala:59) 18/06/26 10:55:48 INFO ReceiverSupervisorImpl: Stopped receiver 0 18/06/26 10:55:49 INFO JobScheduler: Added jobs for time 1529981749000 ms 18/06/26 10:55:49 INFO JobScheduler: Starting job streaming job 1529981749000 ms.0 from job set of time 1529981749000 ms 18/06/26 10:55:49 INFO SparkContext: Starting job: print at SparkWordCount.scala:55 18/06/26 10:55:49 INFO DAGScheduler: Registering RDD 7 (map at SparkWordCount.scala:51) 18/06/26 10:55:49 INFO DAGScheduler: Got job 3 (print at SparkWordCount.scala:55) with 1 output partitions 18/06/26 10:55:49 INFO DAGScheduler: Final stage: ResultStage 6 (print at SparkWordCount.scala:55) 18/06/26 10:55:49 INFO DAGScheduler: Parents of final stage: List(ShuffleMapStage 5) 18/06/26 10:55:49 INFO DAGScheduler: Missing parents: List() 18/06/26 10:55:49 INFO DAGScheduler: Submitting ResultStage 6 (ShuffledRDD[8] at reduceByKey at SparkWordCount.scala:52), which has no missing parents 18/06/26 10:55:49 INFO MemoryStore: Block broadcast_3 stored as values in memory (estimated size 2.6 KB, free 56.1 KB) 18/06/26 10:55:49 INFO MemoryStore: Block broadcast_3_piece0 stored as bytes in memory (estimated size 1632.0 B, free 56.1 KB) 18/06/26 10:55:49 INFO BlockManagerInfo: Removed broadcast_2_piece0 on localhost:55667 in memory (size: 1641.0 B, free: 2.4 GB) 18/06/26 10:55:49 INFO BlockManagerInfo: Added broadcast_3_piece0 in memory on localhost:55667 (size: 1632.0 B, free: 2.4 GB) 18/06/26 10:55:49 INFO SparkContext: Created broadcast 3 from broadcast at DAGScheduler.scala:1015 18/06/26 10:55:49 INFO DAGScheduler: Submitting 1 missing tasks from ResultStage 6 (ShuffledRDD[8] at reduceByKey at SparkWordCount.scala:52) 18/06/26 10:55:49 INFO TaskSchedulerImpl: Adding task set 6.0 with 1 tasks 18/06/26 10:55:49 INFO TaskSetManager: Starting task 0.0 in stage 6.0 (TID 3, localhost, partition 0,PROCESS_LOCAL, 1894 bytes) 18/06/26 10:55:49 INFO Executor: Running task 0.0 in stage 6.0 (TID 3) 18/06/26 10:55:49 INFO ContextCleaner: Cleaned accumulator 3 18/06/26 10:55:49 INFO ShuffleBlockFetcherIterator: Getting 0 non-empty blocks out of 0 blocks 18/06/26 10:55:49 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 1 ms 18/06/26 10:55:49 INFO BlockManagerInfo: Removed broadcast_1_piece0 on localhost:55667 in memory (size: 1641.0 B, free: 2.4 GB) 18/06/26 10:55:49 INFO ContextCleaner: Cleaned accumulator 2 18/06/26 10:55:49 INFO Executor: Finished task 0.0 in stage 6.0 (TID 3). 1161 bytes result sent to driver 18/06/26 10:55:49 INFO TaskSetManager: Finished task 0.0 in stage 6.0 (TID 3) in 5 ms on localhost (1/1) 18/06/26 10:55:49 INFO TaskSchedulerImpl: Removed TaskSet 6.0, whose tasks have all completed, from pool 18/06/26 10:55:49 INFO DAGScheduler: ResultStage 6 (print at SparkWordCount.scala:55) finished in 0.006 s 18/06/26 10:55:49 INFO DAGScheduler: Job 3 finished: print at SparkWordCount.scala:55, took 0.018014 s 18/06/26 10:55:49 INFO SparkContext: Starting job: print at SparkWordCount.scala:55 18/06/26 10:55:49 INFO MapOutputTrackerMaster: Size of output statuses for shuffle 1 is 82 bytes 18/06/26 10:55:49 INFO DAGScheduler: Got job 4 (print at SparkWordCount.scala:55) with 1 output partitions 18/06/26 10:55:49 INFO DAGScheduler: Final stage: ResultStage 8 (print at SparkWordCount.scala:55) 18/06/26 10:55:49 INFO DAGScheduler: Parents of final stage: List(ShuffleMapStage 7) 18/06/26 10:55:49 INFO DAGScheduler: Missing parents: List() 18/06/26 10:55:49 INFO DAGScheduler: Submitting ResultStage 8 (ShuffledRDD[8] at reduceByKey at SparkWordCount.scala:52), which has no missing parents 18/06/26 10:55:49 INFO MemoryStore: Block broadcast_4 stored as values in memory (estimated size 2.6 KB, free 51.8 KB) 18/06/26 10:55:49 INFO MemoryStore: Block broadcast_4_piece0 stored as bytes in memory (estimated size 1632.0 B, free 53.4 KB) 18/06/26 10:55:49 INFO BlockManagerInfo: Added broadcast_4_piece0 in memory on localhost:55667 (size: 1632.0 B, free: 2.4 GB) 18/06/26 10:55:49 INFO SparkContext: Created broadcast 4 from broadcast at DAGScheduler.scala:1015 18/06/26 10:55:49 INFO DAGScheduler: Submitting 1 missing tasks from ResultStage 8 (ShuffledRDD[8] at reduceByKey at SparkWordCount.scala:52) 18/06/26 10:55:49 INFO TaskSchedulerImpl: Adding task set 8.0 with 1 tasks 18/06/26 10:55:49 INFO TaskSetManager: Starting task 0.0 in stage 8.0 (TID 4, localhost, partition 1,PROCESS_LOCAL, 1894 bytes) 18/06/26 10:55:49 INFO Executor: Running task 0.0 in stage 8.0 (TID 4) 18/06/26 10:55:49 INFO ShuffleBlockFetcherIterator: Getting 0 non-empty blocks out of 0 blocks 18/06/26 10:55:49 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 18/06/26 10:55:49 INFO Executor: Finished task 0.0 in stage 8.0 (TID 4). 1161 bytes result sent to driver 18/06/26 10:55:49 INFO DAGScheduler: ResultStage 8 (print at SparkWordCount.scala:55) finished in 0.004 s 18/06/26 10:55:49 INFO TaskSetManager: Finished task 0.0 in stage 8.0 (TID 4) in 4 ms on localhost (1/1) 18/06/26 10:55:49 INFO DAGScheduler: Job 4 finished: print at SparkWordCount.scala:55, took 0.009080 s 18/06/26 10:55:49 INFO TaskSchedulerImpl: Removed TaskSet 8.0, whose tasks have all completed, from pool ------------------------------------------- Time: 1529981749000 ms ------------------------------------------- 18/06/26 10:55:49 INFO JobScheduler: Finished job streaming job 1529981749000 ms.0 from job set of time 1529981749000 ms 18/06/26 10:55:49 INFO JobScheduler: Total delay: 0.076 s for time 1529981749000 ms (execution: 0.053 s) 18/06/26 10:55:49 INFO ShuffledRDD: Removing RDD 4 from persistence list 18/06/26 10:55:49 INFO BlockManager: Removing RDD 4 18/06/26 10:55:49 INFO MapPartitionsRDD: Removing RDD 3 from persistence list 18/06/26 10:55:49 INFO BlockManager: Removing RDD 3 18/06/26 10:55:49 INFO MapPartitionsRDD: Removing RDD 2 from persistence list 18/06/26 10:55:49 INFO BlockManager: Removing RDD 2 18/06/26 10:55:49 INFO BlockRDD: Removing RDD 1 from persistence list 18/06/26 10:55:49 INFO BlockManager: Removing RDD 1 18/06/26 10:55:49 INFO SocketInputDStream: Removing blocks of RDD BlockRDD[1] at socketTextStream at SparkWordCount.scala:46 of time 1529981749000 ms 18/06/26 10:55:49 INFO ReceivedBlockTracker: Deleting batches ArrayBuffer() 18/06/26 10:55:49 INFO InputInfoTracker: remove old batch metadata: 18/06/26 10:55:50 INFO JobScheduler: Added jobs for time 1529981750000 ms 18/06/26 10:55:50 INFO JobScheduler: Starting job streaming job 1529981750000 ms.0 from job set of time 1529981750000 ms 18/06/26 10:55:50 INFO SparkContext: Starting job: print at SparkWordCount.scala:55 18/06/26 10:55:50 INFO DAGScheduler: Registering RDD 11 (map at SparkWordCount.scala:51) 18/06/26 10:55:50 INFO DAGScheduler: Got job 5 (print at SparkWordCount.scala:55) with 1 output partitions 18/06/26 10:55:50 INFO DAGScheduler: Final stage: ResultStage 10 (print at SparkWordCount.scala:55) 18/06/26 10:55:50 INFO DAGScheduler: Parents of final stage: List(ShuffleMapStage 9) 18/06/26 10:55:50 INFO DAGScheduler: Missing parents: List() 18/06/26 10:55:50 INFO DAGScheduler: Submitting ResultStage 10 (ShuffledRDD[12] at reduceByKey at SparkWordCount.scala:52), which has no missing parents 18/06/26 10:55:50 INFO MemoryStore: Block broadcast_5 stored as values in memory (estimated size 2.6 KB, free 56.0 KB) 18/06/26 10:55:50 INFO MemoryStore: Block broadcast_5_piece0 stored as bytes in memory (estimated size 1634.0 B, free 57.6 KB) 18/06/26 10:55:50 INFO BlockManagerInfo: Added broadcast_5_piece0 in memory on localhost:55667 (size: 1634.0 B, free: 2.4 GB) 18/06/26 10:55:50 INFO SparkContext: Created broadcast 5 from broadcast at DAGScheduler.scala:1015 18/06/26 10:55:50 INFO DAGScheduler: Submitting 1 missing tasks from ResultStage 10 (ShuffledRDD[12] at reduceByKey at SparkWordCount.scala:52) 18/06/26 10:55:50 INFO TaskSchedulerImpl: Adding task set 10.0 with 1 tasks 18/06/26 10:55:50 INFO TaskSetManager: Starting task 0.0 in stage 10.0 (TID 5, localhost, partition 0,PROCESS_LOCAL, 1894 bytes) 18/06/26 10:55:50 INFO Executor: Running task 0.0 in stage 10.0 (TID 5) 18/06/26 10:55:50 INFO ShuffleBlockFetcherIterator: Getting 0 non-empty blocks out of 0 blocks 18/06/26 10:55:50 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 18/06/26 10:55:50 INFO Executor: Finished task 0.0 in stage 10.0 (TID 5). 1161 bytes result sent to driver 18/06/26 10:55:50 INFO TaskSetManager: Finished task 0.0 in stage 10.0 (TID 5) in 6 ms on localhost (1/1) 18/06/26 10:55:50 INFO DAGScheduler: ResultStage 10 (print at SparkWordCount.scala:55) finished in 0.008 s 18/06/26 10:55:50 INFO TaskSchedulerImpl: Removed TaskSet 10.0, whose tasks have all completed, from pool 18/06/26 10:55:50 INFO DAGScheduler: Job 5 finished: print at SparkWordCount.scala:55, took 0.017396 s 18/06/26 10:55:50 INFO SparkContext: Starting job: print at SparkWordCount.scala:55 18/06/26 10:55:50 INFO MapOutputTrackerMaster: Size of output statuses for shuffle 2 is 82 bytes 18/06/26 10:55:50 INFO DAGScheduler: Got job 6 (print at SparkWordCount.scala:55) with 1 output partitions 18/06/26 10:55:50 INFO DAGScheduler: Final stage: ResultStage 12 (print at SparkWordCount.scala:55) 18/06/26 10:55:50 INFO DAGScheduler: Parents of final stage: List(ShuffleMapStage 11) 18/06/26 10:55:50 INFO DAGScheduler: Missing parents: List() 18/06/26 10:55:50 INFO DAGScheduler: Submitting ResultStage 12 (ShuffledRDD[12] at reduceByKey at SparkWordCount.scala:52), which has no missing parents 18/06/26 10:55:50 INFO MemoryStore: Block broadcast_6 stored as values in memory (estimated size 2.6 KB, free 60.2 KB) 18/06/26 10:55:50 INFO MemoryStore: Block broadcast_6_piece0 stored as bytes in memory (estimated size 1634.0 B, free 61.8 KB) 18/06/26 10:55:50 INFO BlockManagerInfo: Added broadcast_6_piece0 in memory on localhost:55667 (size: 1634.0 B, free: 2.4 GB) 18/06/26 10:55:50 INFO SparkContext: Created broadcast 6 from broadcast at DAGScheduler.scala:1015 18/06/26 10:55:50 INFO DAGScheduler: Submitting 1 missing tasks from ResultStage 12 (ShuffledRDD[12] at reduceByKey at SparkWordCount.scala:52) 18/06/26 10:55:50 INFO TaskSchedulerImpl: Adding task set 12.0 with 1 tasks 18/06/26 10:55:50 INFO TaskSetManager: Starting task 0.0 in stage 12.0 (TID 6, localhost, partition 1,PROCESS_LOCAL, 1894 bytes) 18/06/26 10:55:50 INFO Executor: Running task 0.0 in stage 12.0 (TID 6) 18/06/26 10:55:50 INFO ShuffleBlockFetcherIterator: Getting 0 non-empty blocks out of 0 blocks 18/06/26 10:55:50 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 18/06/26 10:55:50 INFO Executor: Finished task 0.0 in stage 12.0 (TID 6). 1161 bytes result sent to driver 18/06/26 10:55:50 INFO TaskSetManager: Finished task 0.0 in stage 12.0 (TID 6) in 4 ms on localhost (1/1) ------------------------------------------- Time: 1529981750000 ms ------------------------------------------- 18/06/26 10:55:50 INFO TaskSchedulerImpl: Removed TaskSet 12.0, whose tasks have all completed, from pool 18/06/26 10:55:50 INFO DAGScheduler: ResultStage 12 (print at SparkWordCount.scala:55) finished in 0.005 s 18/06/26 10:55:50 INFO DAGScheduler: Job 6 finished: print at SparkWordCount.scala:55, took 0.009884 s 18/06/26 10:55:50 INFO JobScheduler: Finished job streaming job 1529981750000 ms.0 from job set of time 1529981750000 ms 18/06/26 10:55:50 INFO JobScheduler: Total delay: 0.069 s for time 1529981750000 ms (execution: 0.041 s) 18/06/26 10:55:50 INFO ShuffledRDD: Removing RDD 8 from persistence list 18/06/26 10:55:50 INFO BlockManager: Removing RDD 8 18/06/26 10:55:50 INFO MapPartitionsRDD: Removing RDD 7 from persistence list 18/06/26 10:55:50 INFO BlockManager: Removing RDD 7 18/06/26 10:55:50 INFO MapPartitionsRDD: Removing RDD 6 from persistence list 18/06/26 10:55:50 INFO BlockManager: Removing RDD 6 18/06/26 10:55:50 INFO BlockRDD: Removing RDD 5 from persistence list 18/06/26 10:55:50 INFO BlockManager: Removing RDD 5 18/06/26 10:55:50 INFO SocketInputDStream: Removing blocks of RDD BlockRDD[5] at socketTextStream at SparkWordCount.scala:46 of time 1529981750000 ms 18/06/26 10:55:50 INFO ReceivedBlockTracker: Deleting batches ArrayBuffer(1529981748000 ms) 18/06/26 10:55:50 INFO InputInfoTracker: remove old batch metadata: 1529981748000 ms 18/06/26 10:55:50 INFO ReceiverSupervisorImpl: Starting receiver again 18/06/26 10:55:50 INFO ReceiverTracker: Registered receiver for stream 0 from 10.41.88.38:55640 18/06/26 10:55:50 INFO ReceiverSupervisorImpl: Starting receiver 18/06/26 10:55:50 INFO ReceiverSupervisorImpl: Called receiver onStart 18/06/26 10:55:50 INFO SocketReceiver: Connecting to 192.168.56.101:9999 18/06/26 10:55:50 INFO ReceiverSupervisorImpl: Receiver started again 18/06/26 10:55:51 INFO JobScheduler: Added jobs for time 1529981751000 ms 18/06/26 10:55:51 INFO JobScheduler: Starting job streaming job 1529981751000 ms.0 from job set of time 1529981751000 ms 18/06/26 10:55:51 INFO SparkContext: Starting job: print at SparkWordCount.scala:55 18/06/26 10:55:51 INFO DAGScheduler: Registering RDD 15 (map at SparkWordCount.scala:51) 18/06/26 10:55:51 INFO DAGScheduler: Got job 7 (print at SparkWordCount.scala:55) with 1 output partitions 18/06/26 10:55:51 INFO DAGScheduler: Final stage: ResultStage 14 (print at SparkWordCount.scala:55) 18/06/26 10:55:51 INFO DAGScheduler: Parents of final stage: List(ShuffleMapStage 13) 18/06/26 10:55:51 INFO DAGScheduler: Missing parents: List() 18/06/26 10:55:51 INFO DAGScheduler: Submitting ResultStage 14 (ShuffledRDD[16] at reduceByKey at SparkWordCount.scala:52), which has no missing parents 18/06/26 10:55:51 INFO MemoryStore: Block broadcast_7 stored as values in memory (estimated size 2.6 KB, free 64.4 KB) 18/06/26 10:55:51 INFO MemoryStore: Block broadcast_7_piece0 stored as bytes in memory (estimated size 1638.0 B, free 66.0 KB) 18/06/26 10:55:51 INFO BlockManagerInfo: Added broadcast_7_piece0 in memory on localhost:55667 (size: 1638.0 B, free: 2.4 GB) 18/06/26 10:55:51 INFO SparkContext: Created broadcast 7 from broadcast at DAGScheduler.scala:1015 18/06/26 10:55:51 INFO DAGScheduler: Submitting 1 missing tasks from ResultStage 14 (ShuffledRDD[16] at reduceByKey at SparkWordCount.scala:52) 18/06/26 10:55:51 INFO TaskSchedulerImpl: Adding task set 14.0 with 1 tasks 18/06/26 10:55:51 INFO TaskSetManager: Starting task 0.0 in stage 14.0 (TID 7, localhost, partition 0,PROCESS_LOCAL, 1894 bytes) 18/06/26 10:55:51 INFO Executor: Running task 0.0 in stage 14.0 (TID 7) 18/06/26 10:55:51 INFO ShuffleBlockFetcherIterator: Getting 0 non-empty blocks out of 0 blocks 18/06/26 10:55:51 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 1 ms 18/06/26 10:55:51 INFO Executor: Finished task 0.0 in stage 14.0 (TID 7). 1161 bytes result sent to driver 18/06/26 10:55:51 INFO TaskSetManager: Finished task 0.0 in stage 14.0 (TID 7) in 3 ms on localhost (1/1) 18/06/26 10:55:51 INFO TaskSchedulerImpl: Removed TaskSet 14.0, whose tasks have all completed, from pool 18/06/26 10:55:51 INFO DAGScheduler: ResultStage 14 (print at SparkWordCount.scala:55) finished in 0.003 s 18/06/26 10:55:51 INFO DAGScheduler: Job 7 finished: print at SparkWordCount.scala:55, took 0.009481 s 18/06/26 10:55:51 INFO SparkContext: Starting job: print at SparkWordCount.scala:55 18/06/26 10:55:51 INFO MapOutputTrackerMaster: Size of output statuses for shuffle 3 is 82 bytes 18/06/26 10:55:51 INFO DAGScheduler: Got job 8 (print at SparkWordCount.scala:55) with 1 output partitions 18/06/26 10:55:51 INFO DAGScheduler: Final stage: ResultStage 16 (print at SparkWordCount.scala:55) 18/06/26 10:55:51 INFO DAGScheduler: Parents of final stage: List(ShuffleMapStage 15) 18/06/26 10:55:51 INFO DAGScheduler: Missing parents: List() 18/06/26 10:55:51 INFO DAGScheduler: Submitting ResultStage 16 (ShuffledRDD[16] at reduceByKey at SparkWordCount.scala:52), which has no missing parents 18/06/26 10:55:51 INFO MemoryStore: Block broadcast_8 stored as values in memory (estimated size 2.6 KB, free 68.6 KB) 18/06/26 10:55:51 INFO MemoryStore: Block broadcast_8_piece0 stored as bytes in memory (estimated size 1638.0 B, free 70.2 KB) 18/06/26 10:55:51 INFO BlockManagerInfo: Added broadcast_8_piece0 in memory on localhost:55667 (size: 1638.0 B, free: 2.4 GB) 18/06/26 10:55:51 INFO SparkContext: Created broadcast 8 from broadcast at DAGScheduler.scala:1015 18/06/26 10:55:51 INFO DAGScheduler: Submitting 1 missing tasks from ResultStage 16 (ShuffledRDD[16] at reduceByKey at SparkWordCount.scala:52) 18/06/26 10:55:51 INFO TaskSchedulerImpl: Adding task set 16.0 with 1 tasks 18/06/26 10:55:51 INFO TaskSetManager: Starting task 0.0 in stage 16.0 (TID 8, localhost, partition 1,PROCESS_LOCAL, 1894 bytes) 18/06/26 10:55:51 INFO Executor: Running task 0.0 in stage 16.0 (TID 8) 18/06/26 10:55:51 INFO ShuffleBlockFetcherIterator: Getting 0 non-empty blocks out of 0 blocks 18/06/26 10:55:51 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 18/06/26 10:55:51 INFO Executor: Finished task 0.0 in stage 16.0 (TID 8). 1161 bytes result sent to driver 18/06/26 10:55:51 INFO TaskSetManager: Finished task 0.0 in stage 16.0 (TID 8) in 3 ms on localhost (1/1) 18/06/26 10:55:51 INFO TaskSchedulerImpl: Removed TaskSet 16.0, whose tasks have all completed, from pool 18/06/26 10:55:51 INFO DAGScheduler: ResultStage 16 (print at SparkWordCount.scala:55) finished in 0.004 s 18/06/26 10:55:51 INFO DAGScheduler: Job 8 finished: print at SparkWordCount.scala:55, took 0.011080 s ------------------------------------------- Time: 1529981751000 ms ------------------------------------------- 18/06/26 10:55:51 INFO JobScheduler: Finished job streaming job 1529981751000 ms.0 from job set of time 1529981751000 ms 18/06/26 10:55:51 INFO JobScheduler: Total delay: 0.042 s for time 1529981751000 ms (execution: 0.026 s) 18/06/26 10:55:51 INFO ShuffledRDD: Removing RDD 12 from persistence list 18/06/26 10:55:51 INFO BlockManager: Removing RDD 12 18/06/26 10:55:51 INFO MapPartitionsRDD: Removing RDD 11 from persistence list 18/06/26 10:55:51 INFO BlockManager: Removing RDD 11 18/06/26 10:55:51 INFO MapPartitionsRDD: Removing RDD 10 from persistence list 18/06/26 10:55:51 INFO BlockManager: Removing RDD 10 18/06/26 10:55:51 INFO BlockRDD: Removing RDD 9 from persistence list 18/06/26 10:55:51 INFO BlockManager: Removing RDD 9 18/06/26 10:55:51 INFO SocketInputDStream: Removing blocks of RDD BlockRDD[9] at socketTextStream at SparkWordCount.scala:46 of time 1529981751000 ms 18/06/26 10:55:51 INFO ReceivedBlockTracker: Deleting batches ArrayBuffer(1529981749000 ms) 18/06/26 10:55:51 INFO InputInfoTracker: remove old batch metadata: 1529981749000 ms 18/06/26 10:55:51 WARN ReceiverSupervisorImpl: Restarting receiver with delay 2000 ms: Error connecting to 192.168.56.101:9999 java.net.ConnectException: Connection refused: connect at java.net.DualStackPlainSocketImpl.connect0(Native Method) at java.net.DualStackPlainSocketImpl.socketConnect(DualStackPlainSocketImpl.java:79) at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350) at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206) at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188) at java.net.PlainSocketImpl.connect(PlainSocketImpl.java:172) at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392) at java.net.Socket.connect(Socket.java:589) at java.net.Socket.connect(Socket.java:538) at java.net.Socket.<init>(Socket.java:434) at java.net.Socket.<init>(Socket.java:211) at org.apache.spark.streaming.dstream.SocketReceiver.receive(SocketInputDStream.scala:73) at org.apache.spark.streaming.dstream.SocketReceiver$$anon$2.run(SocketInputDStream.scala:59) 18/06/26 10:55:51 INFO ReceiverSupervisorImpl: Stopping receiver with message: Restarting receiver with delay 2000ms: Error connecting to 192.168.56.101:9999: java.net.ConnectException: Connection refused: connect 18/06/26 10:55:51 INFO ReceiverSupervisorImpl: Called receiver onStop 18/06/26 10:55:51 INFO ReceiverSupervisorImpl: Deregistering receiver 0 18/06/26 10:55:51 ERROR ReceiverTracker: Deregistered receiver for stream 0: Restarting receiver with delay 2000ms: Error connecting to 192.168.56.101:9999 - java.net.ConnectException: Connection refused: connect at java.net.DualStackPlainSocketImpl.connect0(Native Method) at java.net.DualStackPlainSocketImpl.socketConnect(DualStackPlainSocketImpl.java:79) at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350) at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206) at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188) at java.net.PlainSocketImpl.connect(PlainSocketImpl.java:172) at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392) at java.net.Socket.connect(Socket.java:589) at java.net.Socket.connect(Socket.java:538) at java.net.Socket.<init>(Socket.java:434) at java.net.Socket.<init>(Socket.java:211) at org.apache.spark.streaming.dstream.SocketReceiver.receive(SocketInputDStream.scala:73) at org.apache.spark.streaming.dstream.SocketReceiver$$anon$2.run(SocketInputDStream.scala:59) 18/06/26 10:55:51 INFO ReceiverSupervisorImpl: Stopped receiver 0 18/06/26 10:55:52 INFO JobScheduler: Added jobs for time 1529981752000 ms 18/06/26 10:55:52 INFO JobScheduler: Starting job streaming job 1529981752000 ms.0 from job set of time 1529981752000 ms 18/06/26 10:55:52 INFO SparkContext: Starting job: print at SparkWordCount.scala:55 18/06/26 10:55:52 INFO DAGScheduler: Registering RDD 19 (map at SparkWordCount.scala:51) 18/06/26 10:55:52 INFO DAGScheduler: Got job 9 (print at SparkWordCount.scala:55) with 1 output partitions 18/06/26 10:55:52 INFO DAGScheduler: Final stage: ResultStage 18 (print at SparkWordCount.scala:55) 18/06/26 10:55:52 INFO DAGScheduler: Parents of final stage: List(ShuffleMapStage 17) 18/06/26 10:55:52 INFO DAGScheduler: Missing parents: List() 18/06/26 10:55:52 INFO DAGScheduler: Submitting ResultStage 18 (ShuffledRDD[20] at reduceByKey at SparkWordCount.scala:52), which has no missing parents 18/06/26 10:55:52 INFO MemoryStore: Block broadcast_9 stored as values in memory (estimated size 2.6 KB, free 72.8 KB) 18/06/26 10:55:52 INFO MemoryStore: Block broadcast_9_piece0 stored as bytes in memory (estimated size 1637.0 B, free 74.4 KB) 18/06/26 10:55:52 INFO BlockManagerInfo: Added broadcast_9_piece0 in memory on localhost:55667 (size: 1637.0 B, free: 2.4 GB) 18/06/26 10:55:52 INFO SparkContext: Created broadcast 9 from broadcast at DAGScheduler.scala:1015 18/06/26 10:55:52 INFO DAGScheduler: Submitting 1 missing tasks from ResultStage 18 (ShuffledRDD[20] at reduceByKey at SparkWordCount.scala:52) 18/06/26 10:55:52 INFO TaskSchedulerImpl: Adding task set 18.0 with 1 tasks 18/06/26 10:55:52 INFO TaskSetManager: Starting task 0.0 in stage 18.0 (TID 9, localhost, partition 0,PROCESS_LOCAL, 1894 bytes) 18/06/26 10:55:52 INFO Executor: Running task 0.0 in stage 18.0 (TID 9) 18/06/26 10:55:52 INFO ShuffleBlockFetcherIterator: Getting 0 non-empty blocks out of 0 blocks 18/06/26 10:55:52 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 18/06/26 10:55:52 INFO Executor: Finished task 0.0 in stage 18.0 (TID 9). 1161 bytes result sent to driver 18/06/26 10:55:52 INFO TaskSetManager: Finished task 0.0 in stage 18.0 (TID 9) in 10 ms on localhost (1/1) 18/06/26 10:55:52 INFO TaskSchedulerImpl: Removed TaskSet 18.0, whose tasks have all completed, from pool 18/06/26 10:55:52 INFO DAGScheduler: ResultStage 18 (print at SparkWordCount.scala:55) finished in 0.011 s 18/06/26 10:55:52 INFO DAGScheduler: Job 9 finished: print at SparkWordCount.scala:55, took 0.028769 s 18/06/26 10:55:52 INFO SparkContext: Starting job: print at SparkWordCount.scala:55 18/06/26 10:55:52 INFO MapOutputTrackerMaster: Size of output statuses for shuffle 4 is 82 bytes 18/06/26 10:55:52 INFO DAGScheduler: Got job 10 (print at SparkWordCount.scala:55) with 1 output partitions 18/06/26 10:55:52 INFO DAGScheduler: Final stage: ResultStage 20 (print at SparkWordCount.scala:55) 18/06/26 10:55:52 INFO DAGScheduler: Parents of final stage: List(ShuffleMapStage 19) 18/06/26 10:55:52 INFO DAGScheduler: Missing parents: List() 18/06/26 10:55:52 INFO DAGScheduler: Submitting ResultStage 20 (ShuffledRDD[20] at reduceByKey at SparkWordCount.scala:52), which has no missing parents 18/06/26 10:55:52 INFO MemoryStore: Block broadcast_10 stored as values in memory (estimated size 2.6 KB, free 77.0 KB) 18/06/26 10:55:52 INFO MemoryStore: Block broadcast_10_piece0 stored as bytes in memory (estimated size 1637.0 B, free 78.6 KB) 18/06/26 10:55:52 INFO BlockManagerInfo: Added broadcast_10_piece0 in memory on localhost:55667 (size: 1637.0 B, free: 2.4 GB) 18/06/26 10:55:52 INFO SparkContext: Created broadcast 10 from broadcast at DAGScheduler.scala:1015 18/06/26 10:55:52 INFO DAGScheduler: Submitting 1 missing tasks from ResultStage 20 (ShuffledRDD[20] at reduceByKey at SparkWordCount.scala:52) 18/06/26 10:55:52 INFO TaskSchedulerImpl: Adding task set 20.0 with 1 tasks 18/06/26 10:55:52 INFO TaskSetManager: Starting task 0.0 in stage 20.0 (TID 10, localhost, partition 1,PROCESS_LOCAL, 1894 bytes) 18/06/26 10:55:52 INFO Executor: Running task 0.0 in stage 20.0 (TID 10) 18/06/26 10:55:52 INFO ShuffleBlockFetcherIterator: Getting 0 non-empty blocks out of 0 blocks 18/06/26 10:55:52 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 18/06/26 10:55:52 INFO Executor: Finished task 0.0 in stage 20.0 (TID 10). 1161 bytes result sent to driver 18/06/26 10:55:52 INFO TaskSetManager: Finished task 0.0 in stage 20.0 (TID 10) in 6 ms on localhost (1/1) 18/06/26 10:55:52 INFO TaskSchedulerImpl: Removed TaskSet 20.0, whose tasks have all completed, from pool 18/06/26 10:55:52 INFO DAGScheduler: ResultStage 20 (print at SparkWordCount.scala:55) finished in 0.006 s 18/06/26 10:55:52 INFO DAGScheduler: Job 10 finished: print at SparkWordCount.scala:55, took 0.015599 s 18/06/26 10:55:52 INFO JobScheduler: Finished job streaming job 1529981752000 ms.0 from job set of time 1529981752000 ms 18/06/26 10:55:52 INFO JobScheduler: Total delay: 0.091 s for time 1529981752000 ms (execution: 0.064 s) 18/06/26 10:55:52 INFO ShuffledRDD: Removing RDD 16 from persistence list 18/06/26 10:55:52 INFO MapPartitionsRDD: Removing RDD 15 from persistence list 18/06/26 10:55:52 INFO BlockManager: Removing RDD 16 18/06/26 10:55:52 INFO MapPartitionsRDD: Removing RDD 14 from persistence list 18/06/26 10:55:52 INFO BlockManager: Removing RDD 15 18/06/26 10:55:52 INFO BlockManager: Removing RDD 14 18/06/26 10:55:52 INFO BlockRDD: Removing RDD 13 from persistence list ------------------------------------------- Time: 1529981752000 ms ------------------------------------------- 18/06/26 10:55:52 INFO BlockManager: Removing RDD 13 18/06/26 10:55:52 INFO SocketInputDStream: Removing blocks of RDD BlockRDD[13] at socketTextStream at SparkWordCount.scala:46 of time 1529981752000 ms 18/06/26 10:55:52 INFO ReceivedBlockTracker: Deleting batches ArrayBuffer(1529981750000 ms) 18/06/26 10:55:52 INFO InputInfoTracker: remove old batch metadata: 1529981750000 ms 18/06/26 10:55:53 INFO JobScheduler: Added jobs for time 1529981753000 ms 18/06/26 10:55:53 INFO JobScheduler: Starting job streaming job 1529981753000 ms.0 from job set of time 1529981753000 ms 18/06/26 10:55:53 INFO SparkContext: Starting job: print at SparkWordCount.scala:55 18/06/26 10:55:53 INFO DAGScheduler: Registering RDD 23 (map at SparkWordCount.scala:51) 18/06/26 10:55:53 INFO DAGScheduler: Got job 11 (print at SparkWordCount.scala:55) with 1 output partitions 18/06/26 10:55:53 INFO DAGScheduler: Final stage: ResultStage 22 (print at SparkWordCount.scala:55) 18/06/26 10:55:53 INFO DAGScheduler: Parents of final stage: List(ShuffleMapStage 21) 18/06/26 10:55:53 INFO DAGScheduler: Missing parents: List() 18/06/26 10:55:53 INFO DAGScheduler: Submitting ResultStage 22 (ShuffledRDD[24] at reduceByKey at SparkWordCount.scala:52), which has no missing parents 18/06/26 10:55:53 INFO MemoryStore: Block broadcast_11 stored as values in memory (estimated size 2.6 KB, free 81.2 KB) 18/06/26 10:55:53 INFO MemoryStore: Block broadcast_11_piece0 stored as bytes in memory (estimated size 1636.0 B, free 82.8 KB) 18/06/26 10:55:53 INFO BlockManagerInfo: Added broadcast_11_piece0 in memory on localhost:55667 (size: 1636.0 B, free: 2.4 GB) 18/06/26 10:55:53 INFO SparkContext: Created broadcast 11 from broadcast at DAGScheduler.scala:1015 18/06/26 10:55:53 INFO DAGScheduler: Submitting 1 missing tasks from ResultStage 22 (ShuffledRDD[24] at reduceByKey at SparkWordCount.scala:52) 18/06/26 10:55:53 INFO TaskSchedulerImpl: Adding task set 22.0 with 1 tasks 18/06/26 10:55:53 INFO TaskSetManager: Starting task 0.0 in stage 22.0 (TID 11, localhost, partition 0,PROCESS_LOCAL, 1894 bytes) 18/06/26 10:55:53 INFO Executor: Running task 0.0 in stage 22.0 (TID 11) 18/06/26 10:55:53 INFO ShuffleBlockFetcherIterator: Getting 0 non-empty blocks out of 0 blocks 18/06/26 10:55:53 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 1 ms 18/06/26 10:55:53 INFO Executor: Finished task 0.0 in stage 22.0 (TID 11). 1161 bytes result sent to driver 18/06/26 10:55:53 INFO TaskSetManager: Finished task 0.0 in stage 22.0 (TID 11) in 7 ms on localhost (1/1) 18/06/26 10:55:53 INFO TaskSchedulerImpl: Removed TaskSet 22.0, whose tasks have all completed, from pool 18/06/26 10:55:53 INFO DAGScheduler: ResultStage 22 (print at SparkWordCount.scala:55) finished in 0.008 s 18/06/26 10:55:53 INFO DAGScheduler: Job 11 finished: print at SparkWordCount.scala:55, took 0.024801 s 18/06/26 10:55:53 INFO SparkContext: Starting job: print at SparkWordCount.scala:55 18/06/26 10:55:53 INFO MapOutputTrackerMaster: Size of output statuses for shuffle 5 is 82 bytes 18/06/26 10:55:53 INFO DAGScheduler: Got job 12 (print at SparkWordCount.scala:55) with 1 output partitions 18/06/26 10:55:53 INFO DAGScheduler: Final stage: ResultStage 24 (print at SparkWordCount.scala:55) 18/06/26 10:55:53 INFO DAGScheduler: Parents of final stage: List(ShuffleMapStage 23) 18/06/26 10:55:53 INFO DAGScheduler: Missing parents: List() 18/06/26 10:55:53 INFO DAGScheduler: Submitting ResultStage 24 (ShuffledRDD[24] at reduceByKey at SparkWordCount.scala:52), which has no missing parents 18/06/26 10:55:53 INFO MemoryStore: Block broadcast_12 stored as values in memory (estimated size 2.6 KB, free 85.4 KB) 18/06/26 10:55:53 INFO MemoryStore: Block broadcast_12_piece0 stored as bytes in memory (estimated size 1636.0 B, free 87.0 KB) 18/06/26 10:55:53 INFO BlockManagerInfo: Added broadcast_12_piece0 in memory on localhost:55667 (size: 1636.0 B, free: 2.4 GB) 18/06/26 10:55:53 INFO SparkContext: Created broadcast 12 from broadcast at DAGScheduler.scala:1015 18/06/26 10:55:53 INFO DAGScheduler: Submitting 1 missing tasks from ResultStage 24 (ShuffledRDD[24] at reduceByKey at SparkWordCount.scala:52) 18/06/26 10:55:53 INFO TaskSchedulerImpl: Adding task set 24.0 with 1 tasks 18/06/26 10:55:53 INFO TaskSetManager: Starting task 0.0 in stage 24.0 (TID 12, localhost, partition 1,PROCESS_LOCAL, 1894 bytes) 18/06/26 10:55:53 INFO Executor: Running task 0.0 in stage 24.0 (TID 12) 18/06/26 10:55:53 INFO ShuffleBlockFetcherIterator: Getting 0 non-empty blocks out of 0 blocks 18/06/26 10:55:53 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 18/06/26 10:55:53 INFO Executor: Finished task 0.0 in stage 24.0 (TID 12). 1161 bytes result sent to driver 18/06/26 10:55:53 INFO TaskSetManager: Finished task 0.0 in stage 24.0 (TID 12) in 4 ms on localhost (1/1) 18/06/26 10:55:53 INFO TaskSchedulerImpl: Removed TaskSet 24.0, whose tasks have all completed, from pool 18/06/26 10:55:53 INFO DAGScheduler: ResultStage 24 (print at SparkWordCount.scala:55) finished in 0.004 s 18/06/26 10:55:53 INFO DAGScheduler: Job 12 finished: print at SparkWordCount.scala:55, took 0.013128 s ------------------------------------------- Time: 1529981753000 ms ------------------------------------------- 18/06/26 10:55:53 INFO JobScheduler: Finished job streaming job 1529981753000 ms.0 from job set of time 1529981753000 ms 18/06/26 10:55:53 INFO JobScheduler: Total delay: 0.080 s for time 1529981753000 ms (execution: 0.054 s) 18/06/26 10:55:53 INFO ShuffledRDD: Removing RDD 20 from persistence list 18/06/26 10:55:53 INFO BlockManager: Removing RDD 20 18/06/26 10:55:53 INFO MapPartitionsRDD: Removing RDD 19 from persistence list 18/06/26 10:55:53 INFO MapPartitionsRDD: Removing RDD 18 from persistence list 18/06/26 10:55:53 INFO BlockManager: Removing RDD 19 18/06/26 10:55:53 INFO BlockRDD: Removing RDD 17 from persistence list 18/06/26 10:55:53 INFO BlockManager: Removing RDD 18 18/06/26 10:55:53 INFO SocketInputDStream: Removing blocks of RDD BlockRDD[17] at socketTextStream at SparkWordCount.scala:46 of time 1529981753000 ms 18/06/26 10:55:53 INFO ReceivedBlockTracker: Deleting batches ArrayBuffer(1529981751000 ms) 18/06/26 10:55:53 INFO InputInfoTracker: remove old batch metadata: 1529981751000 ms 18/06/26 10:55:53 INFO BlockManager: Removing RDD 17 18/06/26 10:55:53 INFO ReceiverSupervisorImpl: Starting receiver again 18/06/26 10:55:53 INFO ReceiverTracker: Registered receiver for stream 0 from 10.41.88.38:55640 18/06/26 10:55:53 INFO ReceiverSupervisorImpl: Starting receiver 18/06/26 10:55:53 INFO ReceiverSupervisorImpl: Called receiver onStart 18/06/26 10:55:53 INFO ReceiverSupervisorImpl: Receiver started again 18/06/26 10:55:53 INFO SocketReceiver: Connecting to 192.168.56.101:9999 18/06/26 10:55:53 INFO SocketReceiver: Connected to 192.168.56.101:9999 18/06/26 10:55:54 INFO JobScheduler: Added jobs for time 1529981754000 ms 18/06/26 10:55:54 INFO JobScheduler: Starting job streaming job 1529981754000 ms.0 from job set of time 1529981754000 ms 18/06/26 10:55:54 INFO SparkContext: Starting job: print at SparkWordCount.scala:55 18/06/26 10:55:54 INFO DAGScheduler: Registering RDD 27 (map at SparkWordCount.scala:51) 18/06/26 10:55:54 INFO DAGScheduler: Got job 13 (print at SparkWordCount.scala:55) with 1 output partitions 18/06/26 10:55:54 INFO DAGScheduler: Final stage: ResultStage 26 (print at SparkWordCount.scala:55) 18/06/26 10:55:54 INFO DAGScheduler: Parents of final stage: List(ShuffleMapStage 25) 18/06/26 10:55:54 INFO DAGScheduler: Missing parents: List() 18/06/26 10:55:54 INFO DAGScheduler: Submitting ResultStage 26 (ShuffledRDD[28] at reduceByKey at SparkWordCount.scala:52), which has no missing parents 18/06/26 10:55:54 INFO MemoryStore: Block broadcast_13 stored as values in memory (estimated size 2.6 KB, free 89.6 KB) 18/06/26 10:55:54 INFO MemoryStore: Block broadcast_13_piece0 stored as bytes in memory (estimated size 1637.0 B, free 91.2 KB) 18/06/26 10:55:54 INFO BlockManagerInfo: Added broadcast_13_piece0 in memory on localhost:55667 (size: 1637.0 B, free: 2.4 GB) 18/06/26 10:55:54 INFO SparkContext: Created broadcast 13 from broadcast at DAGScheduler.scala:1015 18/06/26 10:55:54 INFO DAGScheduler: Submitting 1 missing tasks from ResultStage 26 (ShuffledRDD[28] at reduceByKey at SparkWordCount.scala:52) 18/06/26 10:55:54 INFO TaskSchedulerImpl: Adding task set 26.0 with 1 tasks 18/06/26 10:55:54 INFO TaskSetManager: Starting task 0.0 in stage 26.0 (TID 13, localhost, partition 0,PROCESS_LOCAL, 1894 bytes) 18/06/26 10:55:54 INFO Executor: Running task 0.0 in stage 26.0 (TID 13) 18/06/26 10:55:54 INFO ShuffleBlockFetcherIterator: Getting 0 non-empty blocks out of 0 blocks 18/06/26 10:55:54 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 18/06/26 10:55:54 INFO Executor: Finished task 0.0 in stage 26.0 (TID 13). 1161 bytes result sent to driver 18/06/26 10:55:54 INFO TaskSetManager: Finished task 0.0 in stage 26.0 (TID 13) in 2 ms on localhost (1/1) 18/06/26 10:55:54 INFO TaskSchedulerImpl: Removed TaskSet 26.0, whose tasks have all completed, from pool 18/06/26 10:55:54 INFO DAGScheduler: ResultStage 26 (print at SparkWordCount.scala:55) finished in 0.003 s 18/06/26 10:55:54 INFO DAGScheduler: Job 13 finished: print at SparkWordCount.scala:55, took 0.010641 s 18/06/26 10:55:54 INFO SparkContext: Starting job: print at SparkWordCount.scala:55 18/06/26 10:55:54 INFO MapOutputTrackerMaster: Size of output statuses for shuffle 6 is 82 bytes 18/06/26 10:55:54 INFO DAGScheduler: Got job 14 (print at SparkWordCount.scala:55) with 1 output partitions 18/06/26 10:55:54 INFO DAGScheduler: Final stage: ResultStage 28 (print at SparkWordCount.scala:55) 18/06/26 10:55:54 INFO DAGScheduler: Parents of final stage: List(ShuffleMapStage 27) 18/06/26 10:55:54 INFO DAGScheduler: Missing parents: List() 18/06/26 10:55:54 INFO DAGScheduler: Submitting ResultStage 28 (ShuffledRDD[28] at reduceByKey at SparkWordCount.scala:52), which has no missing parents 18/06/26 10:55:54 INFO MemoryStore: Block broadcast_14 stored as values in memory (estimated size 2.6 KB, free 93.8 KB) 18/06/26 10:55:54 INFO MemoryStore: Block broadcast_14_piece0 stored as bytes in memory (estimated size 1637.0 B, free 95.4 KB) 18/06/26 10:55:54 INFO BlockManagerInfo: Added broadcast_14_piece0 in memory on localhost:55667 (size: 1637.0 B, free: 2.4 GB) 18/06/26 10:55:54 INFO SparkContext: Created broadcast 14 from broadcast at DAGScheduler.scala:1015 18/06/26 10:55:54 INFO DAGScheduler: Submitting 1 missing tasks from ResultStage 28 (ShuffledRDD[28] at reduceByKey at SparkWordCount.scala:52) 18/06/26 10:55:54 INFO TaskSchedulerImpl: Adding task set 28.0 with 1 tasks 18/06/26 10:55:54 INFO TaskSetManager: Starting task 0.0 in stage 28.0 (TID 14, localhost, partition 1,PROCESS_LOCAL, 1894 bytes) 18/06/26 10:55:54 INFO Executor: Running task 0.0 in stage 28.0 (TID 14) 18/06/26 10:55:54 INFO ShuffleBlockFetcherIterator: Getting 0 non-empty blocks out of 0 blocks 18/06/26 10:55:54 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 18/06/26 10:55:54 INFO Executor: Finished task 0.0 in stage 28.0 (TID 14). 1161 bytes result sent to driver 18/06/26 10:55:54 INFO TaskSetManager: Finished task 0.0 in stage 28.0 (TID 14) in 5 ms on localhost (1/1) 18/06/26 10:55:54 INFO TaskSchedulerImpl: Removed TaskSet 28.0, whose tasks have all completed, from pool 18/06/26 10:55:54 INFO DAGScheduler: ResultStage 28 (print at SparkWordCount.scala:55) finished in 0.005 s 18/06/26 10:55:54 INFO DAGScheduler: Job 14 finished: print at SparkWordCount.scala:55, took 0.010471 s 18/06/26 10:55:54 INFO JobScheduler: Finished job streaming job 1529981754000 ms.0 from job set of time 1529981754000 ms 18/06/26 10:55:54 INFO JobScheduler: Total delay: 0.042 s for time 1529981754000 ms (execution: 0.032 s) ------------------------------------------- Time: 1529981754000 ms ------------------------------------------- 18/06/26 10:55:54 INFO ShuffledRDD: Removing RDD 24 from persistence list 18/06/26 10:55:54 INFO BlockManager: Removing RDD 24 18/06/26 10:55:54 INFO MapPartitionsRDD: Removing RDD 23 from persistence list 18/06/26 10:55:54 INFO BlockManager: Removing RDD 23 18/06/26 10:55:54 INFO MapPartitionsRDD: Removing RDD 22 from persistence list 18/06/26 10:55:54 INFO BlockRDD: Removing RDD 21 from persistence list 18/06/26 10:55:54 INFO BlockManager: Removing RDD 22 18/06/26 10:55:54 INFO SocketInputDStream: Removing blocks of RDD BlockRDD[21] at socketTextStream at SparkWordCount.scala:46 of time 1529981754000 ms 18/06/26 10:55:54 INFO ReceivedBlockTracker: Deleting batches ArrayBuffer(1529981752000 ms) 18/06/26 10:55:54 INFO InputInfoTracker: remove old batch metadata: 1529981752000 ms 18/06/26 10:55:54 INFO BlockManager: Removing RDD 21 18/06/26 10:55:55 INFO JobScheduler: Added jobs for time 1529981755000 ms 18/06/26 10:55:55 INFO JobScheduler: Starting job streaming job 1529981755000 ms.0 from job set of time 1529981755000 ms 18/06/26 10:55:55 INFO SparkContext: Starting job: print at SparkWordCount.scala:55 18/06/26 10:55:55 INFO DAGScheduler: Registering RDD 31 (map at SparkWordCount.scala:51) 18/06/26 10:55:55 INFO DAGScheduler: Got job 15 (print at SparkWordCount.scala:55) with 1 output partitions 18/06/26 10:55:55 INFO DAGScheduler: Final stage: ResultStage 30 (print at SparkWordCount.scala:55) 18/06/26 10:55:55 INFO DAGScheduler: Parents of final stage: List(ShuffleMapStage 29) 18/06/26 10:55:55 INFO DAGScheduler: Missing parents: List() 18/06/26 10:55:55 INFO DAGScheduler: Submitting ResultStage 30 (ShuffledRDD[32] at reduceByKey at SparkWordCount.scala:52), which has no missing parents 18/06/26 10:55:55 INFO MemoryStore: Block broadcast_15 stored as values in memory (estimated size 2.6 KB, free 98.0 KB) 18/06/26 10:55:55 INFO MemoryStore: Block broadcast_15_piece0 stored as bytes in memory (estimated size 1637.0 B, free 99.6 KB) 18/06/26 10:55:55 INFO BlockManagerInfo: Added broadcast_15_piece0 in memory on localhost:55667 (size: 1637.0 B, free: 2.4 GB) 18/06/26 10:55:55 INFO SparkContext: Created broadcast 15 from broadcast at DAGScheduler.scala:1015 18/06/26 10:55:55 INFO DAGScheduler: Submitting 1 missing tasks from ResultStage 30 (ShuffledRDD[32] at reduceByKey at SparkWordCount.scala:52) 18/06/26 10:55:55 INFO TaskSchedulerImpl: Adding task set 30.0 with 1 tasks 18/06/26 10:55:55 INFO TaskSetManager: Starting task 0.0 in stage 30.0 (TID 15, localhost, partition 0,PROCESS_LOCAL, 1894 bytes) 18/06/26 10:55:55 INFO Executor: Running task 0.0 in stage 30.0 (TID 15) 18/06/26 10:55:55 INFO ShuffleBlockFetcherIterator: Getting 0 non-empty blocks out of 0 blocks 18/06/26 10:55:55 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 18/06/26 10:55:55 INFO Executor: Finished task 0.0 in stage 30.0 (TID 15). 1161 bytes result sent to driver 18/06/26 10:55:55 INFO TaskSetManager: Finished task 0.0 in stage 30.0 (TID 15) in 5 ms on localhost (1/1) 18/06/26 10:55:55 INFO TaskSchedulerImpl: Removed TaskSet 30.0, whose tasks have all completed, from pool 18/06/26 10:55:55 INFO DAGScheduler: ResultStage 30 (print at SparkWordCount.scala:55) finished in 0.005 s 18/06/26 10:55:55 INFO DAGScheduler: Job 15 finished: print at SparkWordCount.scala:55, took 0.012644 s 18/06/26 10:55:55 INFO SparkContext: Starting job: print at SparkWordCount.scala:55 18/06/26 10:55:55 INFO MapOutputTrackerMaster: Size of output statuses for shuffle 7 is 82 bytes 18/06/26 10:55:55 INFO DAGScheduler: Got job 16 (print at SparkWordCount.scala:55) with 1 output partitions 18/06/26 10:55:55 INFO DAGScheduler: Final stage: ResultStage 32 (print at SparkWordCount.scala:55) 18/06/26 10:55:55 INFO DAGScheduler: Parents of final stage: List(ShuffleMapStage 31) 18/06/26 10:55:55 INFO DAGScheduler: Missing parents: List() 18/06/26 10:55:55 INFO DAGScheduler: Submitting ResultStage 32 (ShuffledRDD[32] at reduceByKey at SparkWordCount.scala:52), which has no missing parents 18/06/26 10:55:55 INFO MemoryStore: Block broadcast_16 stored as values in memory (estimated size 2.6 KB, free 102.2 KB) 18/06/26 10:55:55 INFO MemoryStore: Block broadcast_16_piece0 stored as bytes in memory (estimated size 1637.0 B, free 103.8 KB) 18/06/26 10:55:55 INFO BlockManagerInfo: Added broadcast_16_piece0 in memory on localhost:55667 (size: 1637.0 B, free: 2.4 GB) 18/06/26 10:55:55 INFO SparkContext: Created broadcast 16 from broadcast at DAGScheduler.scala:1015 18/06/26 10:55:55 INFO DAGScheduler: Submitting 1 missing tasks from ResultStage 32 (ShuffledRDD[32] at reduceByKey at SparkWordCount.scala:52) 18/06/26 10:55:55 INFO TaskSchedulerImpl: Adding task set 32.0 with 1 tasks 18/06/26 10:55:55 INFO TaskSetManager: Starting task 0.0 in stage 32.0 (TID 16, localhost, partition 1,PROCESS_LOCAL, 1894 bytes) 18/06/26 10:55:55 INFO Executor: Running task 0.0 in stage 32.0 (TID 16) 18/06/26 10:55:55 INFO ShuffleBlockFetcherIterator: Getting 0 non-empty blocks out of 0 blocks 18/06/26 10:55:55 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 18/06/26 10:55:55 INFO Executor: Finished task 0.0 in stage 32.0 (TID 16). 1161 bytes result sent to driver 18/06/26 10:55:55 INFO TaskSetManager: Finished task 0.0 in stage 32.0 (TID 16) in 5 ms on localhost (1/1) 18/06/26 10:55:55 INFO TaskSchedulerImpl: Removed TaskSet 32.0, whose tasks have all completed, from pool 18/06/26 10:55:55 INFO DAGScheduler: ResultStage 32 (print at SparkWordCount.scala:55) finished in 0.005 s 18/06/26 10:55:55 INFO DAGScheduler: Job 16 finished: print at SparkWordCount.scala:55, took 0.012046 s ------------------------------------------- Time: 1529981755000 ms ------------------------------------------- 18/06/26 10:55:55 INFO JobScheduler: Finished job streaming job 1529981755000 ms.0 from job set of time 1529981755000 ms 18/06/26 10:55:55 INFO JobScheduler: Total delay: 0.068 s for time 1529981755000 ms (execution: 0.044 s) 18/06/26 10:55:55 INFO ShuffledRDD: Removing RDD 28 from persistence list 18/06/26 10:55:55 INFO BlockManager: Removing RDD 28 18/06/26 10:55:55 INFO MapPartitionsRDD: Removing RDD 27 from persistence list 18/06/26 10:55:55 INFO BlockManager: Removing RDD 27 18/06/26 10:55:55 INFO MapPartitionsRDD: Removing RDD 26 from persistence list 18/06/26 10:55:55 INFO BlockManager: Removing RDD 26 18/06/26 10:55:55 INFO BlockRDD: Removing RDD 25 from persistence list 18/06/26 10:55:55 INFO BlockManager: Removing RDD 25 18/06/26 10:55:55 INFO SocketInputDStream: Removing blocks of RDD BlockRDD[25] at socketTextStream at SparkWordCount.scala:46 of time 1529981755000 ms 18/06/26 10:55:55 INFO ReceivedBlockTracker: Deleting batches ArrayBuffer(1529981753000 ms) 18/06/26 10:55:55 INFO InputInfoTracker: remove old batch metadata: 1529981753000 ms 18/06/26 10:55:56 INFO MemoryStore: Block input-0-1529981755800 stored as bytes in memory (estimated size 18.0 B, free 103.9 KB) 18/06/26 10:55:56 INFO BlockManagerInfo: Added input-0-1529981755800 in memory on localhost:55667 (size: 18.0 B, free: 2.4 GB) 18/06/26 10:55:56 WARN BlockManager: Block input-0-1529981755800 replicated to only 0 peer(s) instead of 1 peers 18/06/26 10:55:56 INFO JobScheduler: Added jobs for time 1529981756000 ms 18/06/26 10:55:56 INFO JobScheduler: Starting job streaming job 1529981756000 ms.0 from job set of time 1529981756000 ms 18/06/26 10:55:56 INFO SparkContext: Starting job: print at SparkWordCount.scala:55 18/06/26 10:55:56 INFO DAGScheduler: Registering RDD 35 (map at SparkWordCount.scala:51) 18/06/26 10:55:56 INFO BlockGenerator: Pushed block input-0-1529981755800 18/06/26 10:55:56 INFO DAGScheduler: Got job 17 (print at SparkWordCount.scala:55) with 1 output partitions 18/06/26 10:55:56 INFO DAGScheduler: Final stage: ResultStage 34 (print at SparkWordCount.scala:55) 18/06/26 10:55:56 INFO DAGScheduler: Parents of final stage: List(ShuffleMapStage 33) 18/06/26 10:55:56 INFO DAGScheduler: Missing parents: List() 18/06/26 10:55:56 INFO DAGScheduler: Submitting ResultStage 34 (ShuffledRDD[36] at reduceByKey at SparkWordCount.scala:52), which has no missing parents 18/06/26 10:55:56 INFO MemoryStore: Block broadcast_17 stored as values in memory (estimated size 2.6 KB, free 106.5 KB) 18/06/26 10:55:56 INFO MemoryStore: Block broadcast_17_piece0 stored as bytes in memory (estimated size 1637.0 B, free 108.1 KB) 18/06/26 10:55:56 INFO BlockManagerInfo: Added broadcast_17_piece0 in memory on localhost:55667 (size: 1637.0 B, free: 2.4 GB) 18/06/26 10:55:56 INFO SparkContext: Created broadcast 17 from broadcast at DAGScheduler.scala:1015 18/06/26 10:55:56 INFO DAGScheduler: Submitting 1 missing tasks from ResultStage 34 (ShuffledRDD[36] at reduceByKey at SparkWordCount.scala:52) 18/06/26 10:55:56 INFO TaskSchedulerImpl: Adding task set 34.0 with 1 tasks 18/06/26 10:55:56 INFO TaskSetManager: Starting task 0.0 in stage 34.0 (TID 17, localhost, partition 0,PROCESS_LOCAL, 1894 bytes) 18/06/26 10:55:56 INFO Executor: Running task 0.0 in stage 34.0 (TID 17) 18/06/26 10:55:56 INFO ShuffleBlockFetcherIterator: Getting 0 non-empty blocks out of 0 blocks 18/06/26 10:55:56 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 18/06/26 10:55:56 INFO Executor: Finished task 0.0 in stage 34.0 (TID 17). 1161 bytes result sent to driver 18/06/26 10:55:56 INFO TaskSetManager: Finished task 0.0 in stage 34.0 (TID 17) in 4 ms on localhost (1/1) 18/06/26 10:55:56 INFO TaskSchedulerImpl: Removed TaskSet 34.0, whose tasks have all completed, from pool 18/06/26 10:55:56 INFO DAGScheduler: ResultStage 34 (print at SparkWordCount.scala:55) finished in 0.005 s 18/06/26 10:55:56 INFO DAGScheduler: Job 17 finished: print at SparkWordCount.scala:55, took 0.013957 s 18/06/26 10:55:56 INFO SparkContext: Starting job: print at SparkWordCount.scala:55 18/06/26 10:55:56 INFO MapOutputTrackerMaster: Size of output statuses for shuffle 8 is 82 bytes 18/06/26 10:55:56 INFO DAGScheduler: Got job 18 (print at SparkWordCount.scala:55) with 1 output partitions 18/06/26 10:55:56 INFO DAGScheduler: Final stage: ResultStage 36 (print at SparkWordCount.scala:55) 18/06/26 10:55:56 INFO DAGScheduler: Parents of final stage: List(ShuffleMapStage 35) 18/06/26 10:55:56 INFO DAGScheduler: Missing parents: List() 18/06/26 10:55:56 INFO DAGScheduler: Submitting ResultStage 36 (ShuffledRDD[36] at reduceByKey at SparkWordCount.scala:52), which has no missing parents 18/06/26 10:55:56 INFO MemoryStore: Block broadcast_18 stored as values in memory (estimated size 2.6 KB, free 110.7 KB) 18/06/26 10:55:56 INFO MemoryStore: Block broadcast_18_piece0 stored as bytes in memory (estimated size 1637.0 B, free 112.3 KB) 18/06/26 10:55:56 INFO BlockManagerInfo: Added broadcast_18_piece0 in memory on localhost:55667 (size: 1637.0 B, free: 2.4 GB) 18/06/26 10:55:56 INFO SparkContext: Created broadcast 18 from broadcast at DAGScheduler.scala:1015 18/06/26 10:55:56 INFO DAGScheduler: Submitting 1 missing tasks from ResultStage 36 (ShuffledRDD[36] at reduceByKey at SparkWordCount.scala:52) 18/06/26 10:55:56 INFO TaskSchedulerImpl: Adding task set 36.0 with 1 tasks 18/06/26 10:55:56 INFO TaskSetManager: Starting task 0.0 in stage 36.0 (TID 18, localhost, partition 1,PROCESS_LOCAL, 1894 bytes) 18/06/26 10:55:56 INFO Executor: Running task 0.0 in stage 36.0 (TID 18) 18/06/26 10:55:56 INFO ShuffleBlockFetcherIterator: Getting 0 non-empty blocks out of 0 blocks 18/06/26 10:55:56 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 18/06/26 10:55:56 INFO Executor: Finished task 0.0 in stage 36.0 (TID 18). 1161 bytes result sent to driver 18/06/26 10:55:56 INFO TaskSetManager: Finished task 0.0 in stage 36.0 (TID 18) in 1 ms on localhost (1/1) 18/06/26 10:55:56 INFO TaskSchedulerImpl: Removed TaskSet 36.0, whose tasks have all completed, from pool 18/06/26 10:55:56 INFO DAGScheduler: ResultStage 36 (print at SparkWordCount.scala:55) finished in 0.002 s 18/06/26 10:55:56 INFO DAGScheduler: Job 18 finished: print at SparkWordCount.scala:55, took 0.005934 s ------------------------------------------- Time: 1529981756000 ms ------------------------------------------- 18/06/26 10:55:56 INFO JobScheduler: Finished job streaming job 1529981756000 ms.0 from job set of time 1529981756000 ms 18/06/26 10:55:56 INFO JobScheduler: Total delay: 0.062 s for time 1529981756000 ms (execution: 0.032 s) 18/06/26 10:55:56 INFO ShuffledRDD: Removing RDD 32 from persistence list 18/06/26 10:55:56 INFO BlockManager: Removing RDD 32 18/06/26 10:55:56 INFO MapPartitionsRDD: Removing RDD 31 from persistence list 18/06/26 10:55:56 INFO BlockManager: Removing RDD 31 18/06/26 10:55:56 INFO MapPartitionsRDD: Removing RDD 30 from persistence list 18/06/26 10:55:56 INFO BlockManager: Removing RDD 30 18/06/26 10:55:56 INFO BlockRDD: Removing RDD 29 from persistence list 18/06/26 10:55:56 INFO SocketInputDStream: Removing blocks of RDD BlockRDD[29] at socketTextStream at SparkWordCount.scala:46 of time 1529981756000 ms 18/06/26 10:55:56 INFO ReceivedBlockTracker: Deleting batches ArrayBuffer(1529981754000 ms) 18/06/26 10:55:56 INFO InputInfoTracker: remove old batch metadata: 1529981754000 ms 18/06/26 10:55:56 INFO BlockManager: Removing RDD 29 18/06/26 10:55:57 INFO JobScheduler: Added jobs for time 1529981757000 ms 18/06/26 10:55:57 INFO JobScheduler: Starting job streaming job 1529981757000 ms.0 from job set of time 1529981757000 ms 18/06/26 10:55:57 INFO SparkContext: Starting job: print at SparkWordCount.scala:55 18/06/26 10:55:57 INFO DAGScheduler: Registering RDD 39 (map at SparkWordCount.scala:51) 18/06/26 10:55:57 INFO DAGScheduler: Got job 19 (print at SparkWordCount.scala:55) with 1 output partitions 18/06/26 10:55:57 INFO DAGScheduler: Final stage: ResultStage 38 (print at SparkWordCount.scala:55) 18/06/26 10:55:57 INFO DAGScheduler: Parents of final stage: List(ShuffleMapStage 37) 18/06/26 10:55:57 INFO DAGScheduler: Missing parents: List(ShuffleMapStage 37) 18/06/26 10:55:57 INFO DAGScheduler: Submitting ShuffleMapStage 37 (MapPartitionsRDD[39] at map at SparkWordCount.scala:51), which has no missing parents 18/06/26 10:55:57 INFO MemoryStore: Block broadcast_19 stored as values in memory (estimated size 2.7 KB, free 114.9 KB) 18/06/26 10:55:57 INFO MemoryStore: Block broadcast_19_piece0 stored as bytes in memory (estimated size 1643.0 B, free 116.5 KB) 18/06/26 10:55:57 INFO BlockManagerInfo: Added broadcast_19_piece0 in memory on localhost:55667 (size: 1643.0 B, free: 2.4 GB) 18/06/26 10:55:57 INFO SparkContext: Created broadcast 19 from broadcast at DAGScheduler.scala:1015 18/06/26 10:55:57 INFO DAGScheduler: Submitting 1 missing tasks from ShuffleMapStage 37 (MapPartitionsRDD[39] at map at SparkWordCount.scala:51) 18/06/26 10:55:57 INFO TaskSchedulerImpl: Adding task set 37.0 with 1 tasks 18/06/26 10:55:57 INFO TaskSetManager: Starting task 0.0 in stage 37.0 (TID 19, localhost, partition 0,NODE_LOCAL, 2006 bytes) 18/06/26 10:55:57 INFO Executor: Running task 0.0 in stage 37.0 (TID 19) 18/06/26 10:55:57 INFO BlockManager: Found block input-0-1529981755800 locally 18/06/26 10:55:57 INFO Executor: Finished task 0.0 in stage 37.0 (TID 19). 1159 bytes result sent to driver 18/06/26 10:55:57 INFO TaskSetManager: Finished task 0.0 in stage 37.0 (TID 19) in 31 ms on localhost (1/1) 18/06/26 10:55:57 INFO TaskSchedulerImpl: Removed TaskSet 37.0, whose tasks have all completed, from pool 18/06/26 10:55:57 INFO DAGScheduler: ShuffleMapStage 37 (map at SparkWordCount.scala:51) finished in 0.033 s 18/06/26 10:55:57 INFO DAGScheduler: looking for newly runnable stages 18/06/26 10:55:57 INFO DAGScheduler: running: Set(ResultStage 0) 18/06/26 10:55:57 INFO DAGScheduler: waiting: Set(ResultStage 38) 18/06/26 10:55:57 INFO DAGScheduler: failed: Set() 18/06/26 10:55:57 INFO DAGScheduler: Submitting ResultStage 38 (ShuffledRDD[40] at reduceByKey at SparkWordCount.scala:52), which has no missing parents 18/06/26 10:55:57 INFO MemoryStore: Block broadcast_20 stored as values in memory (estimated size 2.6 KB, free 119.1 KB) 18/06/26 10:55:57 INFO MemoryStore: Block broadcast_20_piece0 stored as bytes in memory (estimated size 1633.0 B, free 120.7 KB) 18/06/26 10:55:57 INFO BlockManagerInfo: Added broadcast_20_piece0 in memory on localhost:55667 (size: 1633.0 B, free: 2.4 GB) 18/06/26 10:55:57 INFO SparkContext: Created broadcast 20 from broadcast at DAGScheduler.scala:1015 18/06/26 10:55:57 INFO DAGScheduler: Submitting 1 missing tasks from ResultStage 38 (ShuffledRDD[40] at reduceByKey at SparkWordCount.scala:52) 18/06/26 10:55:57 INFO TaskSchedulerImpl: Adding task set 38.0 with 1 tasks 18/06/26 10:55:57 INFO TaskSetManager: Starting task 0.0 in stage 38.0 (TID 20, localhost, partition 0,NODE_LOCAL, 1894 bytes) 18/06/26 10:55:57 INFO Executor: Running task 0.0 in stage 38.0 (TID 20) 18/06/26 10:55:57 INFO ShuffleBlockFetcherIterator: Getting 1 non-empty blocks out of 1 blocks 18/06/26 10:55:57 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 1 ms 18/06/26 10:55:57 INFO Executor: Finished task 0.0 in stage 38.0 (TID 20). 1330 bytes result sent to driver 18/06/26 10:55:57 INFO TaskSetManager: Finished task 0.0 in stage 38.0 (TID 20) in 8 ms on localhost (1/1) 18/06/26 10:55:57 INFO TaskSchedulerImpl: Removed TaskSet 38.0, whose tasks have all completed, from pool 18/06/26 10:55:57 INFO DAGScheduler: ResultStage 38 (print at SparkWordCount.scala:55) finished in 0.009 s 18/06/26 10:55:57 INFO DAGScheduler: Job 19 finished: print at SparkWordCount.scala:55, took 0.071868 s 18/06/26 10:55:57 INFO SparkContext: Starting job: print at SparkWordCount.scala:55 18/06/26 10:55:57 INFO MapOutputTrackerMaster: Size of output statuses for shuffle 9 is 145 bytes 18/06/26 10:55:57 INFO DAGScheduler: Got job 20 (print at SparkWordCount.scala:55) with 1 output partitions 18/06/26 10:55:57 INFO DAGScheduler: Final stage: ResultStage 40 (print at SparkWordCount.scala:55) 18/06/26 10:55:57 INFO DAGScheduler: Parents of final stage: List(ShuffleMapStage 39) 18/06/26 10:55:57 INFO DAGScheduler: Missing parents: List() 18/06/26 10:55:57 INFO DAGScheduler: Submitting ResultStage 40 (ShuffledRDD[40] at reduceByKey at SparkWordCount.scala:52), which has no missing parents 18/06/26 10:55:57 INFO MemoryStore: Block broadcast_21 stored as values in memory (estimated size 2.6 KB, free 123.3 KB) 18/06/26 10:55:57 INFO MemoryStore: Block broadcast_21_piece0 stored as bytes in memory (estimated size 1631.0 B, free 124.9 KB) 18/06/26 10:55:57 INFO BlockManagerInfo: Added broadcast_21_piece0 in memory on localhost:55667 (size: 1631.0 B, free: 2.4 GB) 18/06/26 10:55:57 INFO SparkContext: Created broadcast 21 from broadcast at DAGScheduler.scala:1015 18/06/26 10:55:57 INFO DAGScheduler: Submitting 1 missing tasks from ResultStage 40 (ShuffledRDD[40] at reduceByKey at SparkWordCount.scala:52) 18/06/26 10:55:57 INFO TaskSchedulerImpl: Adding task set 40.0 with 1 tasks 18/06/26 10:55:57 INFO TaskSetManager: Starting task 0.0 in stage 40.0 (TID 21, localhost, partition 1,PROCESS_LOCAL, 1894 bytes) 18/06/26 10:55:57 INFO Executor: Running task 0.0 in stage 40.0 (TID 21) 18/06/26 10:55:57 INFO ShuffleBlockFetcherIterator: Getting 0 non-empty blocks out of 1 blocks 18/06/26 10:55:57 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 18/06/26 10:55:57 INFO Executor: Finished task 0.0 in stage 40.0 (TID 21). 1161 bytes result sent to driver 18/06/26 10:55:57 INFO TaskSetManager: Finished task 0.0 in stage 40.0 (TID 21) in 2 ms on localhost (1/1) 18/06/26 10:55:57 INFO TaskSchedulerImpl: Removed TaskSet 40.0, whose tasks have all completed, from pool 18/06/26 10:55:57 INFO DAGScheduler: ResultStage 40 (print at SparkWordCount.scala:55) finished in 0.002 s 18/06/26 10:55:57 INFO DAGScheduler: Job 20 finished: print at SparkWordCount.scala:55, took 0.007949 s ------------------------------------------- Time: 1529981757000 ms ------------------------------------------- 18/06/26 10:55:57 INFO JobScheduler: Finished job streaming job 1529981757000 ms.0 from job set of time 1529981757000 ms 18/06/26 10:55:57 INFO JobScheduler: Total delay: 0.127 s for time 1529981757000 ms (execution: 0.096 s) 18/06/26 10:55:57 INFO ShuffledRDD: Removing RDD 36 from persistence list (hello,1) (world,1) 18/06/26 10:55:57 INFO BlockManager: Removing RDD 36 18/06/26 10:55:57 INFO MapPartitionsRDD: Removing RDD 35 from persistence list 18/06/26 10:55:57 INFO BlockManager: Removing RDD 35 18/06/26 10:55:57 INFO MapPartitionsRDD: Removing RDD 34 from persistence list 18/06/26 10:55:57 INFO BlockManager: Removing RDD 34 18/06/26 10:55:57 INFO BlockRDD: Removing RDD 33 from persistence list 18/06/26 10:55:57 INFO BlockManager: Removing RDD 33 18/06/26 10:55:57 INFO SocketInputDStream: Removing blocks of RDD BlockRDD[33] at socketTextStream at SparkWordCount.scala:46 of time 1529981757000 ms 18/06/26 10:55:57 INFO ReceivedBlockTracker: Deleting batches ArrayBuffer(1529981755000 ms) 18/06/26 10:55:57 INFO InputInfoTracker: remove old batch metadata: 1529981755000 ms 18/06/26 10:55:57 INFO MemoryStore: Block input-0-1529981757000 stored as bytes in memory (estimated size 18.0 B, free 124.9 KB) 18/06/26 10:55:57 INFO BlockManagerInfo: Added input-0-1529981757000 in memory on localhost:55667 (size: 18.0 B, free: 2.4 GB) 18/06/26 10:55:57 WARN BlockManager: Block input-0-1529981757000 replicated to only 0 peer(s) instead of 1 peers 18/06/26 10:55:57 INFO BlockGenerator: Pushed block input-0-1529981757000 18/06/26 10:55:58 INFO JobScheduler: Added jobs for time 1529981758000 ms 18/06/26 10:55:58 INFO JobScheduler: Starting job streaming job 1529981758000 ms.0 from job set of time 1529981758000 ms 18/06/26 10:55:58 INFO SparkContext: Starting job: print at SparkWordCount.scala:55 18/06/26 10:55:58 INFO DAGScheduler: Registering RDD 43 (map at SparkWordCount.scala:51) 18/06/26 10:55:58 INFO DAGScheduler: Got job 21 (print at SparkWordCount.scala:55) with 1 output partitions 18/06/26 10:55:58 INFO DAGScheduler: Final stage: ResultStage 42 (print at SparkWordCount.scala:55) 18/06/26 10:55:58 INFO DAGScheduler: Parents of final stage: List(ShuffleMapStage 41) 18/06/26 10:55:58 INFO DAGScheduler: Missing parents: List(ShuffleMapStage 41) 18/06/26 10:55:58 INFO DAGScheduler: Submitting ShuffleMapStage 41 (MapPartitionsRDD[43] at map at SparkWordCount.scala:51), which has no missing parents 18/06/26 10:55:58 INFO MemoryStore: Block broadcast_22 stored as values in memory (estimated size 2.7 KB, free 127.6 KB) 18/06/26 10:55:58 INFO MemoryStore: Block broadcast_22_piece0 stored as bytes in memory (estimated size 1643.0 B, free 129.2 KB) 18/06/26 10:55:58 INFO BlockManagerInfo: Added broadcast_22_piece0 in memory on localhost:55667 (size: 1643.0 B, free: 2.4 GB) 18/06/26 10:55:58 INFO SparkContext: Created broadcast 22 from broadcast at DAGScheduler.scala:1015 18/06/26 10:55:58 INFO DAGScheduler: Submitting 1 missing tasks from ShuffleMapStage 41 (MapPartitionsRDD[43] at map at SparkWordCount.scala:51) 18/06/26 10:55:58 INFO TaskSchedulerImpl: Adding task set 41.0 with 1 tasks 18/06/26 10:55:58 INFO TaskSetManager: Starting task 0.0 in stage 41.0 (TID 22, localhost, partition 0,NODE_LOCAL, 2006 bytes) 18/06/26 10:55:58 INFO Executor: Running task 0.0 in stage 41.0 (TID 22) 18/06/26 10:55:58 INFO BlockManager: Found block input-0-1529981757000 locally 18/06/26 10:55:58 INFO Executor: Finished task 0.0 in stage 41.0 (TID 22). 1159 bytes result sent to driver 18/06/26 10:55:58 INFO TaskSetManager: Finished task 0.0 in stage 41.0 (TID 22) in 10 ms on localhost (1/1) 18/06/26 10:55:58 INFO TaskSchedulerImpl: Removed TaskSet 41.0, whose tasks have all completed, from pool 18/06/26 10:55:58 INFO DAGScheduler: ShuffleMapStage 41 (map at SparkWordCount.scala:51) finished in 0.010 s 18/06/26 10:55:58 INFO DAGScheduler: looking for newly runnable stages 18/06/26 10:55:58 INFO DAGScheduler: running: Set(ResultStage 0) 18/06/26 10:55:58 INFO DAGScheduler: waiting: Set(ResultStage 42) 18/06/26 10:55:58 INFO DAGScheduler: failed: Set() 18/06/26 10:55:58 INFO DAGScheduler: Submitting ResultStage 42 (ShuffledRDD[44] at reduceByKey at SparkWordCount.scala:52), which has no missing parents 18/06/26 10:55:58 INFO MemoryStore: Block broadcast_23 stored as values in memory (estimated size 2.6 KB, free 131.8 KB) 18/06/26 10:55:58 INFO MemoryStore: Block broadcast_23_piece0 stored as bytes in memory (estimated size 1633.0 B, free 133.4 KB) 18/06/26 10:55:58 INFO BlockManagerInfo: Added broadcast_23_piece0 in memory on localhost:55667 (size: 1633.0 B, free: 2.4 GB) 18/06/26 10:55:58 INFO SparkContext: Created broadcast 23 from broadcast at DAGScheduler.scala:1015 18/06/26 10:55:58 INFO DAGScheduler: Submitting 1 missing tasks from ResultStage 42 (ShuffledRDD[44] at reduceByKey at SparkWordCount.scala:52) 18/06/26 10:55:58 INFO TaskSchedulerImpl: Adding task set 42.0 with 1 tasks 18/06/26 10:55:58 INFO TaskSetManager: Starting task 0.0 in stage 42.0 (TID 23, localhost, partition 0,NODE_LOCAL, 1894 bytes) 18/06/26 10:55:58 INFO Executor: Running task 0.0 in stage 42.0 (TID 23) 18/06/26 10:55:58 INFO ShuffleBlockFetcherIterator: Getting 1 non-empty blocks out of 1 blocks 18/06/26 10:55:58 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 18/06/26 10:55:58 INFO Executor: Finished task 0.0 in stage 42.0 (TID 23). 1330 bytes result sent to driver 18/06/26 10:55:58 INFO TaskSetManager: Finished task 0.0 in stage 42.0 (TID 23) in 4 ms on localhost (1/1) 18/06/26 10:55:58 INFO TaskSchedulerImpl: Removed TaskSet 42.0, whose tasks have all completed, from pool 18/06/26 10:55:58 INFO DAGScheduler: ResultStage 42 (print at SparkWordCount.scala:55) finished in 0.004 s 18/06/26 10:55:58 INFO DAGScheduler: Job 21 finished: print at SparkWordCount.scala:55, took 0.028594 s 18/06/26 10:55:58 INFO SparkContext: Starting job: print at SparkWordCount.scala:55 18/06/26 10:55:58 INFO MapOutputTrackerMaster: Size of output statuses for shuffle 10 is 145 bytes 18/06/26 10:55:58 INFO DAGScheduler: Got job 22 (print at SparkWordCount.scala:55) with 1 output partitions 18/06/26 10:55:58 INFO DAGScheduler: Final stage: ResultStage 44 (print at SparkWordCount.scala:55) 18/06/26 10:55:58 INFO DAGScheduler: Parents of final stage: List(ShuffleMapStage 43) 18/06/26 10:55:58 INFO DAGScheduler: Missing parents: List() 18/06/26 10:55:58 INFO DAGScheduler: Submitting ResultStage 44 (ShuffledRDD[44] at reduceByKey at SparkWordCount.scala:52), which has no missing parents 18/06/26 10:55:58 INFO MemoryStore: Block broadcast_24 stored as values in memory (estimated size 2.6 KB, free 136.0 KB) 18/06/26 10:55:58 INFO MemoryStore: Block broadcast_24_piece0 stored as bytes in memory (estimated size 1633.0 B, free 137.6 KB) 18/06/26 10:55:58 INFO BlockManagerInfo: Added broadcast_24_piece0 in memory on localhost:55667 (size: 1633.0 B, free: 2.4 GB) 18/06/26 10:55:58 INFO SparkContext: Created broadcast 24 from broadcast at DAGScheduler.scala:1015 18/06/26 10:55:58 INFO DAGScheduler: Submitting 1 missing tasks from ResultStage 44 (ShuffledRDD[44] at reduceByKey at SparkWordCount.scala:52) 18/06/26 10:55:58 INFO TaskSchedulerImpl: Adding task set 44.0 with 1 tasks 18/06/26 10:55:58 INFO TaskSetManager: Starting task 0.0 in stage 44.0 (TID 24, localhost, partition 1,PROCESS_LOCAL, 1894 bytes) 18/06/26 10:55:58 INFO Executor: Running task 0.0 in stage 44.0 (TID 24) 18/06/26 10:55:58 INFO ShuffleBlockFetcherIterator: Getting 0 non-empty blocks out of 1 blocks 18/06/26 10:55:58 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 18/06/26 10:55:58 INFO Executor: Finished task 0.0 in stage 44.0 (TID 24). 1161 bytes result sent to driver 18/06/26 10:55:58 INFO TaskSetManager: Finished task 0.0 in stage 44.0 (TID 24) in 2 ms on localhost (1/1) 18/06/26 10:55:58 INFO TaskSchedulerImpl: Removed TaskSet 44.0, whose tasks have all completed, from pool 18/06/26 10:55:58 INFO DAGScheduler: ResultStage 44 (print at SparkWordCount.scala:55) finished in 0.002 s 18/06/26 10:55:58 INFO DAGScheduler: Job 22 finished: print at SparkWordCount.scala:55, took 0.010047 s ------------------------------------------- Time: 1529981758000 ms ------------------------------------------- (hello,1) (world,1) 18/06/26 10:55:58 INFO JobScheduler: Finished job streaming job 1529981758000 ms.0 from job set of time 1529981758000 ms 18/06/26 10:55:58 INFO JobScheduler: Total delay: 0.082 s for time 1529981758000 ms (execution: 0.052 s) 18/06/26 10:55:58 INFO ShuffledRDD: Removing RDD 40 from persistence list 18/06/26 10:55:58 INFO BlockManager: Removing RDD 40 18/06/26 10:55:58 INFO MapPartitionsRDD: Removing RDD 39 from persistence list 18/06/26 10:55:58 INFO BlockManager: Removing RDD 39 18/06/26 10:55:58 INFO MapPartitionsRDD: Removing RDD 38 from persistence list 18/06/26 10:55:58 INFO BlockRDD: Removing RDD 37 from persistence list 18/06/26 10:55:58 INFO BlockManager: Removing RDD 38 18/06/26 10:55:58 INFO BlockManager: Removing RDD 37 18/06/26 10:55:58 INFO SocketInputDStream: Removing blocks of RDD BlockRDD[37] at socketTextStream at SparkWordCount.scala:46 of time 1529981758000 ms 18/06/26 10:55:58 INFO ReceivedBlockTracker: Deleting batches ArrayBuffer(1529981756000 ms) 18/06/26 10:55:58 INFO InputInfoTracker: remove old batch metadata: 1529981756000 ms 18/06/26 10:55:58 INFO BlockManagerInfo: Removed input-0-1529981755800 on localhost:55667 in memory (size: 18.0 B, free: 2.4 GB) 18/06/26 10:55:58 INFO MemoryStore: Block input-0-1529981758400 stored as bytes in memory (estimated size 18.0 B, free 137.6 KB) 18/06/26 10:55:58 INFO BlockManagerInfo: Added input-0-1529981758400 in memory on localhost:55667 (size: 18.0 B, free: 2.4 GB) 18/06/26 10:55:58 WARN BlockManager: Block input-0-1529981758400 replicated to only 0 peer(s) instead of 1 peers 18/06/26 10:55:58 INFO BlockGenerator: Pushed block input-0-1529981758400 18/06/26 10:55:59 INFO JobScheduler: Added jobs for time 1529981759000 ms 18/06/26 10:55:59 INFO JobScheduler: Starting job streaming job 1529981759000 ms.0 from job set of time 1529981759000 ms 18/06/26 10:55:59 INFO SparkContext: Starting job: print at SparkWordCount.scala:55 18/06/26 10:55:59 INFO DAGScheduler: Registering RDD 47 (map at SparkWordCount.scala:51) 18/06/26 10:55:59 INFO DAGScheduler: Got job 23 (print at SparkWordCount.scala:55) with 1 output partitions 18/06/26 10:55:59 INFO DAGScheduler: Final stage: ResultStage 46 (print at SparkWordCount.scala:55) 18/06/26 10:55:59 INFO DAGScheduler: Parents of final stage: List(ShuffleMapStage 45) 18/06/26 10:55:59 INFO DAGScheduler: Missing parents: List(ShuffleMapStage 45) 18/06/26 10:55:59 INFO DAGScheduler: Submitting ShuffleMapStage 45 (MapPartitionsRDD[47] at map at SparkWordCount.scala:51), which has no missing parents 18/06/26 10:55:59 INFO MemoryStore: Block broadcast_25 stored as values in memory (estimated size 2.7 KB, free 140.3 KB) 18/06/26 10:55:59 INFO MemoryStore: Block broadcast_25_piece0 stored as bytes in memory (estimated size 1643.0 B, free 141.9 KB) 18/06/26 10:55:59 INFO BlockManagerInfo: Added broadcast_25_piece0 in memory on localhost:55667 (size: 1643.0 B, free: 2.4 GB) 18/06/26 10:55:59 INFO SparkContext: Created broadcast 25 from broadcast at DAGScheduler.scala:1015 18/06/26 10:55:59 INFO DAGScheduler: Submitting 1 missing tasks from ShuffleMapStage 45 (MapPartitionsRDD[47] at map at SparkWordCount.scala:51) 18/06/26 10:55:59 INFO TaskSchedulerImpl: Adding task set 45.0 with 1 tasks 18/06/26 10:55:59 INFO TaskSetManager: Starting task 0.0 in stage 45.0 (TID 25, localhost, partition 0,NODE_LOCAL, 2006 bytes) 18/06/26 10:55:59 INFO Executor: Running task 0.0 in stage 45.0 (TID 25) 18/06/26 10:55:59 INFO BlockManager: Found block input-0-1529981758400 locally 18/06/26 10:55:59 INFO Executor: Finished task 0.0 in stage 45.0 (TID 25). 1159 bytes result sent to driver 18/06/26 10:55:59 INFO TaskSetManager: Finished task 0.0 in stage 45.0 (TID 25) in 6 ms on localhost (1/1) 18/06/26 10:55:59 INFO TaskSchedulerImpl: Removed TaskSet 45.0, whose tasks have all completed, from pool 18/06/26 10:55:59 INFO DAGScheduler: ShuffleMapStage 45 (map at SparkWordCount.scala:51) finished in 0.006 s 18/06/26 10:55:59 INFO DAGScheduler: looking for newly runnable stages 18/06/26 10:55:59 INFO DAGScheduler: running: Set(ResultStage 0) 18/06/26 10:55:59 INFO DAGScheduler: waiting: Set(ResultStage 46) 18/06/26 10:55:59 INFO DAGScheduler: failed: Set() 18/06/26 10:55:59 INFO DAGScheduler: Submitting ResultStage 46 (ShuffledRDD[48] at reduceByKey at SparkWordCount.scala:52), which has no missing parents 18/06/26 10:55:59 INFO MemoryStore: Block broadcast_26 stored as values in memory (estimated size 2.6 KB, free 144.5 KB) 18/06/26 10:55:59 INFO MemoryStore: Block broadcast_26_piece0 stored as bytes in memory (estimated size 1631.0 B, free 146.1 KB) 18/06/26 10:55:59 INFO BlockManagerInfo: Added broadcast_26_piece0 in memory on localhost:55667 (size: 1631.0 B, free: 2.4 GB) 18/06/26 10:55:59 INFO SparkContext: Created broadcast 26 from broadcast at DAGScheduler.scala:1015 18/06/26 10:55:59 INFO DAGScheduler: Submitting 1 missing tasks from ResultStage 46 (ShuffledRDD[48] at reduceByKey at SparkWordCount.scala:52) 18/06/26 10:55:59 INFO TaskSchedulerImpl: Adding task set 46.0 with 1 tasks 18/06/26 10:55:59 INFO TaskSetManager: Starting task 0.0 in stage 46.0 (TID 26, localhost, partition 0,NODE_LOCAL, 1894 bytes) 18/06/26 10:55:59 INFO Executor: Running task 0.0 in stage 46.0 (TID 26) 18/06/26 10:55:59 INFO ShuffleBlockFetcherIterator: Getting 1 non-empty blocks out of 1 blocks 18/06/26 10:55:59 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 18/06/26 10:55:59 INFO Executor: Finished task 0.0 in stage 46.0 (TID 26). 1330 bytes result sent to driver 18/06/26 10:55:59 INFO TaskSetManager: Finished task 0.0 in stage 46.0 (TID 26) in 2 ms on localhost (1/1) 18/06/26 10:55:59 INFO TaskSchedulerImpl: Removed TaskSet 46.0, whose tasks have all completed, from pool 18/06/26 10:55:59 INFO DAGScheduler: ResultStage 46 (print at SparkWordCount.scala:55) finished in 0.002 s 18/06/26 10:55:59 INFO DAGScheduler: Job 23 finished: print at SparkWordCount.scala:55, took 0.018360 s 18/06/26 10:55:59 INFO SparkContext: Starting job: print at SparkWordCount.scala:55 18/06/26 10:55:59 INFO MapOutputTrackerMaster: Size of output statuses for shuffle 11 is 145 bytes 18/06/26 10:55:59 INFO DAGScheduler: Got job 24 (print at SparkWordCount.scala:55) with 1 output partitions 18/06/26 10:55:59 INFO DAGScheduler: Final stage: ResultStage 48 (print at SparkWordCount.scala:55) 18/06/26 10:55:59 INFO DAGScheduler: Parents of final stage: List(ShuffleMapStage 47) 18/06/26 10:55:59 INFO DAGScheduler: Missing parents: List() 18/06/26 10:55:59 INFO DAGScheduler: Submitting ResultStage 48 (ShuffledRDD[48] at reduceByKey at SparkWordCount.scala:52), which has no missing parents 18/06/26 10:55:59 INFO MemoryStore: Block broadcast_27 stored as values in memory (estimated size 2.6 KB, free 148.7 KB) 18/06/26 10:55:59 INFO MemoryStore: Block broadcast_27_piece0 stored as bytes in memory (estimated size 1633.0 B, free 150.3 KB) 18/06/26 10:55:59 INFO BlockManagerInfo: Added broadcast_27_piece0 in memory on localhost:55667 (size: 1633.0 B, free: 2.4 GB) 18/06/26 10:55:59 INFO SparkContext: Created broadcast 27 from broadcast at DAGScheduler.scala:1015 18/06/26 10:55:59 INFO DAGScheduler: Submitting 1 missing tasks from ResultStage 48 (ShuffledRDD[48] at reduceByKey at SparkWordCount.scala:52) 18/06/26 10:55:59 INFO BlockManagerInfo: Removed broadcast_7_piece0 on localhost:55667 in memory (size: 1638.0 B, free: 2.4 GB) 18/06/26 10:55:59 INFO TaskSchedulerImpl: Adding task set 48.0 with 1 tasks 18/06/26 10:55:59 INFO ContextCleaner: Cleaned accumulator 8 18/06/26 10:55:59 INFO TaskSetManager: Starting task 0.0 in stage 48.0 (TID 27, localhost, partition 1,PROCESS_LOCAL, 1894 bytes) 18/06/26 10:55:59 INFO Executor: Running task 0.0 in stage 48.0 (TID 27) 18/06/26 10:55:59 INFO ShuffleBlockFetcherIterator: Getting 0 non-empty blocks out of 1 blocks 18/06/26 10:55:59 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 18/06/26 10:55:59 INFO Executor: Finished task 0.0 in stage 48.0 (TID 27). 1161 bytes result sent to driver 18/06/26 10:55:59 INFO TaskSetManager: Finished task 0.0 in stage 48.0 (TID 27) in 2 ms on localhost (1/1) 18/06/26 10:55:59 INFO TaskSchedulerImpl: Removed TaskSet 48.0, whose tasks have all completed, from pool 18/06/26 10:55:59 INFO ContextCleaner: Cleaned shuffle 3 18/06/26 10:55:59 INFO DAGScheduler: ResultStage 48 (print at SparkWordCount.scala:55) finished in 0.003 s 18/06/26 10:55:59 INFO DAGScheduler: Job 24 finished: print at SparkWordCount.scala:55, took 0.016824 s 18/06/26 10:55:59 INFO BlockManagerInfo: Removed broadcast_6_piece0 on localhost:55667 in memory (size: 1634.0 B, free: 2.4 GB) 18/06/26 10:55:59 INFO JobScheduler: Finished job streaming job 1529981759000 ms.0 from job set of time 1529981759000 ms 18/06/26 10:55:59 INFO JobScheduler: Total delay: 0.054 s for time 1529981759000 ms (execution: 0.044 s) ------------------------------------------- Time: 1529981759000 ms ------------------------------------------- (hello,1) (world,1) 18/06/26 10:55:59 INFO ShuffledRDD: Removing RDD 44 from persistence list 18/06/26 10:55:59 INFO ContextCleaner: Cleaned accumulator 7 18/06/26 10:55:59 INFO BlockManager: Removing RDD 44 18/06/26 10:55:59 INFO MapPartitionsRDD: Removing RDD 43 from persistence list 18/06/26 10:55:59 INFO BlockManagerInfo: Removed broadcast_5_piece0 on localhost:55667 in memory (size: 1634.0 B, free: 2.4 GB) 18/06/26 10:55:59 INFO BlockManager: Removing RDD 43 18/06/26 10:55:59 INFO ContextCleaner: Cleaned accumulator 6 18/06/26 10:55:59 INFO MapPartitionsRDD: Removing RDD 42 from persistence list 18/06/26 10:55:59 INFO ContextCleaner: Cleaned shuffle 2 18/06/26 10:55:59 INFO BlockManager: Removing RDD 42 18/06/26 10:55:59 INFO BlockRDD: Removing RDD 41 from persistence list 18/06/26 10:55:59 INFO BlockManager: Removing RDD 41 18/06/26 10:55:59 INFO BlockManagerInfo: Removed broadcast_4_piece0 on localhost:55667 in memory (size: 1632.0 B, free: 2.4 GB) 18/06/26 10:55:59 INFO SocketInputDStream: Removing blocks of RDD BlockRDD[41] at socketTextStream at SparkWordCount.scala:46 of time 1529981759000 ms 18/06/26 10:55:59 INFO ContextCleaner: Cleaned accumulator 5 18/06/26 10:55:59 INFO ReceivedBlockTracker: Deleting batches ArrayBuffer(1529981757000 ms) 18/06/26 10:55:59 INFO InputInfoTracker: remove old batch metadata: 1529981757000 ms 18/06/26 10:55:59 INFO BlockManagerInfo: Removed input-0-1529981757000 on localhost:55667 in memory (size: 18.0 B, free: 2.4 GB) 18/06/26 10:55:59 INFO BlockManagerInfo: Removed broadcast_3_piece0 on localhost:55667 in memory (size: 1632.0 B, free: 2.4 GB) 18/06/26 10:55:59 INFO ContextCleaner: Cleaned accumulator 4 18/06/26 10:55:59 INFO ContextCleaner: Cleaned shuffle 1 18/06/26 10:55:59 INFO ContextCleaner: Cleaned shuffle 0 18/06/26 10:55:59 INFO BlockManagerInfo: Removed broadcast_26_piece0 on localhost:55667 in memory (size: 1631.0 B, free: 2.4 GB) 18/06/26 10:55:59 INFO ContextCleaner: Cleaned accumulator 27 18/06/26 10:55:59 INFO BlockManagerInfo: Removed broadcast_25_piece0 on localhost:55667 in memory (size: 1643.0 B, free: 2.4 GB) 18/06/26 10:55:59 INFO ContextCleaner: Cleaned accumulator 26 18/06/26 10:55:59 INFO BlockManagerInfo: Removed broadcast_24_piece0 on localhost:55667 in memory (size: 1633.0 B, free: 2.4 GB) 18/06/26 10:55:59 INFO ContextCleaner: Cleaned accumulator 25 18/06/26 10:55:59 INFO BlockManagerInfo: Removed broadcast_23_piece0 on localhost:55667 in memory (size: 1633.0 B, free: 2.4 GB) 18/06/26 10:55:59 INFO ContextCleaner: Cleaned accumulator 24 18/06/26 10:55:59 INFO BlockManagerInfo: Removed broadcast_22_piece0 on localhost:55667 in memory (size: 1643.0 B, free: 2.4 GB) 18/06/26 10:55:59 INFO ContextCleaner: Cleaned accumulator 23 18/06/26 10:55:59 INFO BlockManagerInfo: Removed broadcast_21_piece0 on localhost:55667 in memory (size: 1631.0 B, free: 2.4 GB) 18/06/26 10:55:59 INFO ContextCleaner: Cleaned accumulator 22 18/06/26 10:55:59 INFO BlockManagerInfo: Removed broadcast_20_piece0 on localhost:55667 in memory (size: 1633.0 B, free: 2.4 GB) 18/06/26 10:55:59 INFO ContextCleaner: Cleaned accumulator 21 18/06/26 10:55:59 INFO BlockManagerInfo: Removed broadcast_19_piece0 on localhost:55667 in memory (size: 1643.0 B, free: 2.4 GB) 18/06/26 10:55:59 INFO ContextCleaner: Cleaned accumulator 20 18/06/26 10:55:59 INFO ContextCleaner: Cleaned shuffle 9 18/06/26 10:55:59 INFO BlockManagerInfo: Removed broadcast_18_piece0 on localhost:55667 in memory (size: 1637.0 B, free: 2.4 GB) 18/06/26 10:55:59 INFO ContextCleaner: Cleaned accumulator 19 18/06/26 10:55:59 INFO BlockManagerInfo: Removed broadcast_17_piece0 on localhost:55667 in memory (size: 1637.0 B, free: 2.4 GB) 18/06/26 10:55:59 INFO ContextCleaner: Cleaned accumulator 18 18/06/26 10:55:59 INFO ContextCleaner: Cleaned shuffle 8 18/06/26 10:55:59 INFO BlockManagerInfo: Removed broadcast_16_piece0 on localhost:55667 in memory (size: 1637.0 B, free: 2.4 GB) 18/06/26 10:55:59 INFO ContextCleaner: Cleaned accumulator 17 18/06/26 10:55:59 INFO BlockManagerInfo: Removed broadcast_15_piece0 on localhost:55667 in memory (size: 1637.0 B, free: 2.4 GB) 18/06/26 10:55:59 INFO ContextCleaner: Cleaned accumulator 16 18/06/26 10:55:59 INFO ContextCleaner: Cleaned shuffle 7 18/06/26 10:55:59 INFO BlockManagerInfo: Removed broadcast_14_piece0 on localhost:55667 in memory (size: 1637.0 B, free: 2.4 GB) 18/06/26 10:55:59 INFO ContextCleaner: Cleaned accumulator 15 18/06/26 10:55:59 INFO BlockManagerInfo: Removed broadcast_13_piece0 on localhost:55667 in memory (size: 1637.0 B, free: 2.4 GB) 18/06/26 10:55:59 INFO ContextCleaner: Cleaned accumulator 14 18/06/26 10:55:59 INFO ContextCleaner: Cleaned shuffle 6 18/06/26 10:55:59 INFO BlockManagerInfo: Removed broadcast_12_piece0 on localhost:55667 in memory (size: 1636.0 B, free: 2.4 GB) 18/06/26 10:55:59 INFO ContextCleaner: Cleaned accumulator 13 18/06/26 10:55:59 INFO BlockManagerInfo: Removed broadcast_11_piece0 on localhost:55667 in memory (size: 1636.0 B, free: 2.4 GB) 18/06/26 10:55:59 INFO ContextCleaner: Cleaned accumulator 12 18/06/26 10:55:59 INFO ContextCleaner: Cleaned shuffle 5 18/06/26 10:55:59 INFO BlockManagerInfo: Removed broadcast_10_piece0 on localhost:55667 in memory (size: 1637.0 B, free: 2.4 GB) 18/06/26 10:55:59 INFO ContextCleaner: Cleaned accumulator 11 18/06/26 10:55:59 INFO BlockManagerInfo: Removed broadcast_9_piece0 on localhost:55667 in memory (size: 1637.0 B, free: 2.4 GB) 18/06/26 10:55:59 INFO ContextCleaner: Cleaned accumulator 10 18/06/26 10:55:59 INFO ContextCleaner: Cleaned shuffle 4 18/06/26 10:55:59 INFO BlockManagerInfo: Removed broadcast_8_piece0 on localhost:55667 in memory (size: 1638.0 B, free: 2.4 GB) 18/06/26 10:55:59 INFO ContextCleaner: Cleaned accumulator 9 18/06/26 10:56:00 INFO JobScheduler: Added jobs for time 1529981760000 ms 18/06/26 10:56:00 INFO JobScheduler: Starting job streaming job 1529981760000 ms.0 from job set of time 1529981760000 ms 18/06/26 10:56:00 INFO SparkContext: Starting job: print at SparkWordCount.scala:55 18/06/26 10:56:00 INFO DAGScheduler: Registering RDD 51 (map at SparkWordCount.scala:51) 18/06/26 10:56:00 INFO DAGScheduler: Got job 25 (print at SparkWordCount.scala:55) with 1 output partitions 18/06/26 10:56:00 INFO DAGScheduler: Final stage: ResultStage 50 (print at SparkWordCount.scala:55) 18/06/26 10:56:00 INFO DAGScheduler: Parents of final stage: List(ShuffleMapStage 49) 18/06/26 10:56:00 INFO DAGScheduler: Missing parents: List() 18/06/26 10:56:00 INFO DAGScheduler: Submitting ResultStage 50 (ShuffledRDD[52] at reduceByKey at SparkWordCount.scala:52), which has no missing parents 18/06/26 10:56:00 INFO MemoryStore: Block broadcast_28 stored as values in memory (estimated size 2.6 KB, free 51.9 KB) 18/06/26 10:56:00 INFO MemoryStore: Block broadcast_28_piece0 stored as bytes in memory (estimated size 1637.0 B, free 53.5 KB) 18/06/26 10:56:00 INFO BlockManagerInfo: Added broadcast_28_piece0 in memory on localhost:55667 (size: 1637.0 B, free: 2.4 GB) 18/06/26 10:56:00 INFO SparkContext: Created broadcast 28 from broadcast at DAGScheduler.scala:1015 18/06/26 10:56:00 INFO DAGScheduler: Submitting 1 missing tasks from ResultStage 50 (ShuffledRDD[52] at reduceByKey at SparkWordCount.scala:52) 18/06/26 10:56:00 INFO TaskSchedulerImpl: Adding task set 50.0 with 1 tasks 18/06/26 10:56:00 INFO TaskSetManager: Starting task 0.0 in stage 50.0 (TID 28, localhost, partition 0,PROCESS_LOCAL, 1894 bytes) 18/06/26 10:56:00 INFO Executor: Running task 0.0 in stage 50.0 (TID 28) 18/06/26 10:56:00 INFO ShuffleBlockFetcherIterator: Getting 0 non-empty blocks out of 0 blocks 18/06/26 10:56:00 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 18/06/26 10:56:00 INFO Executor: Finished task 0.0 in stage 50.0 (TID 28). 1161 bytes result sent to driver 18/06/26 10:56:00 INFO TaskSetManager: Finished task 0.0 in stage 50.0 (TID 28) in 4 ms on localhost (1/1) 18/06/26 10:56:00 INFO TaskSchedulerImpl: Removed TaskSet 50.0, whose tasks have all completed, from pool 18/06/26 10:56:00 INFO DAGScheduler: ResultStage 50 (print at SparkWordCount.scala:55) finished in 0.004 s 18/06/26 10:56:00 INFO DAGScheduler: Job 25 finished: print at SparkWordCount.scala:55, took 0.009608 s 18/06/26 10:56:00 INFO SparkContext: Starting job: print at SparkWordCount.scala:55 18/06/26 10:56:00 INFO MapOutputTrackerMaster: Size of output statuses for shuffle 12 is 82 bytes 18/06/26 10:56:00 INFO DAGScheduler: Got job 26 (print at SparkWordCount.scala:55) with 1 output partitions 18/06/26 10:56:00 INFO DAGScheduler: Final stage: ResultStage 52 (print at SparkWordCount.scala:55) 18/06/26 10:56:00 INFO DAGScheduler: Parents of final stage: List(ShuffleMapStage 51) 18/06/26 10:56:00 INFO DAGScheduler: Missing parents: List() 18/06/26 10:56:00 INFO DAGScheduler: Submitting ResultStage 52 (ShuffledRDD[52] at reduceByKey at SparkWordCount.scala:52), which has no missing parents 18/06/26 10:56:00 INFO MemoryStore: Block broadcast_29 stored as values in memory (estimated size 2.6 KB, free 56.1 KB) 18/06/26 10:56:00 INFO MemoryStore: Block broadcast_29_piece0 stored as bytes in memory (estimated size 1637.0 B, free 57.7 KB) 18/06/26 10:56:00 INFO BlockManagerInfo: Added broadcast_29_piece0 in memory on localhost:55667 (size: 1637.0 B, free: 2.4 GB) 18/06/26 10:56:00 INFO SparkContext: Created broadcast 29 from broadcast at DAGScheduler.scala:1015 18/06/26 10:56:00 INFO DAGScheduler: Submitting 1 missing tasks from ResultStage 52 (ShuffledRDD[52] at reduceByKey at SparkWordCount.scala:52) 18/06/26 10:56:00 INFO TaskSchedulerImpl: Adding task set 52.0 with 1 tasks 18/06/26 10:56:00 INFO TaskSetManager: Starting task 0.0 in stage 52.0 (TID 29, localhost, partition 1,PROCESS_LOCAL, 1894 bytes) 18/06/26 10:56:00 INFO Executor: Running task 0.0 in stage 52.0 (TID 29) 18/06/26 10:56:00 INFO ShuffleBlockFetcherIterator: Getting 0 non-empty blocks out of 0 blocks 18/06/26 10:56:00 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 18/06/26 10:56:00 INFO Executor: Finished task 0.0 in stage 52.0 (TID 29). 1161 bytes result sent to driver 18/06/26 10:56:00 INFO TaskSetManager: Finished task 0.0 in stage 52.0 (TID 29) in 1 ms on localhost (1/1) 18/06/26 10:56:00 INFO TaskSchedulerImpl: Removed TaskSet 52.0, whose tasks have all completed, from pool 18/06/26 10:56:00 INFO DAGScheduler: ResultStage 52 (print at SparkWordCount.scala:55) finished in 0.002 s 18/06/26 10:56:00 INFO DAGScheduler: Job 26 finished: print at SparkWordCount.scala:55, took 0.005037 s ------------------------------------------- Time: 1529981760000 ms ------------------------------------------- 18/06/26 10:56:00 INFO JobScheduler: Finished job streaming job 1529981760000 ms.0 from job set of time 1529981760000 ms 18/06/26 10:56:00 INFO ShuffledRDD: Removing RDD 48 from persistence list 18/06/26 10:56:00 INFO JobScheduler: Total delay: 0.053 s for time 1529981760000 ms (execution: 0.030 s) 18/06/26 10:56:00 INFO BlockManager: Removing RDD 48 18/06/26 10:56:00 INFO MapPartitionsRDD: Removing RDD 47 from persistence list 18/06/26 10:56:00 INFO BlockManager: Removing RDD 47 18/06/26 10:56:00 INFO MapPartitionsRDD: Removing RDD 46 from persistence list 18/06/26 10:56:00 INFO BlockManager: Removing RDD 46 18/06/26 10:56:00 INFO BlockRDD: Removing RDD 45 from persistence list 18/06/26 10:56:00 INFO BlockManager: Removing RDD 45 18/06/26 10:56:00 INFO SocketInputDStream: Removing blocks of RDD BlockRDD[45] at socketTextStream at SparkWordCount.scala:46 of time 1529981760000 ms 18/06/26 10:56:00 INFO ReceivedBlockTracker: Deleting batches ArrayBuffer(1529981758000 ms) 18/06/26 10:56:00 INFO InputInfoTracker: remove old batch metadata: 1529981758000 ms 18/06/26 10:56:00 INFO BlockManagerInfo: Removed input-0-1529981758400 on localhost:55667 in memory (size: 18.0 B, free: 2.4 GB) 18/06/26 10:56:01 INFO JobScheduler: Added jobs for time 1529981761000 ms 18/06/26 10:56:01 INFO JobScheduler: Starting job streaming job 1529981761000 ms.0 from job set of time 1529981761000 ms 18/06/26 10:56:01 INFO SparkContext: Starting job: print at SparkWordCount.scala:55 18/06/26 10:56:01 INFO DAGScheduler: Registering RDD 55 (map at SparkWordCount.scala:51) 18/06/26 10:56:01 INFO DAGScheduler: Got job 27 (print at SparkWordCount.scala:55) with 1 output partitions 18/06/26 10:56:01 INFO DAGScheduler: Final stage: ResultStage 54 (print at SparkWordCount.scala:55) 18/06/26 10:56:01 INFO DAGScheduler: Parents of final stage: List(ShuffleMapStage 53) 18/06/26 10:56:01 INFO DAGScheduler: Missing parents: List() 18/06/26 10:56:01 INFO DAGScheduler: Submitting ResultStage 54 (ShuffledRDD[56] at reduceByKey at SparkWordCount.scala:52), which has no missing parents 18/06/26 10:56:01 INFO MemoryStore: Block broadcast_30 stored as values in memory (estimated size 2.6 KB, free 60.3 KB) 18/06/26 10:56:01 INFO MemoryStore: Block broadcast_30_piece0 stored as bytes in memory (estimated size 1636.0 B, free 61.8 KB) 18/06/26 10:56:01 INFO BlockManagerInfo: Added broadcast_30_piece0 in memory on localhost:55667 (size: 1636.0 B, free: 2.4 GB) 18/06/26 10:56:01 INFO SparkContext: Created broadcast 30 from broadcast at DAGScheduler.scala:1015 18/06/26 10:56:01 INFO DAGScheduler: Submitting 1 missing tasks from ResultStage 54 (ShuffledRDD[56] at reduceByKey at SparkWordCount.scala:52) 18/06/26 10:56:01 INFO TaskSchedulerImpl: Adding task set 54.0 with 1 tasks 18/06/26 10:56:01 INFO TaskSetManager: Starting task 0.0 in stage 54.0 (TID 30, localhost, partition 0,PROCESS_LOCAL, 1894 bytes) 18/06/26 10:56:01 INFO Executor: Running task 0.0 in stage 54.0 (TID 30) 18/06/26 10:56:01 INFO ShuffleBlockFetcherIterator: Getting 0 non-empty blocks out of 0 blocks 18/06/26 10:56:01 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 18/06/26 10:56:01 INFO Executor: Finished task 0.0 in stage 54.0 (TID 30). 1161 bytes result sent to driver 18/06/26 10:56:01 INFO TaskSetManager: Finished task 0.0 in stage 54.0 (TID 30) in 2 ms on localhost (1/1) 18/06/26 10:56:01 INFO TaskSchedulerImpl: Removed TaskSet 54.0, whose tasks have all completed, from pool 18/06/26 10:56:01 INFO DAGScheduler: ResultStage 54 (print at SparkWordCount.scala:55) finished in 0.002 s 18/06/26 10:56:01 INFO DAGScheduler: Job 27 finished: print at SparkWordCount.scala:55, took 0.006114 s 18/06/26 10:56:01 INFO SparkContext: Starting job: print at SparkWordCount.scala:55 18/06/26 10:56:01 INFO MapOutputTrackerMaster: Size of output statuses for shuffle 13 is 82 bytes 18/06/26 10:56:01 INFO DAGScheduler: Got job 28 (print at SparkWordCount.scala:55) with 1 output partitions 18/06/26 10:56:01 INFO DAGScheduler: Final stage: ResultStage 56 (print at SparkWordCount.scala:55) 18/06/26 10:56:01 INFO DAGScheduler: Parents of final stage: List(ShuffleMapStage 55) 18/06/26 10:56:01 INFO DAGScheduler: Missing parents: List() 18/06/26 10:56:01 INFO DAGScheduler: Submitting ResultStage 56 (ShuffledRDD[56] at reduceByKey at SparkWordCount.scala:52), which has no missing parents 18/06/26 10:56:01 INFO MemoryStore: Block broadcast_31 stored as values in memory (estimated size 2.6 KB, free 64.4 KB) 18/06/26 10:56:01 INFO MemoryStore: Block broadcast_31_piece0 stored as bytes in memory (estimated size 1636.0 B, free 66.0 KB) 18/06/26 10:56:01 INFO BlockManagerInfo: Added broadcast_31_piece0 in memory on localhost:55667 (size: 1636.0 B, free: 2.4 GB) 18/06/26 10:56:01 INFO SparkContext: Created broadcast 31 from broadcast at DAGScheduler.scala:1015 18/06/26 10:56:01 INFO DAGScheduler: Submitting 1 missing tasks from ResultStage 56 (ShuffledRDD[56] at reduceByKey at SparkWordCount.scala:52) 18/06/26 10:56:01 INFO TaskSchedulerImpl: Adding task set 56.0 with 1 tasks 18/06/26 10:56:01 INFO TaskSetManager: Starting task 0.0 in stage 56.0 (TID 31, localhost, partition 1,PROCESS_LOCAL, 1894 bytes) 18/06/26 10:56:01 INFO Executor: Running task 0.0 in stage 56.0 (TID 31) 18/06/26 10:56:01 INFO ShuffleBlockFetcherIterator: Getting 0 non-empty blocks out of 0 blocks 18/06/26 10:56:01 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 18/06/26 10:56:01 INFO Executor: Finished task 0.0 in stage 56.0 (TID 31). 1161 bytes result sent to driver 18/06/26 10:56:01 INFO TaskSetManager: Finished task 0.0 in stage 56.0 (TID 31) in 2 ms on localhost (1/1) 18/06/26 10:56:01 INFO TaskSchedulerImpl: Removed TaskSet 56.0, whose tasks have all completed, from pool 18/06/26 10:56:01 INFO DAGScheduler: ResultStage 56 (print at SparkWordCount.scala:55) finished in 0.002 s 18/06/26 10:56:01 INFO DAGScheduler: Job 28 finished: print at SparkWordCount.scala:55, took 0.006372 s ------------------------------------------- Time: 1529981761000 ms ------------------------------------------- 18/06/26 10:56:01 INFO JobScheduler: Finished job streaming job 1529981761000 ms.0 from job set of time 1529981761000 ms 18/06/26 10:56:01 INFO JobScheduler: Total delay: 0.024 s for time 1529981761000 ms (execution: 0.018 s) 18/06/26 10:56:01 INFO ShuffledRDD: Removing RDD 52 from persistence list 18/06/26 10:56:01 INFO BlockManager: Removing RDD 52 18/06/26 10:56:01 INFO MapPartitionsRDD: Removing RDD 51 from persistence list 18/06/26 10:56:01 INFO BlockManager: Removing RDD 51 18/06/26 10:56:01 INFO MapPartitionsRDD: Removing RDD 50 from persistence list 18/06/26 10:56:01 INFO BlockRDD: Removing RDD 49 from persistence list 18/06/26 10:56:01 INFO SocketInputDStream: Removing blocks of RDD BlockRDD[49] at socketTextStream at SparkWordCount.scala:46 of time 1529981761000 ms 18/06/26 10:56:01 INFO ReceivedBlockTracker: Deleting batches ArrayBuffer(1529981759000 ms) 18/06/26 10:56:01 INFO InputInfoTracker: remove old batch metadata: 1529981759000 ms 18/06/26 10:56:01 INFO BlockManager: Removing RDD 49 18/06/26 10:56:01 INFO BlockManager: Removing RDD 50