# Spark Streaming 实时词频统计
# 本示例演示如何使用Spark Streaming处理实时数据流

from pyspark import SparkContext
from pyspark.streaming import StreamingContext

# 初始化SparkContext
sc = SparkContext("local[2]", "NetworkWordCount")
ssc = StreamingContext(sc, 1)

# 创建DStream从TCP源接收数据
lines = ssc.socketTextStream("localhost", 9999)

# 分割行成单词
words = lines.flatMap(lambda line: line.split(" "))

# 将每个单词映射为(word, 1)
pairs = words.map(lambda word: (word, 1))

# 按单词聚合计数
wordCounts = pairs.reduceByKey(lambda x, y: x + y)

# 打印结果
wordCounts.pprint()

# 启动流计算
ssc.start()
ssc.awaitTermination()