Repository navigation
Expand file tree
/
Copy pathStreamApp.java
More file actions
119 lines (88 loc) · 4.13 KB
/
Copy pathStreamApp.java
File metadata and controls
119 lines (88 loc) · 4.13 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
package com.example;
import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.function.VoidFunction2;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Row;
import org.apache.spark.sql.SQLContext;
import org.apache.spark.sql.api.java.UDF1;
import org.apache.spark.sql.functions;
import org.apache.spark.sql.types.DataTypes;
import org.apache.spark.storage.StorageLevel;
import org.apache.spark.streaming.Durations;
import org.apache.spark.streaming.Time;
import org.apache.spark.streaming.api.java.JavaReceiverInputDStream;
import org.apache.spark.streaming.api.java.JavaStreamingContext;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/*
* 1. Create a DStream based on socket stream source
* 2. Specify persistence level to MEMORY_ONLY
* 3. Save the raw stream to file system
* 4. Find frequency distribution of media using a window operation
* */
public class StreamApp {
private static Logger logger = LoggerFactory.getLogger(StreamApp.class);
public static void main(String[] args) throws Exception {
final String rawOutput = "output/stream-raw";
SparkConf sparkConf = new SparkConf().setAppName("StreamApp");
sparkConf.setIfMissing("spark.master", "local[*]");
JavaSparkContext sc = new JavaSparkContext(sparkConf);
final SQLContext sqlContext = new SQLContext(sc);
JavaStreamingContext ssc = new JavaStreamingContext(sc, Durations.seconds(5));
Integer port = 9999;
JavaReceiverInputDStream<String> lines = ssc.socketTextStream("localhost"
, port, StorageLevel.MEMORY_ONLY());
lines.print(10);
//Saving the raw DStream to file system
lines.foreachRDD(new VoidFunction2<JavaRDD<String>, Time>() {
public void call(JavaRDD<String> stringJavaRDD, Time time) throws Exception {
stringJavaRDD.coalesce(1).saveAsTextFile(rawOutput + "/"
+ time.toString().split(" ")[0]);
}
});
sc.getConf().set("spark.sql.shuffle.partitions", "1");
sqlContext.udf().register("findMedia", new UDF1<String, String>() {
public String call(String value){
String media = null;
if(value.startsWith("http://twitter.com/download/")){
String[] tokens = value.split("/");
media = tokens[tokens.length - 1];
}else if(value.startsWith("https://mobile.twitter.com")){
media = "twitter-mobile";
}else if(value.startsWith("http://twitter.com")
|| value.startsWith("http://www.twitter.com")){
media = "twitter.com";
}else{
media = "other";
}
return media;
}
}, DataTypes.StringType);
lines
.window(Durations.seconds(30), Durations.seconds(10))
.foreachRDD(new VoidFunction2<JavaRDD<String>, Time>() {
public void call(JavaRDD<String> stringJavaRDD, Time time)
throws Exception {
logger.info("No of records: " + stringJavaRDD.count());
if(!stringJavaRDD.isEmpty()) {
Dataset<Row> df = sqlContext
.read()
.json(stringJavaRDD)
.filter(functions.col("id").isNotNull())
.select(functions.regexp_extract(functions.col("source"), "href=\\\"(.*?)\\\"", 1).alias("href"))
.withColumn("href", functions.lower(functions.col("href")))
.withColumn("media", functions.callUDF("findMedia", functions.col("href")))
.groupBy("media")
.count();
df.show(100, false);
}
}
});
ssc.start();
ssc.awaitTermination();
ssc.stop();
ssc.close();
}
}