「Spark 手把手 - 快速上手營」適合對於想了解如何應用Apache Spark/Hadoop/Kafka來建構大數據資料平台或是處理分析大數據的工程師或工程主管來參加。參加本次活動，參與者都將會有機會體驗 Apache Spark Family 中各項知名的專案，包括 Spark、Spark-SQL、Spark-Streaming、Kafka、HDFS、Parquet、Zeppelin 以及 Java, Scala等工具與技術。 整個活動設計區分為兩大階段: 階段一 Spark應用於批次型資料處理 (Spark for Batch process): 約4週, 每週2小時 階段二 Spark應用於串流式事件處理 (Spark for Streaming process): 約2週, 每週2小時

1. 1

2. 2

3. 2

4. 2

5. 2

6. 2

7. 2

8. 2

9. 2

10. 2

11. 3 . 1

12. 3 . 2

13. 3 . 2

14. 3 . 2

15. 3 . 2

16. 3 . 2

17. 3 . 2

18. 3 . 3

19. 3 . 4

20. 3 . 4

21. 3 . 4

22. 3 . 4

23. 3 . 4

24. 3 . 5

25. 3 . 5

26. 3 . 5

27. 3 . 5

28. 3 . 6

29. 3 . 6

30. 3 . 7

31. 3 . 7

32. 3 . 8

33. 3 . 8

34. 3 . 9

35. 3 . 9

36. 3 . 10

37. 3 . 10

38. 3 . 11

39. 3 . 11

40. 3 . 12

41. 3 . 12

42. 3 . 13

43. 3 . 13

44. 3 . 14

45. 3 . 14

46. 4 . 1

47. 4 . 2

48. 4 . 2

49. 4 . 3

50. 4 . 3

51. 4 . 3

52. 4 . 3

53. 4 . 4

54. 4 . 4

55. 4 . 4

56. 4 . 4

57. 5 . 1

58. 5 . 2

59. 5 . 2

60. 5 . 3

61. 5 . 4

62. 5 . 5

63. 5 . 6

64. 5 . 7

65. 5 . 8

66. 5 . 9

67. 5 . 10

68. 5 . 11

69. 5 . 12

70. 5 . 13

71. 6 . 1

72. 6 . 2

73. 6 . 3

74. 6 . 4

75. 6 . 5

76. 6 . 6

77. 6 . 7

78. 6 . 8

79. 7 . 1

80. 7 . 2

81. 7 . 3

82. 7 . 4

83. 7 . 5

84. 7 . 6

85. val distData = sc.parallelize(Seq("eighty20", "spark", "traing", "hello", "world"))
val result_count = distData.count()
println("Count result is: " + result_count)
7 . 7

86. 8 . 1

87. 8 . 2

88. val distData = sc.parallelize(Seq("eighty20", "spark", "traing", "hello", "world"))
val result_count = distData.count()
println("Count result is: " + result_count)
8 . 3

89. 9 . 1

90. 9 . 2

91. 9 . 3

92. 10 . 1

93. 10 . 2

94. 10 . 2

95. 10 . 2

96. 10 . 2

97. 10 . 2

98. 10 . 2

99. 10 . 3

100. 10 . 4

101. 10 . 5

102. 10 . 6

103. 11 . 1

104. 11 . 2

105. 11 . 2

106. 11 . 2

107. 11 . 2

108. 11 . 2

109. 11 . 2

110. 11 . 2

111. 12 . 1

112. 12 . 2

113. 12 . 3

114. 12 . 4

115. 12 . 5

116. 12 . 6

117. 12 . 7

118. spark-submit
--class cc.eighty20.spark.s01.sc_00_helloworld
--master local
e2spks01-0.0.1.jar
12 . 8

119. spark-submit
--class cc.eighty20.spark.s01.sc_00_helloworld
--master spark://192.168.0.2:7077
e2spks01-0.0.1.jar
12 . 9

120. 13 . 1

121. 13 . 2

122. 13 . 3

123. 13 . 4

124. 13 . 5

125. 13 . 6

126. 13 . 7

127. spark-submit
--class cc.eighty20.spark.s01.sc_00_helloworld
--master local
e2spks01-0.0.1.jar
13 . 8

128. spark-submit
--class cc.eighty20.spark.s01.sc_00_helloworld
--master spark://192.168.0.2:7077
e2spks01-0.0.1.jar
13 . 9

129. 14 . 1

130. 14 . 2

131. 14 . 3

132. 14 . 4

133. 14 . 4

134. 14 . 4

135. 14 . 4

136. 14 . 4

137. 14 . 4

138. 14 . 4

139. 14 . 4

140. 14 . 5

141. 14 . 5

142. 14 . 5

143. 14 . 5

144. 14 . 6

145. 14 . 6

146. 14 . 6

147. 14 . 6

148. 14 . 6

149. 14 . 7

150. 14 . 8

151. 14 . 8

152. 14 . 8

153. 14 . 8

154. 14 . 8

155. 14 . 9

156. 14 . 10

157. 14 . 11

158. 14 . 12

159. 14 . 13

160. 14 . 14

161. 14 . 15

162. 14 . 16

163. 14 . 17

164. 14 . 17

165. 14 . 17

166. 14 . 17

167. 14 . 17

168. 14 . 17

169. 14 . 17

170. 14 . 17

171. 14 . 18

172. 14 . 18

173. 14 . 18

174. 14 . 18

175. 14 . 18

176. 14 . 18

177. 14 . 19

178. 14 . 19

179. 14 . 19

180. 14 . 19

181. 14 . 19

182. 14 . 19

183. 14 . 20

184. 14 . 20

185. 14 . 20

186. 14 . 20

187. 14 . 20

188. 14 . 21

189. 14 . 21

190. 14 . 22

191. 14 . 22

192. 14 . 22

193. 14 . 22

194. 14 . 22

195. 14 . 22

196. 14 . 22

197. 14 . 22

198. 14 . 22

199. 14 . 22

200. 14 . 23

201. 14 . 24

202. 14 . 24

203. 14 . 24

204. 14 . 24

205. 14 . 24

206. 14 . 25

207. 14 . 26

208. 14 . 26

209. 14 . 26

210. 14 . 26

211. 14 . 26

212. 14 . 26

213. val r00 = sc.parallelize(0 to 9)
val r01 = sc.parallelize(0 to 90 by 10)
val r10 = r00.cartesian(r01)
val r11 = r00.map(n => (n, n))
val r12 = r00.zip(r01)
val r13 = r01.keyBy(_ / 20)
val r20 = Seq(r11, r12, r13).foldLeft(r10)(_ union _)
14 . 27

214. val r00 = sc.parallelize(0 to 9)
val r01 = sc.parallelize(0 to 90 by 10)
val r10 = r00.cartesian(r01)
val r11 = r00.map(n => (n, n))
val r12 = r00.zip(r01)
val r13 = r01.keyBy(_ / 20)
val r20 = Seq(r11, r12, r13).foldLeft(r10)(_ union _)
14 . 27

215. val r00 = sc.parallelize(0 to 9)
val r01 = sc.parallelize(0 to 90 by 10)
val r10 = r00.cartesian(r01)
val r11 = r00.map(n => (n, n))
val r12 = r00.zip(r01)
val r13 = r01.keyBy(_ / 20)
val r20 = Seq(r11, r12, r13).foldLeft(r10)(_ union _)
14 . 27

216. 14 . 28

217. 14 . 28

218. 14 . 28

219. 14 . 28

220. 14 . 28

221. 15

222. 16 . 1

223. package cc.eighty20.spark.s01;
import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
public class sc_01_anatomy_driver {
public static void main(String[] args) {
String masterURL = "local[*]"; // (1)
SparkConf conf = new SparkConf() // (2)
.setAppName("sc_01_anatomy_driver")
.setMaster(masterURL);
JavaSparkContext sc = new JavaSparkContext(conf); // (3)
String fileName = "";
if(args.length>0 && args[0]!=null && !args[0].isEmpty()) // (4)
fileName = args[0];
else
fileName = "pom.xml";
JavaRDD lines_rdd = sc.textFile(fileName); // (5)
long lines_count = lines_rdd.count(); // (6)
System.out.printf("There are %s lines in %sn"
, lines_count, fileName);
sc.close();
}
16 . 2

224. String masterURL = "local[*]"; // (1)
16 . 3

225. SparkConf conf = new SparkConf() // (2)
.setAppName("sc_01_anatomy_driver")
.setMaster(masterURL);
16 . 4

226. JavaSparkContext sc = new JavaSparkContext(conf); // (3)
16 . 5

227. String fileName = "";
if(args.length>0 && args[0]!=null && !args[0].isEmpty()) // (4)
fileName = args[0];
else
fileName = "pom.xml";
JavaRDD lines_rdd = sc.textFile(fileName); // (5)
long lines_count = lines_rdd.count(); // (6)
System.out.printf("There are %s lines in %sn"
, lines_count, fileName);
16 . 6

228. 16 . 7

229. 17 . 1

230. package cc.eighty20.spark.s01
import org.apache.spark.{SparkConf, SparkContext}
object sc_01_anatomy_driver {
def main(args: Array[String]){
val masterURL = "local[*]" // (1)
val conf = new SparkConf() // (2)
.setAppName("sc_01_anatomy_driver")
.setMaster(masterURL)
val sc = new SparkContext(conf) // (3)
val fileName = util.Try(args(0)).getOrElse("pom.xml") // (4)
val lines_rdd = sc.textFile(fileName).cache() // (5)
val lines_count = lines_rdd.count() // (6)
println(s"nThere are $lines_count lines in $fileName")
}
}
17 . 2

231. val masterURL = "local[*]" // (1)
17 . 3

232. val conf = new SparkConf() // (2)
.setAppName("sc_01_anatomy_driver")
.setMaster(masterURL)
17 . 4

233. val sc = new SparkContext(conf) // (3)
17 . 5

234. val fileName = util.Try(args(0)).getOrElse("pom.xml") // (4)
val lines_rdd = sc.textFile(fileName).cache() // (5)
val lines_count = lines_rdd.count() // (6)
println(s"nThere are $lines_count lines in $fileName")
17 . 6

235. 17 . 7

236. 18

237. 19 . 1

238. ERROR php: dying for unknown reasons
WARN dave, are you angry at me?
ERROR did mysql just barf?
WARN xylons approaching
ERROR mysql cluster: replace with spark cluster
19 . 2

239. // base RDD
val lines = sc.textFile("hdfs://sample_log_file_path/log.txt")
// transformed RDDs
val errors = lines.filter(_.startsWith("ERROR"))
val messages = errors.map(_.split("t")).map(r => r(1)).cache()
// action 1
val mysql_errors = messages.filter(_.contains("mysql")).count()
// action 2
val php_errors = messages.filter(_.contains("php")).count()
19 . 3

240. // base RDD
val lines = sc.textFile("hdfs://sample_log_file_path/log.txt")
19 . 4

241. // base RDD
val lines = sc.textFile("hdfs://sample_log_file_path/log.txt")
19 . 5

242. // base RDD
val lines = sc.textFile("hdfs://sample_log_file_path/log.txt")
19 . 6

243. // base RDD
val lines = sc.textFile("hdfs://sample_log_file_path/log.txt")
19 . 7

244. // transformed RDDs
val errors = lines.filter(_.startsWith("ERROR"))
val messages = errors.map(_.split("t")).map(r => r(1)).cache()
19 . 8

245. // action 1
val mysql_errors = messages.filter(_.contains("mysql")).count()
19 . 9

246. // action 2
val php_errors = messages.filter(_.contains("php")).count()
19 . 10

247. // action 2
val php_errors = messages.filter(_.contains("php")).count()
19 . 11

248. // action 2
val php_errors = messages.filter(_.contains("php")).count()
19 . 12

249. // base RDD
val lines = sc.textFile("hdfs://sample_log_file_path/log.txt")
// transformed RDDs
val errors = lines.filter(_.startsWith("ERROR"))
val messages = errors.map(_.split("t")).map(r => r(1)).cache()
// action 1
val mysql_errors = messages.filter(_.contains("mysql")).count()
// action 2
val php_errors = messages.filter(_.contains("php")).count()
19 . 13

250. 19 . 14

251. 19 . 15

252. 19 . 16

253. 19 . 17

254. 19 . 18

255. 20 . 1

256. # Apache Spark
Spark is a fast and general cluster computing system for Big Data. It provides
high-level APIs in Scala, Java, Python, and R, and an optimized engine that
supports general computation graphs for data analysis. It also supports a
rich set of higher-level tools including Spark SQL for SQL and DataFrames,
MLlib for machine learning, GraphX for graph processing,
and Spark Streaming for stream processing.
<http://spark.apache.org/>
## Online Documentation
You can find the latest Spark documentation, including a programming
guide, on the [project web page](http://spark.apache.org/documentation.html)
and [project wiki](https://cwiki.apache.org/confluence/display/SPARK).
This README file only contains basic setup instructions.
## Building Spark
...
20 . 2

257. val topN = 10
val fileName = "hdfs://log_file_path/README.md"
// RDD creation from external data source
val docs = sc.textFile(fileName)
// Split lines into words
val lower = docs.map(line => line.toLowerCase())
val words = lower.flatMap(line => line.split("s+"))
val counts = words.map(word => (word, 1))
// Count all words (automatic combination)
val freq = counts.reduceByKey(_ + _)
// Swap tuples and get top results
val top = freq.map(_.swap).top(topN)
top.foreach(println)
20 . 3

258. 20 . 4

259. 20 . 5

260. 20 . 6

261. 20 . 7

262. 20 . 8

263. 20 . 9

264. 20 . 10

265. 20 . 11

266. 20 . 12

267. 20 . 13

268. 20 . 14

269. 20 . 15

270. 20 . 16

271. 20 . 17

272. 20 . 18

273. 20 . 19

274. 20 . 20

275. 20 . 21

276. 20 . 22

277. 20 . 23

278. 21