Spark

The first spark app: word count

这里介绍怎么编写一个spark app。

通过spark shell编写

1
2
3
4
5
6
cd /usr/local/spark
./bin/spark-shell --master local
val textFile=sc.textFile("file:////home/hadoop/Desktop/tem/1.txt")
val wordCount=textFile.flatMap(line=>line.split(" ")).map(word=>(word,1))reduceByKey((a,b)=>a+b)
wordCount.collect()
wordCount.foreach(println)

独立编写

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
/* SimpleApp.scala */
import org.apache.spark.SparkContext
import org.apache.spark.SparkContext._
import org.apache.spark.SparkConf

object SimpleApp {
    def main(args: Array[String]) {
        val logFile = "file:///usr/local/spark/README.md" // Should be some file on your system
        val conf = new SparkConf().setAppName("Simple Application")
        val sc = new SparkContext(conf)
        val logData = sc.textFile(logFile, 2).cache()
        val numAs = logData.filter(line => line.contains("a")).count()
        val numBs = logData.filter(line => line.contains("b")).count()
        println("Lines with a: %s, Lines with b: %s".format(numAs, numBs))
    }
}

使用intelij编写spark app

这里参考：使用Intellij Idea编写Spark应用程序（Scala+SBT）