Initial generated WordCount commit

Author: holdenk

src/main/scala/com/sparklingpandas/sparklingml/WordCount.scala

@@ -0,0 +1,28 @@
+package com.sparklingpandas.sparklingml
+
+/**
+ * Everyone's favourite wordcount example.
+ */
+
+import org.apache.spark.rdd._
+
+object WordCount {
+  /**
+   * A slightly more complex than normal wordcount example with optional
+   * separators and stopWords. Splits on the provided separators, removes
+   * the stopwords, and converts everything to lower case.
+   */
+  def withStopWordsFiltered(rdd : RDD[String],
+    separators : Array[Char] = " ".toCharArray,
+    stopWords : Set[String] = Set("the")): RDD[(String, Int)] = {
+
+    val tokens: RDD[String] = rdd.flatMap(_.split(separators).
+      map(_.trim.toLowerCase))
+    val lcStopWords = stopWords.map(_.trim.toLowerCase)
+    val words = tokens.filter(token =>
+      !lcStopWords.contains(token) && (token.length > 0))
+    val wordPairs = words.map((_, 1))
+    val wordCounts = wordPairs.reduceByKey(_ + _)
+    wordCounts
+  }
+}

src/test/scala/com/sparklingpandas/sparklingml/WordCountTest.scala

@@ -0,0 +1,28 @@
+package com.sparklingpandas.sparklingml
+
+/**
+ * A simple test for everyone's favourite wordcount example.
+ */
+
+import com.holdenkarau.spark.testing.SharedSparkContext
+import org.scalatest.FunSuite
+
+class WordCountTest extends FunSuite with SharedSparkContext {
+  test("word count with Stop Words Removed"){
+    val linesRDD = sc.parallelize(Seq(
+      "How happy was the panda? You ask.",
+      "Panda is the most happy panda in all the#!?ing land!"))
+
+    val stopWords: Set[String] = Set("a", "the", "in", "was", "there", "she", "he")
+    val splitTokens: Array[Char] = "#%?!. ".toCharArray
+
+    val wordCounts = WordCount.withStopWordsFiltered(
+      linesRDD, splitTokens, stopWords)
+    val wordCountsAsMap = wordCounts.collectAsMap()
+    assert(!wordCountsAsMap.contains("the"))
+    assert(!wordCountsAsMap.contains("?"))
+    assert(!wordCountsAsMap.contains("#!?ing"))
+    assert(wordCountsAsMap.contains("ing"))
+    assert(wordCountsAsMap.get("panda").get.equals(3))
+  }
+}