Skip to content
Closed
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
22 commits
Select commit Hold shift + click to select a range
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
change transform to filter
  • Loading branch information
YY-OnCall committed Mar 2, 2017
commit ca12877c7f7e224268e145c3e8c4c37413596d66
Original file line number Diff line number Diff line change
Expand Up @@ -34,15 +34,13 @@ import org.apache.spark.sql.SparkSession
object FPGrowthExample {

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: remove blank line

def main(args: Array[String]): Unit = {

val spark = SparkSession
.builder
.appName(s"${this.getClass.getSimpleName}")
.getOrCreate()
import spark.implicits._

// $example on$
// Loads data.
val dataset = spark.createDataset(Seq(
"1 2 5",
"1 2 3 5",
Expand All @@ -53,16 +51,15 @@ object FPGrowthExample {
val fpgrowth = new FPGrowth().setMinSupport(0.5).setMinConfidence(0.6)
val model = fpgrowth.fit(dataset)

// get frequent itemsets.
// Display frequent itemsets.
model.freqItemsets.show()

// get generated association rules.
// Display generated association rules.
model.associationRules.show()

// transform examines the input items against all the association rules and summarize the
// consequents as prediction
model.transform(dataset).show()

// $example off$

spark.stop()
Expand Down
9 changes: 2 additions & 7 deletions mllib/src/main/scala/org/apache/spark/ml/fpm/FPGrowth.scala
Original file line number Diff line number Diff line change
Expand Up @@ -240,13 +240,8 @@ class FPGrowthModel private[ml] (
val predictUDF = udf((items: Seq[_]) => {
if (items != null) {
val itemset = items.toSet
brRules.value.flatMap { rule =>
if (rule._1.forall(item => itemset.contains(item))) {
rule._2.filter(item => !itemset.contains(item))
} else {
Seq.empty
}
}
brRules.value.filter(_._1.forall(itemset.contains))
.flatMap(_._2.filter(!itemset.contains(_)))
} else {
Seq.empty
}.distinct }, dt)
Expand Down