comments and doc refine

docs/ml-frequent-pattern-mining.md

@@ -35,15 +35,16 @@ We refer users to the papers for more details.
 
 * `minSupport`: the minimum support for an itemset to be identified as frequent.
   For example, if an item appears 3 out of 5 transactions, it has a support of 3/5=0.6.
-* `minConfidence`: minimum confidence for generating Association Rule. The parameter has no effect during `fit`, but specify
-  the minimum confidence for generating association rules from frequent itemsets.
-* `numPartitions`: the number of partitions used to distribute the work.
+* `minConfidence`: minimum confidence for generating Association Rule. The parameter will not affect the mining
+  for frequent itemsets,, but specify the minimum confidence for generating association rules from frequent itemsets.
+* `numPartitions`: the number of partitions used to distribute the work. By default the param is not set, and
+  partition number of the input dataset is used.
 
 The `FPGrowthModel` provides:
 
-* `freqItemsets`: frequent itemsets in the format of DataFrame("items"[Seq], "freq"[Long])
+* `freqItemsets`: frequent itemsets in the format of DataFrame("items"[Array], "freq"[Long])
 * `associationRules`: association rules generated with confidence above `minConfidence`, in the format of 
-  DataFrame("antecedent"[Seq], "consequent"[Seq], "confidence"[Double]).
+  DataFrame("antecedent"[Array], "consequent"[Array], "confidence"[Double]).
 * `transform`: The transform method examines the input items against all the association rules and
   summarize the consequents as prediction. The prediction column has the same data type as the
   input column and does not contain existing items in the input column.

examples/src/main/java/org/apache/spark/examples/ml/JavaFPGrowthExample.java

@@ -48,24 +48,20 @@ public static void main(String[] args) {
     });
     Dataset<Row> itemsDF = spark.createDataFrame(data, schema);
 
-    // Learn a mapping from words to Vectors.
-    FPGrowth fpgrowth = new FPGrowth()
+    FPGrowthModel model = new FPGrowth()
       .setMinSupport(0.5)
-      .setMinConfidence(0.6);
+      .setMinConfidence(0.6)
+      .fit(itemsDF);
 
-    FPGrowthModel model = fpgrowth.fit(itemsDF);
-
-    // get frequent itemsets.
+    // Display frequent itemsets.
     model.freqItemsets().show();
 
-    // get generated association rules.
+    // Display generated association rules.
     model.associationRules().show();
 
     // transform examines the input items against all the association rules and summarize the
     // consequents as prediction
-    Dataset<Row> result = model.transform(itemsDF);
-
-    result.show();
+    model.transform(itemsDF).show();
     // $example off$
 
     spark.stop();

examples/src/main/scala/org/apache/spark/examples/ml/FPGrowthExample.scala

@@ -47,7 +47,6 @@ object FPGrowthExample {
       "1 2")
     ).map(t => t.split(" ")).toDF("features")
 
-    // Trains a FPGrowth model.
     val fpgrowth = new FPGrowth().setMinSupport(0.5).setMinConfidence(0.6)
     val model = fpgrowth.fit(dataset)
 

mllib/src/main/scala/org/apache/spark/ml/fpm/FPGrowth.scala

@@ -17,7 +17,6 @@
 
 package org.apache.spark.ml.fpm
 
-import scala.collection.mutable.ArrayBuffer
 import scala.reflect.ClassTag
 
 import org.apache.hadoop.fs.Path
@@ -41,7 +40,7 @@ private[fpm] trait FPGrowthParams extends Params with HasFeaturesCol with HasPre
 
   /**
    * Minimal support level of the frequent pattern. [0.0, 1.0]. Any pattern that appears
-   * more than (minSupport * size-of-the-dataset) times will be output
+   * more than (minSupport * size-of-the-dataset) times will be output in the frequent itemsets.
    * Default: 0.3
    * @group param
    */
@@ -69,8 +68,8 @@ private[fpm] trait FPGrowthParams extends Params with HasFeaturesCol with HasPre
   def getNumPartitions: Int = $(numPartitions)
 
   /**
-   * Minimal confidence for generating Association Rule.
-   * Note that minConfidence has no effect during fitting.
+   * Minimal confidence for generating Association Rule. MinConfidence will not affect the mining
+   * for frequent itemsets, but will affect the association rules generation.
    * Default: 0.8
    * @group param
    */
@@ -154,7 +153,6 @@ class FPGrowth @Since("2.2.0") (
     }
     val parentModel = mllibFP.run(items)
     val rows = parentModel.freqItemsets.map(f => Row(f.items, f.freq))
-
     val schema = StructType(Seq(
       StructField("items", dataset.schema($(featuresCol)).dataType, nullable = false),
       StructField("freq", LongType, nullable = false)))
@@ -183,7 +181,7 @@ object FPGrowth extends DefaultParamsReadable[FPGrowth] {
  * :: Experimental ::
  * Model fitted by FPGrowth.
  *
- * @param freqItemsets frequent items in the format of DataFrame("items"[Seq], "freq"[Long])
+ * @param freqItemsets frequent itemsets in the format of DataFrame("items"[Array], "freq"[Long])
  */
 @Since("2.2.0")
 @Experimental
@@ -303,13 +301,13 @@ private[fpm] object AssociationRules {
 
   /**
    * Computes the association rules with confidence above minConfidence.
-   * @param dataset DataFrame("items", "freq") containing frequent itemset obtained from
-   *                algorithms like [[FPGrowth]].
+   * @param dataset DataFrame("items"[Array], "freq"[Long]) containing frequent itemsets obtained
+   *                from algorithms like [[FPGrowth]].
    * @param itemsCol column name for frequent itemsets
-   * @param freqCol column name for frequent itemsets count
-   * @param minConfidence minimum confidence for the result association rules
-   * @return a DataFrame("antecedent", "consequent", "confidence") containing the association
-   *         rules.
+   * @param freqCol column name for appearance count of the frequent itemsets
+   * @param minConfidence minimum confidence for generating the association rules
+   * @return a DataFrame("antecedent"[Array], "consequent"[Array], "confidence"[Double])
+   *         containing the association rules.
    */
   def getAssociationRulesFromFP[T: ClassTag](
         dataset: Dataset[_],

mllib/src/test/scala/org/apache/spark/ml/fpm/FPGrowthSuite.scala

@@ -19,7 +19,7 @@ package org.apache.spark.ml.fpm
 import org.apache.spark.SparkFunSuite
 import org.apache.spark.ml.util.DefaultReadWriteTest
 import org.apache.spark.mllib.util.MLlibTestSparkContext
-import org.apache.spark.sql.{DataFrame, Dataset, Row, SparkSession}
+import org.apache.spark.sql.{DataFrame, Dataset, SparkSession}
 import org.apache.spark.sql.functions._
 import org.apache.spark.sql.types._
 
@@ -91,6 +91,8 @@ class FPGrowthSuite extends SparkFunSuite with MLlibTestSparkContext with Defaul
       .setMinConfidence(0.5678)
     assert(fpGrowth.getMinSupport === 0.4567)
     assert(model.getMinConfidence === 0.5678)
+    // numPartitions should not have default value.
+    assert(fpGrowth.isDefined(fpGrowth.numPartitions) === false)
   }
 
   test("read/write") {