From 7b52f1ebbd4b7afd088c41695c61f4475911271e Mon Sep 17 00:00:00 2001 From: Shahid Date: Mon, 2 Jul 2018 01:09:19 +0530 Subject: [PATCH 1/5] Minor correction in the powerIterationSuite --- .../PowerIterationClusteringSuite.scala | 26 +++++++++++-------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/mllib/src/test/scala/org/apache/spark/ml/clustering/PowerIterationClusteringSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/clustering/PowerIterationClusteringSuite.scala index b7072728d48f..5d12fd2c846a 100644 --- a/mllib/src/test/scala/org/apache/spark/ml/clustering/PowerIterationClusteringSuite.scala +++ b/mllib/src/test/scala/org/apache/spark/ml/clustering/PowerIterationClusteringSuite.scala @@ -17,10 +17,12 @@ package org.apache.spark.ml.clustering +import scala.collection.mutable + import org.apache.spark.{SparkException, SparkFunSuite} import org.apache.spark.ml.util.DefaultReadWriteTest import org.apache.spark.mllib.util.MLlibTestSparkContext -import org.apache.spark.sql.{DataFrame, Dataset, SparkSession} +import org.apache.spark.sql.{DataFrame, Dataset, Row, SparkSession} import org.apache.spark.sql.functions.{col, lit} import org.apache.spark.sql.types._ @@ -76,12 +78,12 @@ class PowerIterationClusteringSuite extends SparkFunSuite .setMaxIter(40) .setWeightCol("weight") .assignClusters(data) - val localAssignments = assignments - .select('id, 'cluster) - .as[(Long, Int)].collect().toSet - val expectedResult = (0 until n1).map(x => (x, 1)).toSet ++ - (n1 until n).map(x => (x, 0)).toSet - assert(localAssignments === expectedResult) + + val predictions = Array.fill(2)(mutable.Set.empty[Long]) + assignments.select("id", "cluster").collect().foreach { + case Row(id: Long, cluster: Integer) => predictions(cluster) += id + } + assert(predictions.toSet == Set((0 until n1).toSet, (n1 until n).toSet)) val assignments2 = new PowerIterationClustering() .setK(2) @@ -89,10 +91,12 @@ class PowerIterationClusteringSuite extends SparkFunSuite .setInitMode("degree") .setWeightCol("weight") .assignClusters(data) - val localAssignments2 = assignments2 - .select('id, 'cluster) - .as[(Long, Int)].collect().toSet - assert(localAssignments2 === expectedResult) + + val predictions2 = Array.fill(2)(mutable.Set.empty[Long]) + assignments2.select("id", "cluster").collect().foreach { + case Row(id: Long, cluster: Integer) => predictions2(cluster) += id + } + assert(predictions2.toSet == Set((0 until n1).toSet, (n1 until n).toSet)) } test("supported input types") { From a591e4fb755df1d10255cf1d4d916a4afa604442 Mon Sep 17 00:00:00 2001 From: Shahid Date: Wed, 4 Jul 2018 13:31:10 +0530 Subject: [PATCH 2/5] Modifications: Minor correction in the powerIterationSuite --- .../PowerIterationClusteringSuite.scala | 20 +++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/mllib/src/test/scala/org/apache/spark/ml/clustering/PowerIterationClusteringSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/clustering/PowerIterationClusteringSuite.scala index 5d12fd2c846a..6beda7319a18 100644 --- a/mllib/src/test/scala/org/apache/spark/ml/clustering/PowerIterationClusteringSuite.scala +++ b/mllib/src/test/scala/org/apache/spark/ml/clustering/PowerIterationClusteringSuite.scala @@ -78,12 +78,17 @@ class PowerIterationClusteringSuite extends SparkFunSuite .setMaxIter(40) .setWeightCol("weight") .assignClusters(data) + .select("id", "cluster") + .as[(Long, Int)] + .collect() val predictions = Array.fill(2)(mutable.Set.empty[Long]) - assignments.select("id", "cluster").collect().foreach { - case Row(id: Long, cluster: Integer) => predictions(cluster) += id + + assignments.foreach{ + case (id, cluster) => predictions(cluster) += id } - assert(predictions.toSet == Set((0 until n1).toSet, (n1 until n).toSet)) + + assert(predictions.toSet === Set((0 until n1).toSet, (n1 until n).toSet)) val assignments2 = new PowerIterationClustering() .setK(2) @@ -91,12 +96,15 @@ class PowerIterationClusteringSuite extends SparkFunSuite .setInitMode("degree") .setWeightCol("weight") .assignClusters(data) + .select("id", "cluster") + .as[(Long, Int)] + .collect() val predictions2 = Array.fill(2)(mutable.Set.empty[Long]) - assignments2.select("id", "cluster").collect().foreach { - case Row(id: Long, cluster: Integer) => predictions2(cluster) += id + assignments2.foreach { + case (id, cluster) => predictions2(cluster) += id } - assert(predictions2.toSet == Set((0 until n1).toSet, (n1 until n).toSet)) + assert(predictions2.toSet === Set((0 until n1).toSet, (n1 until n).toSet)) } test("supported input types") { From 6cee6f0f868862ed579e8bb893e9d642c88fbf9b Mon Sep 17 00:00:00 2001 From: Shahid Date: Wed, 4 Jul 2018 13:34:59 +0530 Subject: [PATCH 3/5] Minor correction in the powerIterationSuite --- .../spark/ml/clustering/PowerIterationClusteringSuite.scala | 2 -- 1 file changed, 2 deletions(-) diff --git a/mllib/src/test/scala/org/apache/spark/ml/clustering/PowerIterationClusteringSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/clustering/PowerIterationClusteringSuite.scala index 6beda7319a18..ba20590383a5 100644 --- a/mllib/src/test/scala/org/apache/spark/ml/clustering/PowerIterationClusteringSuite.scala +++ b/mllib/src/test/scala/org/apache/spark/ml/clustering/PowerIterationClusteringSuite.scala @@ -83,11 +83,9 @@ class PowerIterationClusteringSuite extends SparkFunSuite .collect() val predictions = Array.fill(2)(mutable.Set.empty[Long]) - assignments.foreach{ case (id, cluster) => predictions(cluster) += id } - assert(predictions.toSet === Set((0 until n1).toSet, (n1 until n).toSet)) val assignments2 = new PowerIterationClustering() From b02cae5d3b735ea5c281fe125bf4b5792940294b Mon Sep 17 00:00:00 2001 From: Shahid Date: Wed, 4 Jul 2018 13:38:41 +0530 Subject: [PATCH 4/5] [Modifications]Minor correction in the powerIterationSuite --- .../spark/ml/clustering/PowerIterationClusteringSuite.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mllib/src/test/scala/org/apache/spark/ml/clustering/PowerIterationClusteringSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/clustering/PowerIterationClusteringSuite.scala index ba20590383a5..3a2aeda6e1ab 100644 --- a/mllib/src/test/scala/org/apache/spark/ml/clustering/PowerIterationClusteringSuite.scala +++ b/mllib/src/test/scala/org/apache/spark/ml/clustering/PowerIterationClusteringSuite.scala @@ -22,7 +22,7 @@ import scala.collection.mutable import org.apache.spark.{SparkException, SparkFunSuite} import org.apache.spark.ml.util.DefaultReadWriteTest import org.apache.spark.mllib.util.MLlibTestSparkContext -import org.apache.spark.sql.{DataFrame, Dataset, Row, SparkSession} +import org.apache.spark.sql.{DataFrame, Dataset, SparkSession} import org.apache.spark.sql.functions.{col, lit} import org.apache.spark.sql.types._ From 79447fa9d203f513008c8c650251d72e77fff605 Mon Sep 17 00:00:00 2001 From: Shahid Date: Wed, 4 Jul 2018 19:28:41 +0530 Subject: [PATCH 5/5] checkStyle: Minor correction in the powerIterationSuite --- .../spark/ml/clustering/PowerIterationClusteringSuite.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mllib/src/test/scala/org/apache/spark/ml/clustering/PowerIterationClusteringSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/clustering/PowerIterationClusteringSuite.scala index 3a2aeda6e1ab..55b460f1a452 100644 --- a/mllib/src/test/scala/org/apache/spark/ml/clustering/PowerIterationClusteringSuite.scala +++ b/mllib/src/test/scala/org/apache/spark/ml/clustering/PowerIterationClusteringSuite.scala @@ -83,7 +83,7 @@ class PowerIterationClusteringSuite extends SparkFunSuite .collect() val predictions = Array.fill(2)(mutable.Set.empty[Long]) - assignments.foreach{ + assignments.foreach { case (id, cluster) => predictions(cluster) += id } assert(predictions.toSet === Set((0 until n1).toSet, (n1 until n).toSet))