Skip to content

[SPARK-11549][Docs] Replace example code in mllib-evaluation-metrics.md using include_example #9689

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
vikasnp wants to merge 13 commits into from
Closed

[SPARK-11549][Docs] Replace example code in mllib-evaluation-metrics.md using include_example #9689

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
377d7a9
Intial commit..added all files
Nov 9, 2015
cb9c846
Fixed scala issues
Nov 13, 2015
ed33687
fixed styling issues
Nov 13, 2015
3af5fa3
removed build files
Nov 13, 2015
1106cae
java style issues
Nov 13, 2015
ad3c01e
fixed scala style issues
Nov 13, 2015
4d18447
fixed java style issues
Nov 13, 2015
3c40a35
fixed styling issues
Nov 18, 2015
892591b
fixed mllib-evaluation-metrics.md file
Nov 18, 2015
8d2d508
fixed java issues
Nov 18, 2015
54008ce
fixed import issue
Nov 19, 2015
1c5cc8f
fixed spacing and removed few imports and re-ordered imports
Nov 19, 2015
88512e7
removed extra lines & fixed style issues
Nov 19, 2015
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
fixed scala style issues
  • Loading branch information
Vikas Nelamangala authored and Vikas Nelamangala committed Nov 13, 2015
commit ad3c01ef932d4f4d5a47c7f1fbf6789b2f70caad
2 changes: 1 addition & 1 deletion examples/src/main/java/org/apache/spark/examples/mllib/JavaBinaryClassification.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,13 +23,13 @@
import scala.Tuple2;

import org.apache.spark.api.java.*;
import org.apache.spark.rdd.RDD;
import org.apache.spark.api.java.function.Function;
import org.apache.spark.mllib.classification.LogisticRegressionModel;
import org.apache.spark.mllib.classification.LogisticRegressionWithLBFGS;
import org.apache.spark.mllib.evaluation.BinaryClassificationMetrics;
import org.apache.spark.mllib.regression.LabeledPoint;
import org.apache.spark.mllib.util.MLUtils;
import org.apache.spark.rdd.RDD;
Copy link
Contributor

@yinxusen yinxusen Nov 13, 2015

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

$example off$ here.

import org.apache.spark.SparkConf;
import org.apache.spark.SparkContext;

Expand Down
2 changes: 1 addition & 1 deletion examples/src/main/java/org/apache/spark/examples/mllib/JavaMultiLabelClassification.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,8 @@
import scala.Tuple2;

import org.apache.spark.api.java.*;
import org.apache.spark.rdd.RDD;
import org.apache.spark.mllib.evaluation.MultilabelMetrics;
import org.apache.spark.rdd.RDD;
import org.apache.spark.SparkConf;

import java.util.Arrays;
Expand Down
2 changes: 1 addition & 1 deletion examples/src/main/java/org/apache/spark/examples/mllib/JavaMulticlassClassification.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,14 +23,14 @@
import scala.Tuple2;

import org.apache.spark.api.java.*;
import org.apache.spark.rdd.RDD;
import org.apache.spark.api.java.function.Function;
import org.apache.spark.mllib.classification.LogisticRegressionModel;
import org.apache.spark.mllib.classification.LogisticRegressionWithLBFGS;
import org.apache.spark.mllib.evaluation.MulticlassMetrics;
import org.apache.spark.mllib.regression.LabeledPoint;
import org.apache.spark.mllib.util.MLUtils;
import org.apache.spark.mllib.linalg.Matrix;
import org.apache.spark.rdd.RDD;
import org.apache.spark.SparkConf;
import org.apache.spark.SparkContext;

Expand Down
7 changes: 4 additions & 3 deletions examples/src/main/scala/org/apache/spark/examples/mllib/BinaryClassificationMetrics.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,6 @@
// scalastyle:off println
package org.apache.spark.examples.mllib

Copy link
Contributor

@yinxusen yinxusen Nov 13, 2015

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

remove one blank line

import org.apache.spark.sql.SQLContext
import org.apache.spark.{SparkContext, SparkConf}

// $example on$
import org.apache.spark.mllib.classification.LogisticRegressionWithLBFGS
Expand All @@ -28,6 +26,9 @@ import org.apache.spark.mllib.regression.LabeledPoint
import org.apache.spark.mllib.util.MLUtils
// $example off$

import org.apache.spark.{SparkContext, SparkConf}
import org.apache.spark.sql.SQLContext

object BinaryClassificationMetrics {

def main(args: Array[String]) {
Copy link
Contributor

@yinxusen yinxusen Nov 13, 2015

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Add a return value here. I.e. def main(args: Array[String]): Unit = {.

Expand Down Expand Up @@ -107,4 +108,4 @@ object BinaryClassificationMetrics {
}
}

// scalastyle:on println
// scalastyle:on println
15 changes: 8 additions & 7 deletions examples/src/main/scala/org/apache/spark/examples/mllib/MultiLabelMetrics.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,17 +18,16 @@
// scalastyle:off println
package org.apache.spark.examples.mllib

import org.apache.spark.sql.SQLContext
import org.apache.spark.{SparkContext, SparkConf}

// $example on$
import org.apache.spark.mllib.evaluation.MultilabelMetrics
import org.apache.spark.rdd.RDD;
// $example off$
object MultiLabelMetrics {

def main(args: Array[String]) {
import org.apache.spark.sql.SQLContext
import org.apache.spark.{SparkContext, SparkConf}

object MultiLabelMetrics {
def main(args: Array[String]) {
val conf = new SparkConf().setAppName("MultiLabelMetrics")
val sc = new SparkContext(conf)
val sqlContext = new SQLContext(sc)
Expand All @@ -53,7 +52,8 @@ object MultiLabelMetrics {
println(s"Accuracy = ${metrics.accuracy}")

// Individual label stats
metrics.labels.foreach(label => println(s"Class $label precision = ${metrics.precision(label)}"))
metrics.labels.foreach(label =>
println(s"Class $label precision = ${metrics.precision(label)}"))
metrics.labels.foreach(label => println(s"Class $label recall = ${metrics.recall(label)}"))
metrics.labels.foreach(label => println(s"Class $label F1-score = ${metrics.f1Measure(label)}"))

Expand All @@ -69,4 +69,5 @@ object MultiLabelMetrics {
println(s"Subset accuracy = ${metrics.subsetAccuracy}")
// $example off$
}
}
}
// scalastyle:on println
2 changes: 1 addition & 1 deletion examples/src/main/scala/org/apache/spark/examples/mllib/MulticlassMetrics.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -100,4 +100,4 @@ object MulticlassMetrics {

}
}
// scalastyle:on println
// scalastyle:on println
19 changes: 14 additions & 5 deletions examples/src/main/scala/org/apache/spark/examples/mllib/RankingMetrics.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,9 +20,12 @@ package org.apache.spark.examples.mllib

import org.apache.spark.sql.SQLContext
import org.apache.spark.{SparkContext, SparkConf}

// $example on$

import org.apache.spark.mllib.evaluation.{RegressionMetrics, RankingMetrics}
import org.apache.spark.mllib.recommendation.{ALS, Rating}

// $example off$
object RankingMetrics {

Expand All @@ -40,7 +43,8 @@ object RankingMetrics {
}.cache()

// Map ratings to 1 or 0, 1 indicating a movie that should be recommended
val binarizedRatings = ratings.map(r => Rating(r.user, r.product, if (r.rating > 0) 1.0 else 0.0)).cache()
val binarizedRatings = ratings.map(r => Rating(r.user, r.product,
if (r.rating > 0) 1.0 else 0.0)).cache()

// Summarize ratings
val numRatings = ratings.count()
Expand Down Expand Up @@ -68,7 +72,8 @@ object RankingMetrics {
// Assume that any movie a user rated 3 or higher (which maps to a 1) is a relevant document
// Compare with top ten most relevant documents
val userMovies = binarizedRatings.groupBy(_.user)
val relevantDocuments = userMovies.join(userRecommended).map { case (user, (actual, predictions)) =>
val relevantDocuments = userMovies.join(userRecommended).map { case (user, (actual,
predictions)) =>
(predictions.map(_.product), actual.filter(_.rating > 0.0).map(_.product).toArray)
}

Expand All @@ -89,9 +94,11 @@ object RankingMetrics {
}

// Get predictions for each data point
val allPredictions = model.predict(ratings.map(r => (r.user, r.product))).map(r => ((r.user, r.product), r.rating))
val allPredictions = model.predict(ratings.map(r => (r.user, r.product))).map(r => ((r.user,
r.product), r.rating))
val allRatings = ratings.map(r => ((r.user, r.product), r.rating))
val predictionsAndLabels = allPredictions.join(allRatings).map { case ((user, product), (predicted, actual)) =>
val predictionsAndLabels = allPredictions.join(allRatings).map { case ((user, product),
(predicted, actual)) =>
(predicted, actual)
}

Expand All @@ -103,4 +110,6 @@ object RankingMetrics {
println(s"R-squared = ${regressionMetrics.r2}")
// $example off$
}
}
}
// scalastyle:on println

4 changes: 3 additions & 1 deletion examples/src/main/scala/org/apache/spark/examples/mllib/RegressionMetrics.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,4 +66,6 @@ object RegressionMetrics {
println(s"Explained variance = ${metrics.explainedVariance}")
// $example off$
}
}
}
// scalastyle:on println