Skip to content

[SPARK-20443][MLLIB][ML] set ALS blockify size #17739

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
mpjlu wants to merge 6 commits into from
Closed

[SPARK-20443][MLLIB][ML] set ALS blockify size #17739

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
78e060e
set ALS blockify size
Apr 24, 2017
b4e392e
Add blockSize parameter
Apr 24, 2017
85f6c1d
merge master
May 11, 2017
cbd2402
add ML set blockSize
May 11, 2017
ffe79b8
typo
May 11, 2017
e0d6daa
merge master
May 16, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Add blockSize parameter
  • Loading branch information
Peng committed Apr 24, 2017
commit b4e392ea249d37e91995e1d604a0d463567a7624
16 changes: 13 additions & 3 deletions mllib/src/main/scala/org/apache/spark/mllib/recommendation/MatrixFactorizationModel.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,8 @@ import org.apache.spark.storage.StorageLevel
class MatrixFactorizationModel @Since("0.8.0") (
@Since("0.8.0") val rank: Int,
@Since("0.8.0") val userFeatures: RDD[(Int, Array[Double])],
@Since("0.8.0") val productFeatures: RDD[(Int, Array[Double])])
@Since("0.8.0") val productFeatures: RDD[(Int, Array[Double])],
@Since("2.3.0") var blockSize: Int = 4096)
extends Saveable with Serializable with Logging {

require(rank > 0)
Expand Down Expand Up @@ -216,7 +217,8 @@ class MatrixFactorizationModel @Since("0.8.0") (
*/
@Since("1.4.0")
def recommendProductsForUsers(num: Int): RDD[(Int, Array[Rating])] = {
MatrixFactorizationModel.recommendForAll(rank, userFeatures, productFeatures, num).map {
MatrixFactorizationModel.recommendForAll(rank, userFeatures, productFeatures, num, blockSize)
.map {
case (user, top) =>
val ratings = top.map { case (product, rating) => Rating(user, product, rating) }
(user, ratings)
Expand All @@ -234,12 +236,20 @@ class MatrixFactorizationModel @Since("0.8.0") (
*/
@Since("1.4.0")
def recommendUsersForProducts(num: Int): RDD[(Int, Array[Rating])] = {
MatrixFactorizationModel.recommendForAll(rank, productFeatures, userFeatures, num).map {
MatrixFactorizationModel.recommendForAll(rank, productFeatures, userFeatures, num, blockSize)
.map {
case (product, top) =>
val ratings = top.map { case (user, rating) => Rating(user, product, rating) }
(product, ratings)
}
}

/** Sets blockSize, which will be used for recommendForAll. */
@Since("2.3.0")
def setBlockSize(blockSize: Int): this.type = {
this.blockSize = blockSize
this
}
}

@Since("1.3.0")
Expand Down
3 changes: 3 additions & 0 deletions project/MimaExcludes.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -999,6 +999,9 @@ object MimaExcludes {
ProblemFilters.exclude[IncompatibleResultTypeProblem]("org.apache.spark.ml.classification.RandomForestClassificationModel.setFeatureSubsetStrategy"),
ProblemFilters.exclude[IncompatibleResultTypeProblem]("org.apache.spark.ml.regression.RandomForestRegressionModel.numTrees"),
ProblemFilters.exclude[IncompatibleResultTypeProblem]("org.apache.spark.ml.regression.RandomForestRegressionModel.setFeatureSubsetStrategy")
) ++ Seq(
// [SPARK-20443] set ALS blockify size
ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.mllib.recommendation.MatrixFactorizationModel.this")
)
}

Expand Down