-
Notifications
You must be signed in to change notification settings - Fork 29k
[SPARK-22346][ML] VectorSizeHint Transformer for using VectorAssembler in StructuredSteaming #19746
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from 1 commit
24cc417
2e76297
136d8f8
e117c15
591dcd2
b30e3b1
cafa875
7021552
d63f077
9c3dcec
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
- Loading branch information
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -70,17 +70,23 @@ class VectorSizeHint @Since("2.3.0") (@Since("2.3.0") override val uid: String) | |
| /** | ||
| * Param for how to handle invalid entries. Invalid vectors include nulls and vectors with the | ||
| * wrong size. The options are `skip` (filter out rows with invalid vectors), `error` (throw an | ||
| * error) and `keep` (do not check the vector size, and keep all rows). `error` by default. | ||
| * error) and `optimistic` (do not check the vector size, and keep all row\). `error` by default. | ||
| * | ||
| * Note: Users should take care when setting this param to `optimistic`. The use of the | ||
| * `optimistic` option will prevent the transformer from validating the sizes of vectors in | ||
| * `inputCol`. A mismatch between the metadata of a column and its contents could result in | ||
| * unexpected behaviour or errors when using that column. | ||
| * | ||
| * @group param | ||
| */ | ||
| @Since("2.3.0") | ||
|
||
| override val handleInvalid: Param[String] = new Param[String]( | ||
| this, | ||
| "handleInvalid", | ||
| "How to handle invalid vectors in inputCol. Invalid vectors include nulls and vectors with " + | ||
| "the wrong size. The options are skip (filter out rows with invalid vectors), error " + | ||
| "(throw an error) and keep (do not check the vector size, and keep all rows). `error` by " + | ||
| "default.", | ||
| "the wrong size. The options are `skip` (filter out rows with invalid vectors), `error` " + | ||
| "(throw an error) and `optimistic` (do not check the vector size, and keep all rows). " + | ||
| "`error` by default.", | ||
| ParamValidators.inArray(VectorSizeHint.supportedHandleInvalids)) | ||
|
|
||
| /** @group setParam */ | ||
|
|
@@ -173,6 +179,7 @@ class VectorSizeHint @Since("2.3.0") (@Since("2.3.0") override val uid: String) | |
| override def copy(extra: ParamMap): this.type = defaultCopy(extra) | ||
| } | ||
|
|
||
| /** :: Experimental :: */ | ||
| @Experimental | ||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Add Scala docstring here with |
||
| @Since("2.3.0") | ||
| object VectorSizeHint extends DefaultParamsReadable[VectorSizeHint] { | ||
|
|
||
Uh oh!
There was an error while loading. Please reload this page.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
"row\" ==> "rows"