-
Notifications
You must be signed in to change notification settings - Fork 29k
[SPARK-16289][SQL] Implement posexplode table generating function #13971
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from 1 commit
584eb9e
fcfccee
e255873
1cf723a
c5dee49
153e8eb
0266052
5f3a951
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
- Loading branch information
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -114,16 +114,23 @@ abstract class ExplodeBase(child: Expression, position: Boolean) | |
| override def elementSchema: StructType = child.dataType match { | ||
| case ArrayType(et, containsNull) => | ||
| if (position) { | ||
| new StructType().add("pos", IntegerType, false).add("col", et, containsNull) | ||
| new StructType() | ||
| .add("pos", IntegerType, false) | ||
| .add("col", et, containsNull) | ||
| } else { | ||
| new StructType().add("col", et, containsNull) | ||
| new StructType() | ||
| .add("col", et, containsNull) | ||
| } | ||
| case MapType(kt, vt, valueContainsNull) => | ||
| if (position) { | ||
| new StructType().add("pos", IntegerType, false).add("key", kt, false) | ||
| new StructType() | ||
| .add("pos", IntegerType, false) | ||
| .add("key", kt, false) | ||
| .add("value", vt, valueContainsNull) | ||
| } else { | ||
| new StructType().add("key", kt, false).add("value", vt, valueContainsNull) | ||
| new StructType() | ||
| .add("key", kt, false) | ||
| .add("value", vt, valueContainsNull) | ||
| } | ||
| } | ||
|
|
||
|
|
@@ -159,21 +166,35 @@ abstract class ExplodeBase(child: Expression, position: Boolean) | |
|
|
||
| /** | ||
| * Given an input array produces a sequence of rows for each value in the array. | ||
| * | ||
| * {{{ | ||
| * SELECT explode(array(10,20)) -> | ||
| * 10 | ||
| * 20 | ||
| * }}} | ||
| */ | ||
| // scalastyle:off line.size.limit | ||
| @ExpressionDescription( | ||
| usage = "_FUNC_(a) - Separates the elements of array a into multiple rows, or the elements of map a into multiple rows and columns.") | ||
| usage = "_FUNC_(a) - Separates the elements of array a into multiple rows, or the elements of map a into multiple rows and columns.", | ||
| extended = "> SELECT _FUNC_(array(10,20));\n 10\n 20") | ||
| // scalastyle:on line.size.limit | ||
| case class Explode(child: Expression) | ||
| extends ExplodeBase(child, position = false) with Serializable { | ||
| } | ||
|
|
||
| /** | ||
| * Given an input array produces a sequence of rows for each position and value in the array. | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. btw since the expression description might be difficult to see without line wrapping, it'd also be better to put an example here.
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. also you don't need the |
||
| * | ||
| * {{{ | ||
| * SELECT explode(array(10,20)) -> | ||
| * 0 10 | ||
| * 1 20 | ||
| * }}} | ||
| */ | ||
| // scalastyle:off line.size.limit | ||
| @ExpressionDescription( | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. an example would be useful. |
||
| usage = "_FUNC_(a) - Separates the elements of array a into multiple rows with positions, or the elements of a map into multiple rows and columns with positions.") | ||
| usage = "_FUNC_(a) - Separates the elements of array a into multiple rows with positions, or the elements of a map into multiple rows and columns with positions.", | ||
| extended = "> SELECT _FUNC_(array(10,20));\n 0\t10\n 1\t20") | ||
| // scalastyle:on line.size.limit | ||
| case class PosExplode(child: Expression) | ||
| extends ExplodeBase(child, position = true) with Serializable { | ||
|
||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
case classes are automatically serializable so you don't need this i think
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
also you don't need the
{ }There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Hmm. Okay. I will try to remove Serializable.