-
Notifications
You must be signed in to change notification settings - Fork 29k
[SPARK-20636] Add new optimization rule to transpose adjacent Window expressions. #17899
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from 1 commit
f444b2c
6f8c036
b7793cd
f183b78
1f21841
9a28535
76f1721
f840c69
e9f6928
72a4b3a
94e8115
4328831
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
- Loading branch information
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -88,7 +88,7 @@ abstract class Optimizer(sessionCatalog: SessionCatalog) | |
| CollapseRepartition, | ||
| CollapseProject, | ||
| CollapseWindow, | ||
| FlipWindow, | ||
| TransposeWindow, | ||
| CombineFilters, | ||
| CombineLimits, | ||
| CombineUnions, | ||
|
|
@@ -626,15 +626,15 @@ object CollapseWindow extends Rule[LogicalPlan] { | |
| } | ||
|
|
||
| /** | ||
| * Flip Adjacent Window Expressions. | ||
| * Transpose Adjacent Window Expressions. | ||
| * - If the partition spec of the parent Window expression is a subset of the partition spec | ||
| * of the child window expression, flip them. | ||
| * of the child window expression, transpose them. | ||
| */ | ||
| object FlipWindow extends Rule[LogicalPlan] { | ||
| object TransposeWindow extends Rule[LogicalPlan] { | ||
| def apply(plan: LogicalPlan): LogicalPlan = plan transformUp { | ||
| case w1 @ Window(we1, ps1, os1, w2 @ Window(we2, ps2, os2, grandChild)) | ||
|
||
| if ps2.containsSlice(ps1) => | ||
| Window(we2, ps2, os2, Window(we1, ps1, os1, grandChild)) | ||
| Project(w1.output, Window(we2, ps2, os2, Window(we1, ps1, os1, grandChild))) | ||
|
||
| } | ||
| } | ||
|
|
||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -468,4 +468,25 @@ class DataFrameWindowFunctionsSuite extends QueryTest with SharedSQLContext { | |
| spark.read.schema(sampleSchema).json(input.toDS()).select(c0, c1).foreach { _ => () } | ||
| } | ||
| } | ||
|
|
||
| test("window functions in multiple selects") { | ||
|
||
| val df = Seq( | ||
| ("S1", "P1", 100), | ||
| ("S1", "P1", 700), | ||
| ("S2", "P1", 200), | ||
| ("S2", "P2", 300) | ||
| ).toDF("sno", "pno", "qty") | ||
|
|
||
| val w1 = Window.partitionBy("sno") | ||
| val w2 = Window.partitionBy("sno", "pno") | ||
|
|
||
| checkAnswer( | ||
| df.select($"sno", $"pno", $"qty", sum($"qty").over(w2).alias("sum_qty_2")) | ||
| .select($"sno", $"pno", $"qty", col("sum_qty_2"), sum("qty").over(w1).alias("sum_qty_1")), | ||
| Seq( | ||
| Row("S1", "P1", 100, 800, 800), | ||
| Row("S1", "P1", 700, 800, 800), | ||
| Row("S2", "P1", 200, 200, 500), | ||
| Row("S2", "P2", 300, 300, 500))) | ||
| } | ||
| } | ||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
why is this rule useful?