Skip to content

[SPARK-41162][SQL] Fix anti- and semi-join for self-join with aggregations #39131

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
EnricoMi wants to merge 5 commits into from
Closed

[SPARK-41162][SQL] Fix anti- and semi-join for self-join with aggregations #39131

Changes from 1 commit
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
eebeb34
Use canPushThroughCondition for LeftSemi and LeftAnti joins
EnricoMi Dec 19, 2022
76dcd1e
Fix LeftSemiPushdownSuite class name (match file name)
EnricoMi Dec 19, 2022
ade61b8
Rename LeftSemiAnti to LeftSemi in LeftSemiAntiJoinPushDownSuite
EnricoMi Dec 19, 2022
cc28b21
Adjust test names to one naming scheme
EnricoMi Dec 19, 2022
2ae3a41
Add test to LeftSemiAntiJoinPushDownSuite
EnricoMi Dec 19, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Rename LeftSemiAnti to LeftSemi in LeftSemiAntiJoinPushDownSuite
  • Loading branch information
@EnricoMi
EnricoMi committed Dec 19, 2022
commit ade61b89cce2dc85615473dd386afcf94e33f20c
28 changes: 14 additions & 14 deletions ...rc/test/scala/org/apache/spark/sql/catalyst/optimizer/LeftSemiAntiJoinPushDownSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ class LeftSemiAntiJoinPushDownSuite extends PlanTest {
val testRelation1 = LocalRelation($"d".int)
val testRelation2 = LocalRelation($"e".int)

test("Project: LeftSemiAnti join pushdown") {
test("Project: LeftSemi join pushdown") {
Copy link
Member

@viirya viirya Jan 4, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

These change to test names are necessary?

Copy link
Contributor Author

@EnricoMi EnricoMi Jan 5, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The term LeftSemiAnti is wrong and misleading for individual tests, correcting this while I am touching the file.

val originalQuery = testRelation
.select(star())
.join(testRelation1, joinType = LeftSemi, condition = Some($"b" === $"d"))
Expand All @@ -59,7 +59,7 @@ class LeftSemiAntiJoinPushDownSuite extends PlanTest {
comparePlans(optimized, correctAnswer)
}

test("Project: LeftSemiAnti join no pushdown because of non-deterministic proj exprs") {
test("Project: LeftSemi join no pushdown because of non-deterministic proj exprs") {
val originalQuery = testRelation
.select(Rand(1), $"b", $"c")
.join(testRelation1, joinType = LeftSemi, condition = Some($"b" === $"d"))
Expand All @@ -68,7 +68,7 @@ class LeftSemiAntiJoinPushDownSuite extends PlanTest {
comparePlans(optimized, originalQuery.analyze)
}

test("Project: LeftSemiAnti join non correlated scalar subq") {
test("Project: LeftSemi join non correlated scalar subq") {
val subq = ScalarSubquery(testRelation.groupBy($"b")(sum($"c").as("sum")).analyze)
val originalQuery = testRelation
.select(subq.as("sum"))
Expand All @@ -83,7 +83,7 @@ class LeftSemiAntiJoinPushDownSuite extends PlanTest {
comparePlans(optimized, correctAnswer)
}

test("Project: LeftSemiAnti join no pushdown - correlated scalar subq in projection list") {
test("Project: LeftSemi join no pushdown - correlated scalar subq in projection list") {
val testRelation2 = LocalRelation($"e".int, $"f".int)
val subqPlan = testRelation2.groupBy($"e")(sum($"f").as("sum")).where($"e" === $"a")
val subqExpr = ScalarSubquery(subqPlan)
Expand All @@ -95,7 +95,7 @@ class LeftSemiAntiJoinPushDownSuite extends PlanTest {
comparePlans(optimized, originalQuery.analyze)
}

test("Aggregate: LeftSemiAnti join pushdown") {
test("Aggregate: LeftSemi join pushdown") {
val originalQuery = testRelation
.groupBy($"b")($"b", sum($"c"))
.join(testRelation1, joinType = LeftSemi, condition = Some($"b" === $"d"))
Expand All @@ -109,7 +109,7 @@ class LeftSemiAntiJoinPushDownSuite extends PlanTest {
comparePlans(optimized, correctAnswer)
}

test("Aggregate: LeftSemiAnti join no pushdown due to non-deterministic aggr expressions") {
test("Aggregate: LeftSemi join no pushdown due to non-deterministic aggr expressions") {
val originalQuery = testRelation
.groupBy($"b")($"b", Rand(10).as("c"))
.join(testRelation1, joinType = LeftSemi, condition = Some($"b" === $"d"))
Expand Down Expand Up @@ -142,7 +142,7 @@ class LeftSemiAntiJoinPushDownSuite extends PlanTest {
comparePlans(optimized, originalQuery.analyze)
}

test("LeftSemiAnti join over aggregate - no pushdown") {
test("LeftSemi join over aggregate - no pushdown") {
val originalQuery = testRelation
.groupBy($"b")($"b", sum($"c").as("sum"))
.join(testRelation1, joinType = LeftSemi, condition = Some($"b" === $"d" && $"sum" === $"d"))
Expand All @@ -151,7 +151,7 @@ class LeftSemiAntiJoinPushDownSuite extends PlanTest {
comparePlans(optimized, originalQuery.analyze)
}

test("Aggregate: LeftSemiAnti join non-correlated scalar subq aggr exprs") {
test("Aggregate: LeftSemi join non-correlated scalar subq aggr exprs") {
val subq = ScalarSubquery(testRelation.groupBy($"b")(sum($"c").as("sum")).analyze)
val originalQuery = testRelation
.groupBy($"a") ($"a", subq.as("sum"))
Expand All @@ -166,7 +166,7 @@ class LeftSemiAntiJoinPushDownSuite extends PlanTest {
comparePlans(optimized, correctAnswer)
}

test("LeftSemiAnti join over Window") {
test("LeftSemi join over Window") {
val winExpr = windowExpr(count($"b"),
windowSpec($"a" :: Nil, $"b".asc :: Nil, UnspecifiedFrame))

Expand Down Expand Up @@ -227,7 +227,7 @@ class LeftSemiAntiJoinPushDownSuite extends PlanTest {
comparePlans(optimized, correctAnswer)
}

test("Union: LeftSemiAnti join pushdown") {
test("Union: LeftSemi join pushdown") {
val testRelation2 = LocalRelation($"x".int, $"y".int, $"z".int)

val originalQuery = Union(Seq(testRelation, testRelation2))
Expand All @@ -243,7 +243,7 @@ class LeftSemiAntiJoinPushDownSuite extends PlanTest {
comparePlans(optimized, correctAnswer)
}

test("Union: LeftSemiAnti join pushdown in self join scenario") {
test("Union: LeftSemi join pushdown in self join scenario") {
val testRelation2 = LocalRelation($"x".int, $"y".int, $"z".int)
val attrX = testRelation2.output.head

Expand All @@ -262,7 +262,7 @@ class LeftSemiAntiJoinPushDownSuite extends PlanTest {
comparePlans(optimized, correctAnswer)
}

test("Unary: LeftSemiAnti join pushdown") {
test("Unary: LeftSemi join pushdown") {
val originalQuery = testRelation
.select(star())
.repartition(1)
Expand All @@ -277,7 +277,7 @@ class LeftSemiAntiJoinPushDownSuite extends PlanTest {
comparePlans(optimized, correctAnswer)
}

test("Unary: LeftSemiAnti join pushdown - empty join condition") {
test("Unary: LeftSemi join pushdown - empty join condition") {
val originalQuery = testRelation
.select(star())
.repartition(1)
Expand Down Expand Up @@ -320,7 +320,7 @@ class LeftSemiAntiJoinPushDownSuite extends PlanTest {
comparePlans(optimized, originalQuery.analyze)
}

test("Unary: LeftSemiAnti join pushdown - no pushdown") {
test("Unary: LeftSemi join pushdown - no pushdown") {
val testRelationWithArrayType = LocalRelation($"a".int, $"b".int, $"c_arr".array(IntegerType))
val originalQuery = testRelationWithArrayType
.generate(Explode($"c_arr"), alias = Some("arr"), outputNames = Seq("out_col"))
Expand Down