Skip to content

Commit ef407f3

Browse files
ulysses-youwangyum
authored andcommitted
[SPARK-49179][SQL] Fix v2 multi bucketed inner joins throw AssertionError
### What changes were proposed in this pull request? For SMJ with inner join, it just wraps left and right output partitioning to `PartitioningCollection` so it may not satisfy the target required clustering. ### Why are the changes needed? Fix exception if the query contains multi bucketed inner joins ```sql SELECT * FROM testcat.ns.t1 JOIN testcat.ns.t2 ON t1.id = t2.id JOIN testcat.ns.t3 ON t1.id = t3.id ``` ``` Cause: java.lang.AssertionError: assertion failed at scala.Predef$.assert(Predef.scala:264) at org.apache.spark.sql.execution.exchange.EnsureRequirements.createKeyGroupedShuffleSpec(EnsureRequirements.scala:642) at org.apache.spark.sql.execution.exchange.EnsureRequirements.$anonfun$checkKeyGroupCompatible$1(EnsureRequirements.scala:385) at scala.collection.immutable.List.map(List.scala:247) at scala.collection.immutable.List.map(List.scala:79) at org.apache.spark.sql.execution.exchange.EnsureRequirements.checkKeyGroupCompatible(EnsureRequirements.scala:382) at org.apache.spark.sql.execution.exchange.EnsureRequirements.checkKeyGroupCompatible(EnsureRequirements.scala:364) at org.apache.spark.sql.execution.exchange.EnsureRequirements.org$apache$spark$sql$execution$exchange$EnsureRequirements$$ensureDistributionAndOrdering(EnsureRequirements.scala:166) at org.apache.spark.sql.execution.exchange.EnsureRequirements$$anonfun$1.applyOrElse(EnsureRequirements.scala:714) at org.apache.spark.sql.execution.exchange.EnsureRequirements$$anonfun$1.applyOrElse(EnsureRequirements.scala:689) at org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$transformUpWithPruning$4(TreeNode.scala:528) at org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(origin.scala:84) at org.apache.spark.sql.catalyst.trees.TreeNode.transformUpWithPruning(TreeNode.scala:528) at org.apache.spark.sql.catalyst.trees.TreeNode.transformUp(TreeNode.scala:497) at org.apache.spark.sql.execution.exchange.EnsureRequirements.apply(EnsureRequirements.scala:689) at org.apache.spark.sql.execution.exchange.EnsureRequirements.apply(EnsureRequirements.scala:51) at org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanExec$.$anonfun$applyPhysicalRules$2(AdaptiveSparkPlanExec.scala:882) ``` ### Does this PR introduce _any_ user-facing change? yes, it's a bug fix ### How was this patch tested? add test ### Was this patch authored or co-authored using generative AI tooling? no Closes apache#47683 from ulysses-you/SPARK-49179. Authored-by: ulysses-you <[email protected]> Signed-off-by: youxiduo <[email protected]>
1 parent 10e9e4b commit ef407f3

File tree

1 file changed

+22
-0
lines changed

1 file changed

+22
-0
lines changed

sql/core/src/test/scala/org/apache/spark/sql/connector/KeyGroupedPartitioningSuite.scala

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -441,6 +441,28 @@ class KeyGroupedPartitioningSuite extends DistributionAndOrderingSuiteBase {
441441
}
442442
}
443443

444+
test("SPARK-49179: Fix v2 multi bucketed inner joins throw AssertionError") {
445+
val cols = Array(
446+
Column.create("id", LongType),
447+
Column.create("name", StringType))
448+
val buckets = Array(bucket(8, "id"))
449+
450+
withTable("t1", "t2", "t3") {
451+
Seq("t1", "t2", "t3").foreach { t =>
452+
createTable(t, cols, buckets)
453+
sql(s"INSERT INTO testcat.ns.$t VALUES (1, 'aa'), (2, 'bb'), (3, 'cc')")
454+
}
455+
val df = sql(
456+
"""
457+
|SELECT t1.id, t2.id, t3.name FROM testcat.ns.t1
458+
|JOIN testcat.ns.t2 ON t1.id = t2.id
459+
|JOIN testcat.ns.t3 ON t1.id = t3.id
460+
|""".stripMargin)
461+
checkAnswer(df, Seq(Row(1, 1, "aa"), Row(2, 2, "bb"), Row(3, 3, "cc")))
462+
assert(collectShuffles(df.queryExecution.executedPlan).isEmpty)
463+
}
464+
}
465+
444466
test("partitioned join: join with two partition keys and matching & sorted partitions") {
445467
val items_partitions = Array(bucket(8, "id"), days("arrive_time"))
446468
createTable(items, items_schema, items_partitions)

0 commit comments

Comments
 (0)