Skip to content

[SPARK-25121][SQL] Supports multi-part table names for broadcast hint resolution #22198

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
maropu wants to merge 11 commits into from
Closed

[SPARK-25121][SQL] Supports multi-part table names for broadcast hint resolution #22198

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
24de799
Fix
maropu Aug 23, 2018
a6e4e40
Fix
maropu Aug 25, 2018
f021770
Add test cases
dongjoon-hyun Aug 26, 2018
d434ba7
Fix
maropu Aug 27, 2018
c138b81
Fix
maropu Aug 27, 2018
6a202f2
Fix
maropu Aug 27, 2018
545148b
fix
dongjoon-hyun Aug 27, 2018
bc29a11
Fix
maropu Aug 29, 2018
59e60d4
Fix
maropu Aug 30, 2018
5b2b272
Fix
maropu Feb 4, 2019
b6b9f65
Fix
maropu Feb 13, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Fix
  • Loading branch information
@maropu
maropu committed Feb 13, 2019
commit 59e60d43a68e0641fa5a66028bd5dcc01d5b0804
2 changes: 2 additions & 0 deletions sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisTest.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,8 @@ trait AnalysisTest extends PlanTest {
catalog.createTempView("TaBlE", TestRelations.testRelation, overrideIfExists = true)
catalog.createTempView("TaBlE2", TestRelations.testRelation2, overrideIfExists = true)
catalog.createTempView("TaBlE3", TestRelations.testRelation3, overrideIfExists = true)
catalog.createGlobalTempView("TaBlE4", TestRelations.testRelation4, overrideIfExists = true)
catalog.createGlobalTempView("TaBlE5", TestRelations.testRelation5, overrideIfExists = true)
new Analyzer(catalog, conf) {
override val extendedResolutionRules = EliminateSubqueryAliases :: Nil
}
Expand Down
19 changes: 17 additions & 2 deletions sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/ResolveHintsSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -157,17 +157,32 @@ class ResolveHintsSuite extends AnalysisTest {
}

test("Supports multi-part table names for broadcast hint resolution") {
// local temp table
checkAnalysis(
UnresolvedHint("MAPJOIN", Seq("default.table", "default.table2"),
UnresolvedHint("MAPJOIN", Seq("table", "table2"),
table("table").join(table("table2"))),
Join(ResolvedHint(testRelation, HintInfo(broadcast = true)),
ResolvedHint(testRelation2, HintInfo(broadcast = true)), Inner, None),
caseSensitive = false)

checkAnalysis(
UnresolvedHint("MAPJOIN", Seq("default.TaBlE", "default.table2", "DEfault.TaBlE2"),
UnresolvedHint("MAPJOIN", Seq("TaBlE", "table2"),
table("TaBlE").join(table("TaBlE2"))),
Join(ResolvedHint(testRelation, HintInfo(broadcast = true)), testRelation2, Inner, None),
caseSensitive = true)

// global temp table
checkAnalysis(
UnresolvedHint("MAPJOIN", Seq("global_temp.table4", "GlOBal_TeMP.table5"),
table("global_temp", "table4").join(table("global_temp", "table5"))),
Join(ResolvedHint(testRelation4, HintInfo(broadcast = true)),
ResolvedHint(testRelation5, HintInfo(broadcast = true)), Inner, None),
caseSensitive = false)

checkAnalysis(
UnresolvedHint("MAPJOIN", Seq("global_temp.TaBlE4", "table5"),
table("global_temp", "TaBlE4").join(table("global_temp", "TaBlE5"))),
Join(ResolvedHint(testRelation4, HintInfo(broadcast = true)), testRelation5, Inner, None),
caseSensitive = true)
}
}
2 changes: 2 additions & 0 deletions sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/TestRelations.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,8 @@ object TestRelations {
AttributeReference("g", StringType)(),
AttributeReference("h", MapType(IntegerType, IntegerType))())

val testRelation5 = LocalRelation(AttributeReference("i", StringType)())

val nestedRelation = LocalRelation(
AttributeReference("top", StructType(
StructField("duplicateField", StringType) ::
Expand Down