Skip to content

[SPARK-44355][SQL] Move WithCTE into command queries #42036

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
cloud-fan wants to merge 9 commits into from
Closed

[SPARK-44355][SQL] Move WithCTE into command queries #42036

Changes from 1 commit
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
4eabbd8
Add WithCTEInChildren
MaxGekk Jul 16, 2023
4349764
with WithCTEInChildren
MaxGekk Jul 16, 2023
4f1cbee
Rename WithCTEInChildren to CTEInChildren
MaxGekk Jul 16, 2023
a6633e4
Remove isCommand
MaxGekk Jul 16, 2023
25230f2
Add withCTEDefs()
MaxGekk Jul 16, 2023
58bc261
Fix coding style issues
MaxGekk Jul 16, 2023
fedc9a6
fix
cloud-fan Jul 17, 2023
45672ac
address comments
cloud-fan Jul 22, 2023
8a36db1
Update sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLC…
cloud-fan Jul 25, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Remove isCommand
  • Loading branch information
@MaxGekk
MaxGekk committed Jul 16, 2023
commit a6633e46f55add50455cd91f186a2eb6aeecbcf5
24 changes: 10 additions & 14 deletions sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CTESubstitution.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,17 +52,16 @@ object CTESubstitution extends Rule[LogicalPlan] {
if (!plan.containsPattern(UNRESOLVED_WITH)) {
return plan
}
val isCommand = false
val cteDefs = ArrayBuffer.empty[CTERelationDef]
val (substituted, firstSubstituted) =
LegacyBehaviorPolicy.withName(conf.getConf(LEGACY_CTE_PRECEDENCE_POLICY)) match {
case LegacyBehaviorPolicy.EXCEPTION =>
assertNoNameConflictsInCTE(plan)
traverseAndSubstituteCTE(plan, isCommand, Seq.empty, cteDefs)
traverseAndSubstituteCTE(plan, Seq.empty, cteDefs)
case LegacyBehaviorPolicy.LEGACY =>
(legacyTraverseAndSubstituteCTE(plan, cteDefs), None)
case LegacyBehaviorPolicy.CORRECTED =>
traverseAndSubstituteCTE(plan, isCommand, Seq.empty, cteDefs)
traverseAndSubstituteCTE(plan, Seq.empty, cteDefs)
}
if (cteDefs.isEmpty) {
substituted
Expand Down Expand Up @@ -128,7 +127,7 @@ object CTESubstitution extends Rule[LogicalPlan] {
plan.resolveOperatorsUp {
case UnresolvedWith(child, relations) =>
val resolvedCTERelations =
resolveCTERelations(relations, isLegacy = true, isCommand = false, Seq.empty, cteDefs)
resolveCTERelations(relations, isLegacy = true, Seq.empty, cteDefs)
substituteCTE(child, alwaysInline = true, resolvedCTERelations)
}
}
Expand Down Expand Up @@ -165,27 +164,25 @@ object CTESubstitution extends Rule[LogicalPlan] {
* SELECT * FROM t
* )
* @param plan the plan to be traversed
* @param isCommand if this is a command
* @param outerCTEDefs already resolved outer CTE definitions with names
* @param cteDefs all accumulated CTE definitions
* @return the plan where CTE substitution is applied and optionally the last substituted `With`
* where CTE definitions will be gathered to
*/
private def traverseAndSubstituteCTE(
plan: LogicalPlan,
isCommand: Boolean,
outerCTEDefs: Seq[(String, CTERelationDef)],
cteDefs: ArrayBuffer[CTERelationDef]): (LogicalPlan, Option[LogicalPlan]) = {
var firstSubstituted: Option[LogicalPlan] = None
val newPlan = plan.resolveOperatorsDownWithPruning(
_.containsAnyPattern(UNRESOLVED_WITH, PLAN_EXPRESSION)) {
case UnresolvedWith(child: LogicalPlan, relations) =>
val resolvedCTERelations =
resolveCTERelations(relations, isLegacy = false, isCommand, outerCTEDefs, cteDefs) ++
resolveCTERelations(relations, isLegacy = false, outerCTEDefs, cteDefs) ++
outerCTEDefs
val substituted = substituteCTE(
traverseAndSubstituteCTE(child, isCommand, resolvedCTERelations, cteDefs)._1,
isCommand,
traverseAndSubstituteCTE(child, resolvedCTERelations, cteDefs)._1,
false,
resolvedCTERelations)
if (firstSubstituted.isEmpty) {
firstSubstituted = Some(substituted)
Expand All @@ -203,10 +200,9 @@ object CTESubstitution extends Rule[LogicalPlan] {
private def resolveCTERelations(
relations: Seq[(String, SubqueryAlias)],
isLegacy: Boolean,
isCommand: Boolean,
outerCTEDefs: Seq[(String, CTERelationDef)],
cteDefs: ArrayBuffer[CTERelationDef]): Seq[(String, CTERelationDef)] = {
var resolvedCTERelations = if (isLegacy || isCommand) {
var resolvedCTERelations = if (isLegacy) {
Seq.empty
} else {
outerCTEDefs
Expand All @@ -229,12 +225,12 @@ object CTESubstitution extends Rule[LogicalPlan] {
// WITH t3 AS (SELECT * FROM t1)
// )
// t3 should resolve the t1 to `SELECT 2` instead of `SELECT 1`.
traverseAndSubstituteCTE(relation, isCommand, resolvedCTERelations, cteDefs)._1
traverseAndSubstituteCTE(relation, resolvedCTERelations, cteDefs)._1
}
// CTE definition can reference a previous one
val substituted = substituteCTE(innerCTEResolved, isLegacy || isCommand, resolvedCTERelations)
val substituted = substituteCTE(innerCTEResolved, isLegacy, resolvedCTERelations)
val cteRelation = CTERelationDef(substituted)
if (!(isLegacy || isCommand)) {
if (!(isLegacy)) {
cteDefs += cteRelation
}
// Prepending new CTEs makes sure that those have higher priority over outer ones.
Expand Down