Skip to content

[SPARK-43914][SQL] Assign names to the error class _LEGACY_ERROR_TEMP_[2433-2437] #41476

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
beliefer wants to merge 8 commits into from
Closed

[SPARK-43914][SQL] Assign names to the error class _LEGACY_ERROR_TEMP_[2433-2437] #41476

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
f6e8809
[SPARK-43914][SQL] Assign names to the error class _LEGACY_ERROR_TEMP…
beliefer Jun 6, 2023
e17cc59
Update code
beliefer Jun 6, 2023
3668c24
Update code
beliefer Jun 14, 2023
15987f9
Update code
beliefer Jun 18, 2023
169dfa5
Update code
beliefer Jun 19, 2023
d549eee
Update code
beliefer Jun 20, 2023
a563972
Update code
beliefer Jun 20, 2023
bc9b11f
Fix conflicts
beliefer Jun 21, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 0 additions & 34 deletions core/src/main/resources/error/error-classes.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5667,40 +5667,6 @@
"Cannot change nullable column to non-nullable: <fieldName>."
]
},
"_LEGACY_ERROR_TEMP_2433" : {
"message" : [
"Only a single table generating function is allowed in a SELECT clause, found:",
"<sqlExprs>."
]
},
"_LEGACY_ERROR_TEMP_2434" : {
"message" : [
"Failure when resolving conflicting references in Join:",
"<plan>",
"Conflicting attributes: <conflictingAttributes>."
]
},
"_LEGACY_ERROR_TEMP_2435" : {
"message" : [
"Failure when resolving conflicting references in Intersect:",
"<plan>",
"Conflicting attributes: <conflictingAttributes>."
]
},
"_LEGACY_ERROR_TEMP_2436" : {
"message" : [
"Failure when resolving conflicting references in Except:",
"<plan>",
"Conflicting attributes: <conflictingAttributes>."
]
},
"_LEGACY_ERROR_TEMP_2437" : {
"message" : [
"Failure when resolving conflicting references in AsOfJoin:",
"<plan>",
"Conflicting attributes: <conflictingAttributes>."
]
},
"_LEGACY_ERROR_TEMP_2446" : {
"message" : [
"Operation not allowed: <cmd> only works on table with location provided: <tableIdentWithDB>"
Expand Down
65 changes: 38 additions & 27 deletions sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -674,9 +674,8 @@ trait CheckAnalysis extends PredicateHelper with LookupCatalog with QueryErrorsB
}

case p @ Project(exprs, _) if containsMultipleGenerators(exprs) =>
p.failAnalysis(
errorClass = "_LEGACY_ERROR_TEMP_2433",
messageParameters = Map("sqlExprs" -> exprs.map(_.sql).mkString(",")))
val generators = exprs.filter(expr => expr.exists(_.isInstanceOf[Generator]))
throw QueryCompilationErrors.moreThanOneGeneratorError(generators, "SELECT")

case p @ Project(projectList, _) =>
projectList.foreach(_.transformDownWithPruning(
Expand All @@ -686,36 +685,48 @@ trait CheckAnalysis extends PredicateHelper with LookupCatalog with QueryErrorsB
})

case j: Join if !j.duplicateResolved =>
val conflictingAttributes = j.left.outputSet.intersect(j.right.outputSet)
j.failAnalysis(
errorClass = "_LEGACY_ERROR_TEMP_2434",
messageParameters = Map(
"plan" -> plan.toString,
"conflictingAttributes" -> conflictingAttributes.mkString(",")))
val conflictingAttributes =
j.left.outputSet.intersect(j.right.outputSet).map(toSQLExpr(_)).mkString(", ")
throw SparkException.internalError(
msg = s"""
|Failure when resolving conflicting references in ${j.nodeName}:
|${planToString(plan)}
|Conflicting attributes: $conflictingAttributes.""".stripMargin,
context = j.origin.getQueryContext,
summary = j.origin.context.summary)

case i: Intersect if !i.duplicateResolved =>
val conflictingAttributes = i.left.outputSet.intersect(i.right.outputSet)
i.failAnalysis(
errorClass = "_LEGACY_ERROR_TEMP_2435",
messageParameters = Map(
"plan" -> plan.toString,
"conflictingAttributes" -> conflictingAttributes.mkString(",")))
val conflictingAttributes =
i.left.outputSet.intersect(i.right.outputSet).map(toSQLExpr(_)).mkString(", ")
throw SparkException.internalError(
msg = s"""
|Failure when resolving conflicting references in ${i.nodeName}:
|${planToString(plan)}
|Conflicting attributes: $conflictingAttributes.""".stripMargin,
context = i.origin.getQueryContext,
summary = i.origin.context.summary)

case e: Except if !e.duplicateResolved =>
val conflictingAttributes = e.left.outputSet.intersect(e.right.outputSet)
e.failAnalysis(
errorClass = "_LEGACY_ERROR_TEMP_2436",
messageParameters = Map(
"plan" -> plan.toString,
"conflictingAttributes" -> conflictingAttributes.mkString(",")))
val conflictingAttributes =
e.left.outputSet.intersect(e.right.outputSet).map(toSQLExpr(_)).mkString(", ")
throw SparkException.internalError(
msg = s"""
|Failure when resolving conflicting references in ${e.nodeName}:
|${planToString(plan)}
|Conflicting attributes: $conflictingAttributes.""".stripMargin,
context = e.origin.getQueryContext,
summary = e.origin.context.summary)

case j: AsOfJoin if !j.duplicateResolved =>
val conflictingAttributes = j.left.outputSet.intersect(j.right.outputSet)
j.failAnalysis(
errorClass = "_LEGACY_ERROR_TEMP_2437",
messageParameters = Map(
"plan" -> plan.toString,
"conflictingAttributes" -> conflictingAttributes.mkString(",")))
val conflictingAttributes =
j.left.outputSet.intersect(j.right.outputSet).map(toSQLExpr(_)).mkString(", ")
throw SparkException.internalError(
msg = s"""
|Failure when resolving conflicting references in ${j.nodeName}:
|${planToString(plan)}
|Conflicting attributes: $conflictingAttributes.""".stripMargin,
context = j.origin.getQueryContext,
summary = j.origin.context.summary)

// TODO: although map type is not orderable, technically map type should be able to be
// used in equality comparison, remove this type check once we support it.
Expand Down
74 changes: 68 additions & 6 deletions sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisErrorSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ import org.apache.spark.sql.catalyst.dsl.plans._
import org.apache.spark.sql.catalyst.expressions._
import org.apache.spark.sql.catalyst.expressions.aggregate.{Count, Max}
import org.apache.spark.sql.catalyst.parser.CatalystSqlParser
import org.apache.spark.sql.catalyst.plans.{Cross, LeftOuter, RightOuter}
import org.apache.spark.sql.catalyst.plans.{AsOfJoinDirection, Cross, Inner, LeftOuter, RightOuter}
import org.apache.spark.sql.catalyst.plans.logical._
import org.apache.spark.sql.catalyst.util.{ArrayBasedMapData, GenericArrayData, MapData}
import org.apache.spark.sql.internal.SQLConf
Expand Down Expand Up @@ -781,11 +781,73 @@ class AnalysisErrorSuite extends AnalysisTest {

test("error test for self-join") {
val join = Join(testRelation, testRelation, Cross, None, JoinHint.NONE)
val error = intercept[AnalysisException] {
SimpleAnalyzer.checkAnalysis(join)
}
assert(error.message.contains("Failure when resolving conflicting references in Join"))
assert(error.message.contains("Conflicting attributes"))
checkError(
exception = intercept[SparkException] {
SimpleAnalyzer.checkAnalysis(join)
},
errorClass = "INTERNAL_ERROR",
parameters = Map("message" ->
"""
|Failure when resolving conflicting references in Join:
|'Join Cross
|:- LocalRelation <empty>, [a#x]
|+- LocalRelation <empty>, [a#x]
|
|Conflicting attributes: "a".""".stripMargin))
}

test("error test for self-intersect") {
val intersect = Intersect(testRelation, testRelation, true)
checkError(
exception = intercept[SparkException] {
SimpleAnalyzer.checkAnalysis(intersect)
},
errorClass = "INTERNAL_ERROR",
parameters = Map("message" ->
"""
|Failure when resolving conflicting references in Intersect All:
|'Intersect All true
|:- LocalRelation <empty>, [a#x]
|+- LocalRelation <empty>, [a#x]
|
|Conflicting attributes: "a".""".stripMargin))
}

test("error test for self-except") {
val except = Except(testRelation, testRelation, true)
checkError(
exception = intercept[SparkException] {
SimpleAnalyzer.checkAnalysis(except)
},
errorClass = "INTERNAL_ERROR",
parameters = Map("message" ->
"""
|Failure when resolving conflicting references in Except All:
|'Except All true
|:- LocalRelation <empty>, [a#x]
|+- LocalRelation <empty>, [a#x]
|
|Conflicting attributes: "a".""".stripMargin))
}

test("error test for self-asOfJoin") {
val asOfJoin =
AsOfJoin(testRelation, testRelation, testRelation.output(0), testRelation.output(0),
None, Inner, tolerance = None, allowExactMatches = true,
direction = AsOfJoinDirection("backward"))
checkError(
exception = intercept[SparkException] {
SimpleAnalyzer.checkAnalysis(asOfJoin)
},
errorClass = "INTERNAL_ERROR",
parameters = Map("message" ->
"""
|Failure when resolving conflicting references in AsOfJoin:
|'AsOfJoin (a#x >= a#x), Inner
|:- LocalRelation <empty>, [a#x]
|+- LocalRelation <empty>, [a#x]
|
|Conflicting attributes: "a".""".stripMargin))
}

test("check grouping expression data types") {
Expand Down
14 changes: 14 additions & 0 deletions sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -364,6 +364,20 @@ class DataFrameSuite extends QueryTest
Row("a", Seq("a"), 1) :: Nil)
}

test("more than one generator in SELECT clause") {
val df = Seq((Array("a"), 1)).toDF("a", "b")

checkError(
exception = intercept[AnalysisException] {
df.select(explode($"a").as("a"), explode($"a").as("b"))
},
errorClass = "UNSUPPORTED_GENERATOR.MULTI_GENERATOR",
parameters = Map(
"clause" -> "SELECT",
"num" -> "2",
"generators" -> "\"explode(a)\", \"explode(a)\""))
}

test("sort after generate with join=true") {
val df = Seq((Array("a"), 1)).toDF("a", "b")

Expand Down