Skip to content
Closed
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
34 commits
Select commit Hold shift + click to select a range
1be285c
[SPARK-45022][SQL] Provide context for dataset API errors
peter-toth Sep 18, 2023
4a3bc50
Remove an unused import
MaxGekk Oct 12, 2023
541928c
Merge remote-tracking branch 'origin/master' into context-for-dataset…
MaxGekk Oct 16, 2023
13dc953
Add stubs
MaxGekk Oct 17, 2023
eccd67a
Merge remote-tracking branch 'origin/master' into context-for-dataset…
MaxGekk Oct 17, 2023
6a2a33f
Fix MiMa
MaxGekk Oct 18, 2023
a364f29
Update proto QueryContext
MaxGekk Oct 22, 2023
4dc6cf1
Remove TODO
MaxGekk Oct 22, 2023
c58985a
Merge remote-tracking branch 'origin/master' into context-for-dataset…
MaxGekk Oct 22, 2023
a4e2093
Merge remote-tracking branch 'origin/master' into context-for-dataset…
MaxGekk Oct 22, 2023
fe1381e
Re-gen base_pb2.py
MaxGekk Oct 22, 2023
98fbc97
Fix a compile error
MaxGekk Oct 23, 2023
bc842a7
Fix SparkThrowableSuite
MaxGekk Oct 23, 2023
8de761d
Merge remote-tracking branch 'origin/master' into context-for-dataset…
MaxGekk Oct 23, 2023
8e09da0
Merge remote-tracking branch 'origin/master' into context-for-dataset…
MaxGekk Oct 24, 2023
1c33a00
Merge remote-tracking branch 'origin/master' into context-for-dataset…
MaxGekk Oct 25, 2023
3bf3bb9
Replace code by fragment
MaxGekk Oct 25, 2023
2bfac72
Remove code from proto
MaxGekk Oct 25, 2023
744b8ae
Re-gen base_pb2
MaxGekk Oct 25, 2023
651ee45
code -> fragment
MaxGekk Oct 26, 2023
98a12b5
Resolve conflicts in MiMa
MaxGekk Oct 26, 2023
61b34e6
Fix SparkThrowableSuite
MaxGekk Oct 26, 2023
29110b5
Remove framesToDrop
MaxGekk Oct 26, 2023
810076d
Remove optional
MaxGekk Oct 27, 2023
4042fb4
Re-gen base_pb2
MaxGekk Oct 27, 2023
115eca3
Merge remote-tracking branch 'origin/master' into context-for-dataset…
MaxGekk Oct 30, 2023
50fb94f
Revert override
MaxGekk Oct 31, 2023
9a31a07
since 3.5.0 -> since 4.0.0
MaxGekk Oct 31, 2023
0aa7850
Rename the enum: Dataset -> DataFrame
MaxGekk Oct 31, 2023
fe7b4a8
Rename proto: DATASET -> DATAFRAME
MaxGekk Oct 31, 2023
9ee81a6
Merge remote-tracking branch 'origin/master' into context-for-dataset…
MaxGekk Oct 31, 2023
75e251d
Rename the class: DatasetQueryContext -> DataFrameQueryContext
MaxGekk Oct 31, 2023
605fa50
Update summary: Dataset -> DataFrame
MaxGekk Oct 31, 2023
3cf20d7
Re-gen base_pb2.py
MaxGekk Oct 31, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Replace code by fragment
  • Loading branch information
MaxGekk committed Oct 25, 2023
commit 3bf3bb9396e2dbb169d7e98753ceb3e7a2095c4c
3 changes: 0 additions & 3 deletions common/utils/src/main/java/org/apache/spark/QueryContext.java
Original file line number Diff line number Diff line change
Expand Up @@ -49,9 +49,6 @@ public interface QueryContext {
// The corresponding fragment of the query which throws the exception.
String fragment();

// The Spark code (API) that caused throwing the exception.
String code();

// The user code (call site of the API) that caused throwing the exception.
String callSite();

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -124,7 +124,7 @@ private[spark] object SparkThrowableHelper {
if (stopIndex > 0) g.writeNumberField("stopIndex", stopIndex)
g.writeStringField("fragment", c.fragment())
case QueryContextType.Dataset =>
g.writeStringField("code", c.code())
g.writeStringField("fragment", c.fragment())
g.writeStringField("callSite", c.callSite())
}
g.writeEndObject()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -334,7 +334,6 @@ private[client] object GrpcExceptionConverter {
override def stopIndex(): Int = queryCtx.getStopIndex
override def fragment(): String = queryCtx.getFragment
override def callSite(): String = queryCtx.getCallSite
override def code(): String = queryCtx.getCode
override def summary(): String = queryCtx.getSummary
}
}.toArray
Expand Down
4 changes: 2 additions & 2 deletions core/src/test/scala/org/apache/spark/SparkFunSuite.scala
Original file line number Diff line number Diff line change
Expand Up @@ -378,8 +378,8 @@ abstract class SparkFunSuite
assert(actual.fragment() === expected.fragment,
"Invalid fragment of a query context. Actual:" + actual.toString)
} else if (actual.contextType() == QueryContextType.Dataset) {
assert(actual.code() === expected.code,
"Invalid code of a query context. Actual:" + actual.toString)
assert(actual.fragment() === expected.fragment,
"Invalid code fragment of a query context. Actual:" + actual.toString)
assert(actual.callSite().matches(expected.callSitePattern),
"Invalid callSite of a query context. Actual:" + actual.toString)
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -509,7 +509,6 @@ class SparkThrowableSuite extends SparkFunSuite {
override val startIndex = 2
override val stopIndex = -1
override val fragment = "1 / 0"
override def code: String = throw new UnsupportedOperationException
override def callSite: String = throw new UnsupportedOperationException
override val summary = ""
}
Expand Down Expand Up @@ -588,8 +587,7 @@ class SparkThrowableSuite extends SparkFunSuite {
override def objectType: String = throw new UnsupportedOperationException
override def startIndex: Int = throw new UnsupportedOperationException
override def stopIndex: Int = throw new UnsupportedOperationException
override def fragment: String = throw new UnsupportedOperationException
override val code: String = "div"
override val fragment: String = "div"
override val callSite: String = "SimpleApp$.main(SimpleApp.scala:9)"
override val summary = ""
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -131,27 +131,25 @@ case class SQLQueryContext(
originStartIndex.get <= originStopIndex.get
}

override def code: String = throw new UnsupportedOperationException
override def callSite: String = throw new UnsupportedOperationException
}

case class DatasetQueryContext(
override val code: String,
override val fragment: String,
override val callSite: String) extends QueryContext {
override val contextType = QueryContextType.Dataset

override def objectType: String = throw new UnsupportedOperationException
override def objectName: String = throw new UnsupportedOperationException
override def startIndex: Int = throw new UnsupportedOperationException
override def stopIndex: Int = throw new UnsupportedOperationException
override def fragment: String = throw new UnsupportedOperationException

override lazy val summary: String = {
val builder = new StringBuilder
builder ++= "== Dataset ==\n"
builder ++= "\""

builder ++= code
builder ++= fragment
builder ++= "\""
builder ++= " was called from "
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

shall we add a \n before the call site?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Not sure about this. Now it looks:

== Dataset ==
"col" was called from org.apache.spark.sql.DatasetSuite.$anonfun$new$621(DatasetSuite.scala:2673)

but what you propose:

== Dataset ==
"col" was called from
org.apache.spark.sql.DatasetSuite.$anonfun$new$621(DatasetSuite.scala:2673)

@cloud-fan Are you sure?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

OK let's leave it as it is

builder ++= callSite
Expand Down