Skip to content
Closed
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
24 commits
Select commit Hold shift + click to select a range
2c1d5d8
Prototype
itholic Mar 4, 2024
376fc46
Merge branch 'master' of https://github.com/apache/spark into error_c…
itholic Apr 1, 2024
174a929
Merge branch 'master' of https://github.com/apache/spark into error_c…
itholic Apr 2, 2024
8ab1edf
Support query context testing and added UTs
itholic Apr 2, 2024
5906852
Merge branch 'master' of https://github.com/apache/spark into error_c…
itholic Apr 3, 2024
f3a7bd4
resolve comments
itholic Apr 3, 2024
bbaa399
Add JIRA pointer for testing
itholic Apr 3, 2024
b9f54f1
Silence the linter
itholic Apr 3, 2024
c8d98ea
Adjusted comments
itholic Apr 3, 2024
ef7f1df
Merge branch 'master' of https://github.com/apache/spark into error_c…
itholic Apr 4, 2024
cc52aab
Update displayed string and add comment for PySparkCurrentOrigin
itholic Apr 5, 2024
9c323d4
Using queue to ensure multiple call sites can be logged in order and …
itholic Apr 5, 2024
f5ad1c4
remove unnecessary comment
itholic Apr 5, 2024
4f12dc7
Extends Origin and WithOrigin to PySpark context support
itholic Apr 8, 2024
001c71e
Reusing fn for PySpark logging
itholic Apr 9, 2024
daa08cd
Add document for extended PySpark specific logging functions
itholic Apr 9, 2024
92faffe
remove unused code
itholic Apr 9, 2024
2514afb
Merge branch 'master' of https://github.com/apache/spark into error_c…
itholic Apr 9, 2024
672c176
Adress None properly
itholic Apr 9, 2024
1304c2b
Simplifying
itholic Apr 9, 2024
ff4037b
Merge branch 'master' of https://github.com/apache/spark into error_c…
itholic Apr 10, 2024
1d8df34
Respect spark.sql.stackTracesInDataFrameContext
itholic Apr 10, 2024
95f7848
Add captureStackTrace to remove duplication
itholic Apr 10, 2024
1dd53ed
pysparkLoggingInfo -> pysparkErrorContext
itholic Apr 10, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Update displayed string and add comment for PySparkCurrentOrigin
  • Loading branch information
itholic committed Apr 5, 2024
commit cc52aab0e9b401d82e02bf98607cdab5d67d7766
Original file line number Diff line number Diff line change
Expand Up @@ -160,32 +160,28 @@ case class DataFrameQueryContext(stackTrace: Seq[StackTraceElement]) extends Que
val pysparkOriginInfo: mutable.Map[String, String] = PySparkCurrentOrigin.get()

val pysparkFragment: String = pysparkOriginInfo.getOrElse("fragment", "")

val pysparkCallSite: String = pysparkOriginInfo.getOrElse("callSite", "")

val (displayedFragment, displayedCallsite) = if (pysparkOriginInfo.nonEmpty) {
(pysparkFragment, pysparkCallSite)
} else {
(fragment, callSite)
}

PySparkCurrentOrigin.clear()

override lazy val summary: String = {
val builder = new StringBuilder
builder ++= "== DataFrame ==\n"
builder ++= "\""

builder ++= fragment
builder ++= displayedFragment
builder ++= "\""
builder ++= " was called from\n"
builder ++= callSite
builder ++= displayedCallsite
builder += '\n'

if (pysparkOriginInfo.nonEmpty) {
builder ++= "\n== PySpark call site ==\n"
builder ++= "\""

builder ++= pysparkFragment
builder ++= "\""
builder ++= " was called from\n"
builder ++= pysparkCallSite
builder += '\n'
}

PySparkCurrentOrigin.clear()

builder.result()
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,11 @@ object CurrentOrigin {
/**
* Provides detailed call site information on PySpark.
* This information is generated in PySpark and stored in the form of a Map.
* The 'fragment' represents the method name or the action within the PySpark code
* that initiated the call, serving as a brief descriptor of the operation being performed.
* The 'callSite' provides a more detailed context, including the file name and line number in the
* PySpark user code where the operation originated, offering precise location information for
* debugging purposes.
*/
object PySparkCurrentOrigin {
private val pysparkCallSite = new ThreadLocal[mutable.Map[String, String]]() {
Expand Down