Skip to content

[SPARK-47579][CORE][PART4] Migrate logInfo with variables to structured logging framework #46724

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
zeotuan wants to merge 14 commits into from
Closed

[SPARK-47579][CORE][PART4] Migrate logInfo with variables to structured logging framework #46724

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
3631a04
Migrate to structured logging
zeotuan May 18, 2024
85a560f
Migrate to structured logging
zeotuan May 23, 2024
1337771
Merge branch 'master' of https://github.com/zeotuan/spark into coreInfo3
zeotuan May 23, 2024
06e7f7e
Fix formatting
zeotuan May 23, 2024
ed4624e
Fix formatting
zeotuan May 24, 2024
9ad231b
Order LogKeys
zeotuan May 24, 2024
d214b31
Merge master
zeotuan May 25, 2024
cb0b980
Migrate to structure logging
zeotuan May 25, 2024
6db64bd
Resolve review
zeotuan May 29, 2024
7865b72
Fix key ordering
zeotuan May 29, 2024
e970298
fix ambiguous import. Remove unused key
zeotuan May 29, 2024
a408ebc
Reorder import
zeotuan May 29, 2024
32d0cd2
Reorder import
zeotuan May 29, 2024
f1ed4f2
Update common/utils/src/main/scala/org/apache/spark/internal/LogKey.s…
gengliangwang May 29, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Resolve review
  • Loading branch information
@zeotuan
zeotuan committed May 29, 2024
commit 6db64bd8b3b35df1b48a4f6e8300f48ae983592f
1 change: 1 addition & 0 deletions common/utils/src/main/scala/org/apache/spark/internal/LogKey.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -784,6 +784,7 @@ object LogKeys {
case object URL extends LogKey
case object URL2 extends LogKey
case object URLS extends LogKey
case object EXECUTOR_USER_CLASS_PATH_FIRST extends LogKey
case object USER_ID extends LogKey
case object USER_NAME extends LogKey
case object UUID extends LogKey
Expand Down
7 changes: 4 additions & 3 deletions core/src/main/scala/org/apache/spark/executor/Executor.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -221,7 +221,7 @@ private[spark] class Executor(
if (sessionBasedRoot.isDirectory && sessionBasedRoot.exists()) {
Utils.deleteRecursively(sessionBasedRoot)
}
logInfo(log"Session evicted: ${LogMDC(UUID, state.sessionUUID)}")
logInfo(log"Session evicted: ${LogMDC(SESSION_ID, state.sessionUUID)}")
}
})
.build[String, IsolatedSessionState]
Expand Down Expand Up @@ -1087,7 +1087,8 @@ private[spark] class Executor(
private def createClassLoader(urls: Array[URL], useStub: Boolean): MutableURLClassLoader = {
logInfo(
log"Starting executor with user classpath" +
log" (userClassPathFirst = ${LogMDC(CLASS_PATH, userClassPathFirst)}): " +
log" (userClassPathFirst =" +
log" ${LogMDC(LogKeys.EXECUTOR_USER_CLASS_PATH_FIRST, userClassPathFirst)}): " +
log"${LogMDC(URLS, urls.mkString("'", ",", "'"))}"
)

Expand Down Expand Up @@ -1138,7 +1139,7 @@ private[spark] class Executor(
parent
}
logInfo(log"Created or updated repl class loader ${LogMDC(CLASS_LOADER, classLoader)}" +
log" for ${LogMDC(UUID, sessionUUID)}.")
log" for ${LogMDC(SESSION_ID, sessionUUID)}.")
classLoader
}

Expand Down
4 changes: 2 additions & 2 deletions core/src/main/scala/org/apache/spark/internal/io/SparkHadoopWriter.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ import org.apache.hadoop.mapreduce.task.{TaskAttemptContextImpl => NewTaskAttemp
import org.apache.spark.{SerializableWritable, SparkConf, SparkException, TaskContext}
import org.apache.spark.deploy.SparkHadoopUtil
import org.apache.spark.internal.{Logging, MDC}
import org.apache.spark.internal.LogKeys.{JOB_ID, TASK_ATTEMPT_ID, TOTAL_TIME}
import org.apache.spark.internal.LogKeys.{DURATION, JOB_ID, TASK_ATTEMPT_ID}
import org.apache.spark.internal.io.FileCommitProtocol.TaskCommitMessage
import org.apache.spark.rdd.{HadoopRDD, RDD}
import org.apache.spark.util.{SerializableConfiguration, SerializableJobConf, Utils}
Expand Down Expand Up @@ -102,7 +102,7 @@ object SparkHadoopWriter extends Logging {
val (_, duration) = Utils
.timeTakenMs { committer.commitJob(jobContext, ret.toImmutableArraySeq) }
logInfo(log"Write Job ${MDC(JOB_ID, jobContext.getJobID)} committed." +
log" Elapsed time: ${MDC(TOTAL_TIME, duration)} ms.")
log" Elapsed time: ${MDC(DURATION, duration)} ms.")
} catch {
case cause: Throwable =>
logError(log"Aborting job ${MDC(JOB_ID, jobContext.getJobID)}.", cause)
Expand Down