Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,19 @@ case class MessageWithContext(message: String, context: java.util.HashMap[String
resultMap.putAll(mdc.context)
MessageWithContext(message + mdc.message, resultMap)
}

def ++(s: String): MessageWithContext = {
MessageWithContext(message + s, context)
}
}

object MDC {

implicit class StringImprovements(val s: String) {
def ++(mdc: MessageWithContext): MessageWithContext = {
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The method name cannot be called + and does not take effect, so it is called ++

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

In addition, in String ++ String, ++ is an alias for contact, which sounds reasonable.
image

MessageWithContext(s + mdc.message, mdc.context)
}
}
}

/**
Expand Down Expand Up @@ -107,7 +120,6 @@ trait Logging {
if (Logging.isStructuredLoggingEnabled) {
context.put(mdc.key.toString.toLowerCase(Locale.ROOT), mdc.value.toString)
}

if (processedParts.hasNext) {
sb.append(processedParts.next())
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,14 @@ class PatternLoggingSuite extends LoggingSuiteBase with BeforeAndAfterAll {
override def expectedPatternForMsgWithMDCAndException(level: Level): String =
s""".*$level $className: Error in executor 1.\njava.lang.RuntimeException: OOM\n[\\s\\S]*"""

override def expectedPatternForConcatStringAndMDC(level: Level): String = {
s""".*$level $className: Hello This is a log message, Lost executor 1.\n"""
}

override def expectedPatternForConcatMDCAndString(level: Level): String = {
s""".*$level $className: Lost executor 1. Hello This is a log message.\n"""
}

override def verifyMsgWithConcat(level: Level, logOutput: String): Unit = {
val pattern =
s""".*$level $className: Min Size: 2, Max Size: 4. Please double check.\n"""
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ import org.scalatest.funsuite.AnyFunSuite // scalastyle:ignore funsuite

import org.apache.spark.internal.{LogEntry, Logging, MDC}
import org.apache.spark.internal.LogKey.{EXECUTOR_ID, MAX_SIZE, MIN_SIZE}
import org.apache.spark.internal.MDC._

trait LoggingSuiteBase
extends AnyFunSuite // scalastyle:ignore funsuite
Expand Down Expand Up @@ -62,6 +63,12 @@ trait LoggingSuiteBase
log"Max Size: ${MDC(MAX_SIZE, "4")}. " +
log"Please double check."

def concatMDCAndString: LogEntry = log"Lost executor ${MDC(EXECUTOR_ID, "1")}." ++
s" Hello $basicMsg."

def concatStringAndMDC: LogEntry = s"Hello $basicMsg, " ++
log"Lost executor ${MDC(EXECUTOR_ID, "1")}."

// test for basic message (without any mdc)
def expectedPatternForBasicMsg(level: Level): String

Expand All @@ -74,6 +81,13 @@ trait LoggingSuiteBase
// test for message and exception
def expectedPatternForMsgWithMDCAndException(level: Level): String

// test for concat string and MDC
def expectedPatternForConcatStringAndMDC(level: Level): String

// test for concat MDC and string
def expectedPatternForConcatMDCAndString(level: Level): String

// test for concat MDC and MDC
def verifyMsgWithConcat(level: Level, logOutput: String): Unit

test("Basic logging") {
Expand Down Expand Up @@ -130,6 +144,28 @@ trait LoggingSuiteBase
verifyMsgWithConcat(level, logOutput)
}
}

test("Logging concat string and MDC") {
Seq(
(Level.ERROR, () => logError(concatStringAndMDC)),
(Level.WARN, () => logWarning(concatStringAndMDC)),
(Level.INFO, () => logInfo(concatStringAndMDC))).foreach {
case (level, logFunc) =>
val logOutput = captureLogOutput(logFunc)
assert(expectedPatternForConcatStringAndMDC(level).r.matches(logOutput))
}
}

test("Logging concat MDC and string") {
Seq(
(Level.ERROR, () => logError(concatMDCAndString)),
(Level.WARN, () => logWarning(concatMDCAndString)),
(Level.INFO, () => logInfo(concatMDCAndString))).foreach {
case (level, logFunc) =>
val logOutput = captureLogOutput(logFunc)
assert(expectedPatternForConcatMDCAndString(level).r.matches(logOutput))
}
}
}

class StructuredLoggingSuite extends LoggingSuiteBase {
Expand Down Expand Up @@ -204,6 +240,34 @@ class StructuredLoggingSuite extends LoggingSuiteBase {
}""")
}

override def expectedPatternForConcatStringAndMDC(level: Level): String = {
compactAndToRegexPattern(
s"""
{
"ts": "<timestamp>",
"level": "$level",
"msg": "Hello This is a log message, Lost executor 1.",
"context": {
"executor_id": "1"
},
"logger": "$className"
}""")
}

override def expectedPatternForConcatMDCAndString(level: Level): String = {
compactAndToRegexPattern(
s"""
{
"ts": "<timestamp>",
"level": "$level",
"msg": "Lost executor 1. Hello This is a log message.",
"context": {
"executor_id": "1"
},
"logger": "$className"
}""")
}

override def verifyMsgWithConcat(level: Level, logOutput: String): Unit = {
val pattern1 = compactAndToRegexPattern(
s"""
Expand Down