Skip to content
Closed
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Limit scope of mutable set
  • Loading branch information
Andrew Or committed Sep 13, 2015
commit 35bb6f08befdab26edf5535f27f0b6a405f142f2
9 changes: 7 additions & 2 deletions core/src/main/scala/org/apache/spark/SparkContext.scala
Original file line number Diff line number Diff line change
Expand Up @@ -361,10 +361,15 @@ class SparkContext(config: SparkConf) extends Logging with ExecutorAllocationCli
override protected def initialValue(): Properties = new Properties()
}

// Keys of local properties that should not be inherited by children threads
private val nonInheritedLocalProperties: HashSet[String] = new HashSet[String]

/**
* Keys of local properties that should not be inherited by children threads.
* Mark a local property such that its values are never inherited across the thread hierarchy.
*/
private[spark] val nonInheritedLocalProperties: HashSet[String] = new HashSet[String]
private[spark] def markLocalPropertyNonInherited(key: String): Unit = {
nonInheritedLocalProperties += key
}

/* ------------------------------------------------------------------------------------- *
| Initialization. This code initializes the context in a manner that is exception-safe. |
Expand Down
2 changes: 1 addition & 1 deletion core/src/test/scala/org/apache/spark/ThreadingSuite.scala
Original file line number Diff line number Diff line change
Expand Up @@ -214,7 +214,7 @@ class ThreadingSuite extends SparkFunSuite with LocalSparkContext with Logging {

test("inheritance exclusions (SPARK-10548)") {
sc = new SparkContext("local", "test")
sc.nonInheritedLocalProperties.add("do-not-inherit-me")
sc.markLocalPropertyNonInherited("do-not-inherit-me")
sc.setLocalProperty("do-inherit-me", "parent")
sc.setLocalProperty("do-not-inherit-me", "parent")
var throwable: Option[Throwable] = None
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,7 @@ class SQLContext(@transient val sparkContext: SparkContext)
// Ensure query execution IDs are not inherited across the thread hierarchy, which is
// the default behavior for SparkContext local properties. Otherwise, we may confuse
// the listener as to which query is being executed. (SPARK-10548)
sparkContext.nonInheritedLocalProperties.add(SQLExecution.EXECUTION_ID_KEY)
sparkContext.markLocalPropertyNonInherited(SQLExecution.EXECUTION_ID_KEY)

/**
* Set Spark SQL configuration properties.
Expand Down