Skip to content
Closed
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
27 commits
Select commit Hold shift + click to select a range
4dcea38
Move withSystemProperty to TestUtils class.
JoshRosen Dec 19, 2014
9e3e0dd
Add ResetSystemProperties test fixture mixin; use it in SparkSubmitSu…
JoshRosen Dec 19, 2014
628f46c
Use ResetSystemProperties in DistributedSuite
JoshRosen Dec 19, 2014
14a92e4
Use withSystemProperty in FileServerSuite
JoshRosen Dec 19, 2014
60a63a1
Use ResetSystemProperties in JobCancellationSuite
JoshRosen Dec 19, 2014
51aa870
Use withSystemProperty in ShuffleSuite
JoshRosen Dec 19, 2014
c83ded8
Use ResetSystemProperties in SparkConfSuite
JoshRosen Dec 19, 2014
0995c4b
Use ResetSystemProperties in SparkContextSchedulerCreationSuite
JoshRosen Dec 19, 2014
5b3cb54
Use ResetSystemProperties in SparkListenerSuite
JoshRosen Dec 19, 2014
e9ded62
Use ResetSystemProperties in TaskSchedulerImplSuite
JoshRosen Dec 19, 2014
b0daff2
Use ResetSystemProperties in BlockManagerSuite
JoshRosen Dec 19, 2014
dd9492b
Use ResetSystemProperties in AkkaUtilsSuite
JoshRosen Dec 19, 2014
1d1aa5a
Use ResetSystemProperties in SizeEstimatorSuite
JoshRosen Dec 19, 2014
25bfce2
Use ResetSystemProperties in UtilsSuite
JoshRosen Dec 19, 2014
633a84a
Remove use of system properties in FileServerSuite
JoshRosen Dec 24, 2014
8783ab0
Remove TestUtils.setSystemProperty, since it is subsumed by the Reset…
JoshRosen Dec 24, 2014
cfe9cce
Remove use of system properties in SparkContextSuite
JoshRosen Dec 24, 2014
3f2f955
Remove System.setProperty calls in DistributedSuite
JoshRosen Dec 24, 2014
655587c
Remove setProperty calls in JobCancellationSuite
JoshRosen Dec 24, 2014
bee20df
Remove setProperty calls in SparkContextSchedulerCreationSuite
JoshRosen Dec 24, 2014
3fdb554
Remove setProperty call in TaskSchedulerImplSuite
JoshRosen Dec 24, 2014
7a3d224
Fix trait ordering
JoshRosen Dec 24, 2014
0eaf0b6
Remove setProperty call in TaskResultGetterSuite.
JoshRosen Dec 24, 2014
4742a5b
Clarify ResetSystemProperties trait inheritance ordering.
JoshRosen Dec 25, 2014
4f4031d
Add note on why SparkSubmitSuite needs ResetSystemProperties
JoshRosen Dec 25, 2014
3888fe3
Remove setProperty use in LocalJavaStreamingContext
JoshRosen Dec 25, 2014
0236d66
Replace setProperty uses in two example programs / tools
JoshRosen Dec 25, 2014
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Clarify ResetSystemProperties trait inheritance ordering.
  • Loading branch information
JoshRosen committed Dec 25, 2014
commit 4742a5bfc76f06d6d01eedf188a9ef6147148735
Original file line number Diff line number Diff line change
Expand Up @@ -28,8 +28,8 @@ import org.apache.spark.{LocalSparkContext, SparkContext}
import org.apache.spark.executor.TaskMetrics
import org.apache.spark.util.ResetSystemProperties

class SparkListenerSuite extends FunSuite with ResetSystemProperties with LocalSparkContext
with Matchers with BeforeAndAfter with BeforeAndAfterAll {
class SparkListenerSuite extends FunSuite with LocalSparkContext with Matchers with BeforeAndAfter
with BeforeAndAfterAll with ResetSystemProperties {

/** Length of time to wait while draining listener events. */
val WAIT_TIMEOUT_MILLIS = 10000
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ import akka.util.Timeout

import org.mockito.Mockito.{mock, when}

import org.scalatest.{BeforeAndAfter, FunSuite, Matchers, PrivateMethodTester}
import org.scalatest._
import org.scalatest.concurrent.Eventually._
import org.scalatest.concurrent.Timeouts._

Expand All @@ -47,8 +47,8 @@ import org.apache.spark.storage.BlockManagerMessages.BlockManagerHeartbeat
import org.apache.spark.util._


class BlockManagerSuite extends FunSuite with ResetSystemProperties with Matchers
with BeforeAndAfter with PrivateMethodTester {
class BlockManagerSuite extends FunSuite with Matchers with BeforeAndAfterEach
with PrivateMethodTester with ResetSystemProperties {

private val conf = new SparkConf(false)
var store: BlockManager = null
Expand Down Expand Up @@ -78,7 +78,7 @@ class BlockManagerSuite extends FunSuite with ResetSystemProperties with Matcher
manager
}

before {
override def beforeEach(): Unit = {
val (actorSystem, boundPort) = AkkaUtils.createActorSystem(
"test", "localhost", 0, conf = conf, securityManager = securityMgr)
this.actorSystem = actorSystem
Expand All @@ -99,7 +99,7 @@ class BlockManagerSuite extends FunSuite with ResetSystemProperties with Matcher
SizeEstimator invokePrivate initialize()
}

after {
override def afterEach(): Unit = {
if (store != null) {
store.stop()
store = null
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ import org.apache.spark.storage.BlockManagerId
/**
* Test the AkkaUtils with various security settings.
*/
class AkkaUtilsSuite extends FunSuite with ResetSystemProperties with LocalSparkContext {
class AkkaUtilsSuite extends FunSuite with LocalSparkContext with ResetSystemProperties {

test("remote fetch security bad password") {
val conf = new SparkConf
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,23 +19,39 @@ package org.apache.spark.util

import java.util.Properties

import org.scalatest.{Suite, SuiteMixin}

import org.scalatest.{BeforeAndAfterEach, Suite}

/**
* Mixin for automatically resetting system properties that are modified in ScalaTest tests.
* This resets the properties after each individual test.
*
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Note to self: I should probably add a comment here explaining the ordering concerns between ResetSystemProperties and other traits that implement withFixture. For example, if you're also using BeforeAndAfterEach to modify properties, then you should order the mixins like with ResetSystemProperties with BeforeAndAfterEach so that this mixin can do the final cleanup.

* The order in which fixtures are mixed in affects the order in which they are invoked by tests.
* If we have a suite `MySuite extends FunSuite with Foo with Bar`, then
* Bar's `super` is Foo, so Bar's beforeEach() will and afterEach() methods will be invoked first
* by the rest runner.
*
* This means that ResetSystemProperties should appear as the last trait in test suites that it's
* mixed into in order to ensure that the system properties snapshot occurs as early as possible.
* ResetSystemProperties calls super.afterEach() before performing its own cleanup, ensuring that
* the old properties are restored as late as possible.
*
* See the "Composing fixtures by stacking traits" section at
* http://www.scalatest.org/user_guide/sharing_fixtures for more details about this pattern.
*/
private[spark] trait ResetSystemProperties extends SuiteMixin { this: Suite =>
abstract override def withFixture(test: NoArgTest) = {
val oldProperties = new Properties(System.getProperties)
private[spark] trait ResetSystemProperties extends BeforeAndAfterEach { this: Suite =>
var oldProperties: Properties = null

override def beforeEach(): Unit = {
oldProperties = new Properties(System.getProperties)
super.beforeEach()
}

override def afterEach(): Unit = {
try {
super.withFixture(test)
super.afterEach()
} finally {
System.setProperties(oldProperties)
oldProperties = null
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ class DummyString(val arr: Array[Char]) {
}

class SizeEstimatorSuite
extends FunSuite with ResetSystemProperties with BeforeAndAfterEach with PrivateMethodTester {
extends FunSuite with BeforeAndAfterEach with PrivateMethodTester with ResetSystemProperties {

override def beforeEach() {
// Set the arch to 64-bit and compressedOops to true to get a deterministic test-case
Expand Down