-
Notifications
You must be signed in to change notification settings - Fork 29k
[SPARK-14280][BUILD][WIP] Update change-version.sh and pom.xml to add Scala 2.12 profiles and enable 2.12 compilation #18645
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -54,7 +54,10 @@ class TaskContextSuite extends SparkFunSuite with BeforeAndAfter with LocalSpark | |
| val rdd = new RDD[String](sc, List()) { | ||
| override def getPartitions = Array[Partition](StubPartition(0)) | ||
| override def compute(split: Partition, context: TaskContext) = { | ||
| context.addTaskCompletionListener(context => TaskContextSuite.completed = true) | ||
| context.addTaskCompletionListener(new TaskCompletionListener { | ||
|
||
| override def onTaskCompletion(context: TaskContext): Unit = | ||
| TaskContextSuite.completed = true | ||
| }) | ||
| sys.error("failed") | ||
| } | ||
| } | ||
|
|
@@ -95,9 +98,13 @@ class TaskContextSuite extends SparkFunSuite with BeforeAndAfter with LocalSpark | |
| test("all TaskCompletionListeners should be called even if some fail") { | ||
| val context = TaskContext.empty() | ||
| val listener = mock(classOf[TaskCompletionListener]) | ||
| context.addTaskCompletionListener(_ => throw new Exception("blah")) | ||
| context.addTaskCompletionListener(new TaskCompletionListener { | ||
| override def onTaskCompletion(context: TaskContext): Unit = throw new Exception("blah") | ||
| }) | ||
| context.addTaskCompletionListener(listener) | ||
| context.addTaskCompletionListener(_ => throw new Exception("blah")) | ||
| context.addTaskCompletionListener(new TaskCompletionListener { | ||
| override def onTaskCompletion(context: TaskContext): Unit = throw new Exception("blah") | ||
| }) | ||
|
|
||
| intercept[TaskCompletionListenerException] { | ||
| context.markTaskCompleted(None) | ||
|
|
@@ -109,9 +116,15 @@ class TaskContextSuite extends SparkFunSuite with BeforeAndAfter with LocalSpark | |
| test("all TaskFailureListeners should be called even if some fail") { | ||
| val context = TaskContext.empty() | ||
| val listener = mock(classOf[TaskFailureListener]) | ||
| context.addTaskFailureListener((_, _) => throw new Exception("exception in listener1")) | ||
| context.addTaskFailureListener(new TaskFailureListener { | ||
| override def onTaskFailure(context: TaskContext, error: Throwable): Unit = | ||
| throw new Exception("exception in listener1") | ||
| }) | ||
| context.addTaskFailureListener(listener) | ||
| context.addTaskFailureListener((_, _) => throw new Exception("exception in listener3")) | ||
| context.addTaskFailureListener(new TaskFailureListener { | ||
| override def onTaskFailure(context: TaskContext, error: Throwable): Unit = | ||
| throw new Exception("exception in listener3") | ||
| }) | ||
|
|
||
| val e = intercept[TaskCompletionListenerException] { | ||
| context.markTaskFailed(new Exception("exception in task")) | ||
|
|
@@ -232,7 +245,10 @@ class TaskContextSuite extends SparkFunSuite with BeforeAndAfter with LocalSpark | |
| var invocations = 0 | ||
| val context = TaskContext.empty() | ||
| context.markTaskCompleted(None) | ||
| context.addTaskCompletionListener(_ => invocations += 1) | ||
| context.addTaskCompletionListener(new TaskCompletionListener { | ||
| override def onTaskCompletion(context: TaskContext): Unit = | ||
| invocations += 1 | ||
| }) | ||
| assert(invocations == 1) | ||
| context.markTaskCompleted(None) | ||
| assert(invocations == 1) | ||
|
|
@@ -244,10 +260,12 @@ class TaskContextSuite extends SparkFunSuite with BeforeAndAfter with LocalSpark | |
| val error = new RuntimeException | ||
| val context = TaskContext.empty() | ||
| context.markTaskFailed(error) | ||
| context.addTaskFailureListener { (_, e) => | ||
| lastError = e | ||
| invocations += 1 | ||
| } | ||
| context.addTaskFailureListener(new TaskFailureListener { | ||
| override def onTaskFailure(context: TaskContext, e: Throwable): Unit = { | ||
| lastError = e | ||
| invocations += 1 | ||
| } | ||
| }) | ||
| assert(lastError == error) | ||
| assert(invocations == 1) | ||
| context.markTaskFailed(error) | ||
|
|
@@ -267,9 +285,15 @@ class TaskContextSuite extends SparkFunSuite with BeforeAndAfter with LocalSpark | |
| test("all TaskCompletionListeners should be called even if some fail or a task") { | ||
| val context = TaskContext.empty() | ||
| val listener = mock(classOf[TaskCompletionListener]) | ||
| context.addTaskCompletionListener(_ => throw new Exception("exception in listener1")) | ||
| context.addTaskCompletionListener(new TaskCompletionListener { | ||
| override def onTaskCompletion(context: TaskContext): Unit = | ||
| throw new Exception("exception in listener1") | ||
| }) | ||
| context.addTaskCompletionListener(listener) | ||
| context.addTaskCompletionListener(_ => throw new Exception("exception in listener3")) | ||
| context.addTaskCompletionListener(new TaskCompletionListener { | ||
| override def onTaskCompletion(context: TaskContext): Unit = | ||
| throw new Exception("exception in listener3") | ||
| }) | ||
|
|
||
| val e = intercept[TaskCompletionListenerException] { | ||
| context.markTaskCompleted(Some(new Exception("exception in task"))) | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
These cause a MiMa warning, because they're new methods in a trait that might be extended by users. I'll have to go back and remember whether this is an actual problem, because the trait is providing a default implementation here.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I tried compiling a small app that calls
RDD.countAsync(which returns aFutureAction) and even implements a customFutureActionand compiled vs 2.2.0, then ran vs this build, and it worked. I believe this may be legitimately excluded from MiMa.