Skip to content

Commit 8b52240

Browse files
shahidki31Marcelo Vanzin
authored andcommitted
[SPARK-27145][MINOR] Close store in the SQLAppStatusListenerSuite after test
## What changes were proposed in this pull request? We create many stores in the SQLAppStatusListenerSuite, but we need to the close store after test. ## How was this patch tested? Existing tests Closes #24079 from shahidki31/SPARK-27145. Authored-by: Shahid <shahidki31@gmail.com> Signed-off-by: Marcelo Vanzin <vanzin@cloudera.com>
1 parent da7db9a commit 8b52240

File tree

2 files changed

+33
-11
lines changed

2 files changed

+33
-11
lines changed

sql/core/src/test/scala/org/apache/spark/sql/execution/ui/AllExecutionsPageSuite.scala

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ import javax.servlet.http.HttpServletRequest
2424
import scala.xml.Node
2525

2626
import org.mockito.Mockito.{mock, when, RETURNS_SMART_NULLS}
27+
import org.scalatest.BeforeAndAfter
2728

2829
import org.apache.spark.scheduler.{JobFailed, SparkListenerJobEnd, SparkListenerJobStart}
2930
import org.apache.spark.sql.DataFrame
@@ -32,10 +33,19 @@ import org.apache.spark.sql.test.SharedSQLContext
3233
import org.apache.spark.status.ElementTrackingStore
3334
import org.apache.spark.util.kvstore.InMemoryStore
3435

35-
class AllExecutionsPageSuite extends SharedSQLContext {
36+
class AllExecutionsPageSuite extends SharedSQLContext with BeforeAndAfter {
3637

3738
import testImplicits._
3839

40+
var kvstore: ElementTrackingStore = _
41+
42+
after {
43+
if (kvstore != null) {
44+
kvstore.close()
45+
kvstore = null
46+
}
47+
}
48+
3949
test("SPARK-27019: correctly display SQL page when event reordering happens") {
4050
val statusStore = createStatusStore
4151
val tab = mock(classOf[SQLTab], RETURNS_SMART_NULLS)
@@ -70,9 +80,9 @@ class AllExecutionsPageSuite extends SharedSQLContext {
7080

7181
private def createStatusStore: SQLAppStatusStore = {
7282
val conf = sparkContext.conf
73-
val store = new ElementTrackingStore(new InMemoryStore, conf)
74-
val listener = new SQLAppStatusListener(conf, store, live = true)
75-
new SQLAppStatusStore(store, Some(listener))
83+
kvstore = new ElementTrackingStore(new InMemoryStore, conf)
84+
val listener = new SQLAppStatusListener(conf, kvstore, live = true)
85+
new SQLAppStatusStore(kvstore, Some(listener))
7686
}
7787

7888
private def createTestDataFrame: DataFrame = {

sql/core/src/test/scala/org/apache/spark/sql/execution/ui/SQLAppStatusListenerSuite.scala

Lines changed: 19 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ import java.util.Properties
2222
import scala.collection.mutable.ListBuffer
2323

2424
import org.json4s.jackson.JsonMethods._
25+
import org.scalatest.BeforeAndAfter
2526

2627
import org.apache.spark._
2728
import org.apache.spark.LocalSparkContext._
@@ -43,13 +44,24 @@ import org.apache.spark.util.{AccumulatorMetadata, JsonProtocol, LongAccumulator
4344
import org.apache.spark.util.kvstore.InMemoryStore
4445

4546

46-
class SQLAppStatusListenerSuite extends SparkFunSuite with SharedSQLContext with JsonTestUtils {
47+
class SQLAppStatusListenerSuite extends SparkFunSuite with SharedSQLContext with JsonTestUtils
48+
with BeforeAndAfter {
49+
4750
import testImplicits._
4851

4952
override protected def sparkConf = {
5053
super.sparkConf.set(LIVE_ENTITY_UPDATE_PERIOD, 0L).set(ASYNC_TRACKING_ENABLED, false)
5154
}
5255

56+
private var kvstore: ElementTrackingStore = _
57+
58+
after {
59+
if (kvstore != null) {
60+
kvstore.close()
61+
kvstore = null
62+
}
63+
}
64+
5365
private def createTestDataFrame: DataFrame = {
5466
Seq(
5567
(1, 1),
@@ -126,9 +138,9 @@ class SQLAppStatusListenerSuite extends SparkFunSuite with SharedSQLContext with
126138

127139
private def createStatusStore(): SQLAppStatusStore = {
128140
val conf = sparkContext.conf
129-
val store = new ElementTrackingStore(new InMemoryStore, conf)
130-
val listener = new SQLAppStatusListener(conf, store, live = true)
131-
new SQLAppStatusStore(store, Some(listener))
141+
kvstore = new ElementTrackingStore(new InMemoryStore, conf)
142+
val listener = new SQLAppStatusListener(conf, kvstore, live = true)
143+
new SQLAppStatusStore(kvstore, Some(listener))
132144
}
133145

134146
test("basic") {
@@ -548,9 +560,9 @@ class SQLAppStatusListenerSuite extends SparkFunSuite with SharedSQLContext with
548560

549561
test("eviction should respect execution completion time") {
550562
val conf = sparkContext.conf.clone().set(UI_RETAINED_EXECUTIONS.key, "2")
551-
val store = new ElementTrackingStore(new InMemoryStore, conf)
552-
val listener = new SQLAppStatusListener(conf, store, live = true)
553-
val statusStore = new SQLAppStatusStore(store, Some(listener))
563+
kvstore = new ElementTrackingStore(new InMemoryStore, conf)
564+
val listener = new SQLAppStatusListener(conf, kvstore, live = true)
565+
val statusStore = new SQLAppStatusStore(kvstore, Some(listener))
554566

555567
var time = 0
556568
val df = createTestDataFrame

0 commit comments

Comments
 (0)