Address comments

sql/README.md

@@ -61,6 +61,10 @@ import org.apache.spark.sql.execution
 import org.apache.spark.sql.functions._
 import org.apache.spark.sql.hive._
 import org.apache.spark.sql.types._
+sc: org.apache.spark.SparkContext = org.apache.spark.SparkContext@27fc0441
+hc: org.apache.spark.sql.hive.HiveContext = org.apache.spark.sql.hive.HiveContext@127b5be9
+import hc.implicits._
+import hc._
 Type in expressions to have them evaluated.
 Type :help for more information.
 

sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisErrorSuite.scala

@@ -17,14 +17,10 @@
 
 package org.apache.spark.sql.catalyst.analysis
 
-import org.scalatest.BeforeAndAfter
-
-import org.apache.spark.SparkFunSuite
 import org.apache.spark.sql.AnalysisException
 import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.plans.logical._
 import org.apache.spark.sql.catalyst.plans.Inner
-import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.dsl.expressions._
 import org.apache.spark.sql.catalyst.dsl.plans._
 import org.apache.spark.sql.types._
@@ -42,7 +38,7 @@ case class UnresolvedTestPlan() extends LeafNode {
   override def output: Seq[Attribute] = Nil
 }
 
-class AnalysisErrorSuite extends AnalysisTest with BeforeAndAfter {
+class AnalysisErrorSuite extends AnalysisTest {
   import TestRelations._
 
   def errorTest(

sql/core/src/test/java/test/org/apache/spark/sql/JavaDataFrameSuite.java

@@ -34,9 +34,9 @@
 import org.apache.spark.api.java.JavaRDD;
 import org.apache.spark.api.java.JavaSparkContext;
 import org.apache.spark.sql.*;
+import static org.apache.spark.sql.functions.*;
 import org.apache.spark.sql.test.TestSQLContext;
 import org.apache.spark.sql.types.*;
-import static org.apache.spark.sql.functions.*;
 
 public class JavaDataFrameSuite {
   private transient JavaSparkContext jsc;

sql/core/src/test/scala/org/apache/spark/sql/ListTablesSuite.scala

@@ -27,12 +27,17 @@ class ListTablesSuite extends QueryTest with BeforeAndAfter with SharedSQLContex
 
   private lazy val df = (1 to 10).map(i => (i, s"str$i")).toDF("key", "value")
 
-  before {
+  override def beforeAll(): Unit = {
+    super.beforeAll()
     df.registerTempTable("ListTablesSuiteTable")
   }
 
-  after {
-    ctx.catalog.unregisterTable(Seq("ListTablesSuiteTable"))
+  override def afterAll(): Unit = {
+    try {
+      ctx.catalog.unregisterTable(Seq("ListTablesSuiteTable"))
+    } finally {
+      afterAll()
+    }
   }
 
   test("get all tables") {

sql/core/src/test/scala/org/apache/spark/sql/SQLContextSuite.scala

@@ -23,8 +23,11 @@ import org.apache.spark.sql.test.SharedSQLContext
 class SQLContextSuite extends SparkFunSuite with SharedSQLContext {
 
   override def afterAll(): Unit = {
-    SQLContext.setLastInstantiatedContext(ctx)
-    super.afterAll()
+    try {
+      SQLContext.setLastInstantiatedContext(ctx)
+    } finally {
+      super.afterAll()
+    }
   }
 
   test("getOrCreate instantiates SQLContext") {

sql/core/src/test/scala/org/apache/spark/sql/columnar/PartitionBatchPruningSuite.scala

@@ -24,7 +24,7 @@ import org.apache.spark.sql._
 import org.apache.spark.sql.test.SharedSQLContext
 import org.apache.spark.sql.test.SQLTestData._
 
-class PartitionBatchPruningSuite extends SparkFunSuite with SharedSQLContext with BeforeAndAfter {
+class PartitionBatchPruningSuite extends SparkFunSuite with SharedSQLContext {
   import testImplicits._
 
   private lazy val originalColumnBatchSize = ctx.conf.columnBatchSize
@@ -45,20 +45,17 @@ class PartitionBatchPruningSuite extends SparkFunSuite with SharedSQLContext wit
     ctx.setConf(SQLConf.IN_MEMORY_PARTITION_PRUNING, true)
     // Enable in-memory table scan accumulators
     ctx.setConf("spark.sql.inMemoryTableScanStatistics.enable", "true")
-  }
-
-  override protected def afterAll(): Unit = {
-    ctx.setConf(SQLConf.COLUMN_BATCH_SIZE, originalColumnBatchSize)
-    ctx.setConf(SQLConf.IN_MEMORY_PARTITION_PRUNING, originalInMemoryPartitionPruning)
-    super.afterAll()
-  }
-
-  before {
     ctx.cacheTable("pruningData")
   }
 
-  after {
-    ctx.uncacheTable("pruningData")
+  override protected def afterAll(): Unit = {
+    try {
+      ctx.setConf(SQLConf.COLUMN_BATCH_SIZE, originalColumnBatchSize)
+      ctx.setConf(SQLConf.IN_MEMORY_PARTITION_PRUNING, originalInMemoryPartitionPruning)
+      ctx.uncacheTable("pruningData")
+    } finally {
+      super.afterAll()
+    }
   }
 
   // Comparisons

sql/core/src/test/scala/org/apache/spark/sql/execution/TungstenSortSuite.scala

@@ -36,8 +36,11 @@ class TungstenSortSuite extends SparkPlanTest with SharedSQLContext {
   }
 
   override def afterAll(): Unit = {
-    ctx.conf.setConf(SQLConf.CODEGEN_ENABLED, SQLConf.CODEGEN_ENABLED.defaultValue.get)
-    super.afterAll()
+    try {
+      ctx.conf.setConf(SQLConf.CODEGEN_ENABLED, SQLConf.CODEGEN_ENABLED.defaultValue.get)
+    } finally {
+      super.afterAll()
+    }
   }
 
   test("sort followed by limit") {

sql/core/src/test/scala/org/apache/spark/sql/sources/CreateTableAsSelectSuite.scala

@@ -19,15 +19,13 @@ package org.apache.spark.sql.sources
 
 import java.io.{File, IOException}
 
-import org.scalatest.BeforeAndAfter
-
 import org.apache.spark.sql.AnalysisException
 import org.apache.spark.sql.execution.datasources.DDLException
 import org.apache.spark.sql.test.SharedSQLContext
 import org.apache.spark.util.Utils
 
 
-class CreateTableAsSelectSuite extends DataSourceTest with SharedSQLContext with BeforeAndAfter {
+class CreateTableAsSelectSuite extends DataSourceTest with SharedSQLContext {
   private lazy val sparkContext = caseInsensitiveContext.sparkContext
   private var path: File = null
 
@@ -39,12 +37,12 @@ class CreateTableAsSelectSuite extends DataSourceTest with SharedSQLContext with
   }
 
   override def afterAll(): Unit = {
-    caseInsensitiveContext.dropTempTable("jt")
-    super.afterAll()
-  }
-
-  after {
-    Utils.deleteRecursively(path)
+    try {
+      caseInsensitiveContext.dropTempTable("jt")
+      Utils.deleteRecursively(path)
+    } finally {
+      super.afterAll()
+    }
   }
 
   test("CREATE TEMPORARY TABLE AS SELECT") {

sql/core/src/test/scala/org/apache/spark/sql/sources/DDLTestSuite.scala

@@ -71,9 +71,10 @@ case class SimpleDDLScan(from: Int, to: Int, table: String)(@transient val sqlCo
   }
 }
 
-class DDLTestSuite extends DataSourceTest with SharedSQLContext with BeforeAndAfter {
+class DDLTestSuite extends DataSourceTest with SharedSQLContext {
 
-  before {
+  override def beforeAll(): Unit = {
+    super.beforeAll()
     caseInsensitiveContext.sql(
       """
       |CREATE TEMPORARY TABLE ddlPeople

sql/core/src/test/scala/org/apache/spark/sql/sources/FilteredScanSuite.scala

@@ -19,8 +19,6 @@ package org.apache.spark.sql.sources
 
 import scala.language.existentials
 
-import org.scalatest.BeforeAndAfter
-
 import org.apache.spark.rdd.RDD
 import org.apache.spark.sql._
 import org.apache.spark.sql.test.SharedSQLContext
@@ -99,9 +97,10 @@ object FiltersPushed {
   var list: Seq[Filter] = Nil
 }
 
-class FilteredScanSuite extends DataSourceTest with SharedSQLContext with BeforeAndAfter {
+class FilteredScanSuite extends DataSourceTest with SharedSQLContext {
 
-  before {
+  override def beforeAll(): Unit = {
+    super.beforeAll()
     caseInsensitiveContext.sql(
       """
         |CREATE TEMPORARY TABLE oneToTenFiltered

sql/core/src/test/scala/org/apache/spark/sql/sources/InsertSuite.scala

@@ -43,10 +43,13 @@ class InsertSuite extends DataSourceTest with SharedSQLContext {
   }
 
   override def afterAll(): Unit = {
-    caseInsensitiveContext.dropTempTable("jsonTable")
-    caseInsensitiveContext.dropTempTable("jt")
-    Utils.deleteRecursively(path)
-    super.afterAll()
+    try {
+      caseInsensitiveContext.dropTempTable("jsonTable")
+      caseInsensitiveContext.dropTempTable("jt")
+      Utils.deleteRecursively(path)
+    } finally {
+      super.afterAll()
+    }
   }
 
   test("Simple INSERT OVERWRITE a JSONRelation") {

sql/core/src/test/scala/org/apache/spark/sql/sources/PrunedScanSuite.scala

@@ -19,8 +19,6 @@ package org.apache.spark.sql.sources
 
 import scala.language.existentials
 
-import org.scalatest.BeforeAndAfter
-
 import org.apache.spark.rdd.RDD
 import org.apache.spark.sql._
 import org.apache.spark.sql.test.SharedSQLContext
@@ -54,9 +52,10 @@ case class SimplePrunedScan(from: Int, to: Int)(@transient val sqlContext: SQLCo
   }
 }
 
-class PrunedScanSuite extends DataSourceTest with SharedSQLContext with BeforeAndAfter {
+class PrunedScanSuite extends DataSourceTest with SharedSQLContext {
 
-  before {
+  override def beforeAll(): Unit = {
+    super.beforeAll()
     caseInsensitiveContext.sql(
       """
         |CREATE TEMPORARY TABLE oneToTenPruned

sql/core/src/test/scala/org/apache/spark/sql/sources/SaveLoadSuite.scala

@@ -19,14 +19,12 @@ package org.apache.spark.sql.sources
 
 import java.io.File
 
-import org.scalatest.BeforeAndAfter
-
 import org.apache.spark.sql.{AnalysisException, SaveMode, SQLConf, DataFrame}
 import org.apache.spark.sql.test.SharedSQLContext
 import org.apache.spark.sql.types._
 import org.apache.spark.util.Utils
 
-class SaveLoadSuite extends DataSourceTest with SharedSQLContext with BeforeAndAfter {
+class SaveLoadSuite extends DataSourceTest with SharedSQLContext {
   private lazy val sparkContext = caseInsensitiveContext.sparkContext
   private var originalDefaultSource: String = null
   private var path: File = null
@@ -45,13 +43,12 @@ class SaveLoadSuite extends DataSourceTest with SharedSQLContext with BeforeAndA
   }
 
   override def afterAll(): Unit = {
-    caseInsensitiveContext.conf.setConf(SQLConf.DEFAULT_DATA_SOURCE_NAME, originalDefaultSource)
-    super.afterAll()
-  }
-
-  after {
-    caseInsensitiveContext.conf.setConf(SQLConf.DEFAULT_DATA_SOURCE_NAME, originalDefaultSource)
-    Utils.deleteRecursively(path)
+    try {
+      caseInsensitiveContext.conf.setConf(SQLConf.DEFAULT_DATA_SOURCE_NAME, originalDefaultSource)
+      Utils.deleteRecursively(path)
+    } finally {
+      super.afterAll()
+    }
   }
 
   def checkLoad(expectedDF: DataFrame = df, tbl: String = "jsonTable"): Unit = {

sql/core/src/test/scala/org/apache/spark/sql/sources/TableScanSuite.scala

@@ -20,8 +20,6 @@ package org.apache.spark.sql.sources
 import java.nio.charset.StandardCharsets
 import java.sql.{Date, Timestamp}
 
-import org.scalatest.BeforeAndAfter
-
 import org.apache.spark.rdd.RDD
 import org.apache.spark.sql._
 import org.apache.spark.sql.test.SharedSQLContext
@@ -98,7 +96,7 @@ case class AllDataTypesScan(
   }
 }
 
-class TableScanSuite extends DataSourceTest with SharedSQLContext with BeforeAndAfter {
+class TableScanSuite extends DataSourceTest with SharedSQLContext {
   private lazy val tableWithSchemaExpected = (1 to 10).map { i =>
     Row(
       s"str_$i",
@@ -123,7 +121,8 @@ class TableScanSuite extends DataSourceTest with SharedSQLContext with BeforeAnd
       Row(Seq(s"str_$i", s"str_${i + 1}"), Row(Seq(Date.valueOf(s"1970-01-${i + 1}")))))
   }.toSeq
 
-  before {
+  override def beforeAll(): Unit = {
+    super.beforeAll()
     caseInsensitiveContext.sql(
       """
         |CREATE TEMPORARY TABLE oneToTen

sql/core/src/test/scala/org/apache/spark/sql/test/SharedSQLContext.scala

@@ -48,6 +48,7 @@ private[sql] trait SharedSQLContext extends SQLTestUtils {
     if (_ctx == null) {
       _ctx = new TestSQLContext
     }
+    // Ensure we have initialized the context before calling parent code
     super.beforeAll()
   }
 

sql/hive/src/test/java/org/apache/spark/sql/hive/test/SharedHiveContext.scala

@@ -49,6 +49,8 @@ private[spark] trait SharedHiveContext extends SQLTestUtils {
     if (_ctx == null) {
       _ctx = new TestHiveContext
     }
+    // Ensure we have initialized the context before calling parent code
+    super.beforeAll()
   }
 
   /**

sql/hive/src/test/java/test/org/apache/spark/sql/hive/JavaDataFrameSuite.java

@@ -30,10 +30,10 @@
 import org.apache.spark.sql.*;
 import org.apache.spark.sql.expressions.Window;
 import org.apache.spark.sql.expressions.UserDefinedAggregateFunction;
+import static org.apache.spark.sql.functions.*;
 import org.apache.spark.sql.hive.HiveContext;
 import org.apache.spark.sql.hive.test.TestHiveContext;
 import test.org.apache.spark.sql.hive.aggregate.MyDoubleSum;
-import static org.apache.spark.sql.functions.*;
 
 public class JavaDataFrameSuite {
   private transient JavaSparkContext sc;

sql/hive/src/test/scala/org/apache/spark/sql/hive/ErrorPositionSuite.scala

@@ -29,7 +29,8 @@ import org.apache.spark.sql.{AnalysisException, QueryTest}
 class ErrorPositionSuite extends QueryTest with BeforeAndAfter with SharedHiveContext {
   import testImplicits._
 
-  before {
+  override def beforeAll(): Unit = {
+    super.beforeAll()
     Seq((1, 1, 1)).toDF("a", "a", "b").registerTempTable("dupAttributes")
   }
 

sql/hive/src/test/scala/org/apache/spark/sql/hive/InsertIntoHiveTableSuite.scala

@@ -29,12 +29,12 @@ import org.apache.spark.sql.test.SQLTestData.TestData
 import org.apache.spark.sql.types._
 import org.apache.spark.util.Utils
 
-case class ThreeCloumntable(key: Int, value: String, key1: String)
+case class ThreeColumnTable(key: Int, value: String, key1: String)
 
 class InsertIntoHiveTableSuite extends QueryTest with BeforeAndAfter with SharedHiveContext {
   import testImplicits._
 
-  private val _testData = ctx.sparkContext.parallelize(
+  private lazy val _testData = ctx.sparkContext.parallelize(
     (1 to 100).map(i => TestData(i, i.toString))).toDF()
 
   before {
@@ -217,7 +217,7 @@ class InsertIntoHiveTableSuite extends QueryTest with BeforeAndAfter with Shared
     testData.registerTempTable("testData")
 
     val testDatawithNull = ctx.sparkContext.parallelize(
-      (1 to 10).map(i => ThreeCloumntable(i, i.toString, null))).toDF()
+      (1 to 10).map(i => ThreeColumnTable(i, i.toString, null))).toDF()
 
     val tmpDir = Utils.createTempDir()
     ctx.sql(