Skip to content
Prev Previous commit
Next Next commit
Extend SqlBasedBenchmark by OrcReadBenchmark
  • Loading branch information
MaxGekk committed Sep 18, 2019
commit 9267efc1dcfea5c77ce4def92fa26d152d93323d
Original file line number Diff line number Diff line change
Expand Up @@ -22,9 +22,9 @@ import java.io.File
import scala.util.Random

import org.apache.spark.SparkConf
import org.apache.spark.benchmark.{Benchmark, BenchmarkBase}
import org.apache.spark.benchmark.Benchmark
import org.apache.spark.sql.{DataFrame, SparkSession}
import org.apache.spark.sql.catalyst.plans.SQLHelper
import org.apache.spark.sql.execution.benchmark.SqlBasedBenchmark
import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.types._

Expand All @@ -43,18 +43,23 @@ import org.apache.spark.sql.types._
* This is in `sql/hive` module in order to compare `sql/core` and `sql/hive` ORC data sources.
*/
// scalastyle:off line.size.limit
object OrcReadBenchmark extends BenchmarkBase with SQLHelper {
val conf = new SparkConf()
conf.set("orc.compression", "snappy")

private val spark = SparkSession.builder()
.master("local[1]")
.appName("OrcReadBenchmark")
.config(conf)
.getOrCreate()

// Set default configs. Individual cases will change them if necessary.
spark.conf.set(SQLConf.ORC_FILTER_PUSHDOWN_ENABLED.key, "true")
object OrcReadBenchmark extends SqlBasedBenchmark {

override def getSparkSession: SparkSession = {
val conf = new SparkConf()
conf.set("orc.compression", "snappy")

val sparkSession = SparkSession.builder()
.master("local[1]")
.appName("OrcReadBenchmark")
.config(conf)
.getOrCreate()

// Set default configs. Individual cases will change them if necessary.
sparkSession.conf.set(SQLConf.ORC_FILTER_PUSHDOWN_ENABLED.key, "true")

sparkSession
}

def withTempTable(tableNames: String*)(f: => Unit): Unit = {
try f finally tableNames.foreach(spark.catalog.dropTempView)
Expand Down