Skip to content
Closed
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
58 commits
Select commit Hold shift + click to select a range
16ca7ac
It compiles!!
Aug 4, 2015
5e6a20c
Refactor SQLTestUtils to reduce duplication
Aug 5, 2015
d1d1449
Remove HiveTest singleton
Aug 5, 2015
d4aafb1
Avoid the need to switch to HiveContexts
Aug 5, 2015
6345cee
Clean up JsonSuite
Aug 5, 2015
68ac6fe
Rename the test traits properly
Aug 5, 2015
b15fdc6
Stop SparkContext in Java SQL tests
Aug 5, 2015
0d74a72
Load test data early in case tables are accessed by name
Aug 5, 2015
eee415d
Refactor implicits into SQLTestUtils
Aug 10, 2015
55d0b1b
Fix Java not serializable exception in tests
Aug 10, 2015
4f59bee
Fix DataSourceTest et al.
Aug 10, 2015
88d4f16
Fix hive tests to use the same pattern
Aug 11, 2015
5fe4bfb
Merge branch 'master' of github.com:apache/spark into sql-tests-refactor
Aug 11, 2015
c4a22bc
Fix compile after resolving merge conflicts
Aug 11, 2015
d9a8390
Merge branch 'master' of github.com:apache/spark into sql-tests-refactor
Aug 11, 2015
f5619f8
Fix test compile after resolving merge conflicts
Aug 11, 2015
9395cfa
Merge branch 'master' of github.com:apache/spark into sql-tests-refactor
Aug 11, 2015
c51b3d8
Fix OuterJoinSuite
Aug 11, 2015
997715e
Merge branch 'master' of github.com:apache/spark into sql-tests-refactor
Aug 11, 2015
1400770
Fix style
Aug 11, 2015
c92a3b0
Fix MiMa
Aug 11, 2015
4debedf
Clean up inheritance in test util traits
Aug 11, 2015
ca87dda
Merge branch 'master' of github.com:apache/spark into sql-tests-refactor
Aug 11, 2015
1cf53ad
Create new context in SBT console by default
Aug 11, 2015
bc5c999
Fix SemiJoinSuite
Aug 12, 2015
19fd6c3
Fix InnerJoinSuite
Aug 12, 2015
e1e513e
Merge branch 'master' of github.com:apache/spark into sql-tests-refactor
Aug 12, 2015
1e4c321
Address comments
Aug 12, 2015
54848e2
Merge branch 'master' of github.com:apache/spark into sql-tests-refactor
Aug 12, 2015
94f9c77
Revert the removal of some BeforeAndAfters
Aug 12, 2015
bec7d28
Fix a hive test + minor format updates
Aug 12, 2015
24c086d
Merge branch 'master' of github.com:apache/spark into sql-tests-refactor
Aug 12, 2015
561eacc
Merge branch 'master' of github.com:apache/spark into sql-tests-refactor
Aug 12, 2015
aaba277
Fix places where we override before / after alls
Aug 12, 2015
aac8f7f
Merge branch 'master' of github.com:apache/spark into sql-tests-refactor
Aug 12, 2015
40959bb
Fix test
Aug 12, 2015
55d6624
Merge branch 'master' of github.com:apache/spark into sql-tests-refactor
Aug 12, 2015
4ed58c8
Revert a few merge-conflict-induced unintentional changes
Aug 12, 2015
0ce5638
Minor updates
Aug 13, 2015
814df2f
Add back singletons but deprecate them
Aug 13, 2015
099c74f
Merge branch 'master' of github.com:apache/spark into sql-tests-refactor
Aug 13, 2015
451fa37
Use consistent name for added back singletons
Aug 13, 2015
9ea7f7c
Fix style
Aug 13, 2015
ddc4b05
Revert "Use consistent name for added back singletons"
Aug 13, 2015
ece3a81
Remove TestSQLContext, but keep TestHive
Aug 13, 2015
94cc3b5
Merge branch 'master' of github.com:apache/spark into sql-tests-refactor
Aug 13, 2015
8d69bf8
Fix test
Aug 13, 2015
7a0b1ef
Merge branch 'master' of github.com:apache/spark into sql-tests-refactor
Aug 13, 2015
b58ae73
Fix tests
Aug 13, 2015
828144f
Merge branch 'master' of github.com:apache/spark into sql-tests-refactor
Aug 13, 2015
c4d44c9
Merge branch 'master' of github.com:apache/spark into sql-tests-refactor
Aug 13, 2015
48af8e4
Fix another before / after alls
Aug 13, 2015
0606c82
Merge branch 'master' of github.com:apache/spark into sql-tests-refactor
Aug 13, 2015
f599bbc
Revert all Hive related changes
Aug 13, 2015
aed7fc7
Merge branch 'master' of github.com:apache/spark into sql-tests-refactor
Aug 13, 2015
d85a6d8
Add a shorthand for sqlContext.sql / ctx.sql
Aug 13, 2015
0b60325
Fix hive test compile
Aug 13, 2015
821ea67
Merge branch 'master' of github.com:apache/spark into sql-tests-refactor
Aug 13, 2015
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Fix style
  • Loading branch information
Andrew Or committed Aug 11, 2015
commit 1400770941263d586f94ef59f40199a789579b4a
Original file line number Diff line number Diff line change
Expand Up @@ -304,7 +304,8 @@ class JsonSuite extends QueryTest with SQLTestUtils with TestJsonData {

// Access elements of a BigInteger array (we use DecimalType internally).
checkAnswer(
ctx.sql("select arrayOfBigInteger[0], arrayOfBigInteger[1], arrayOfBigInteger[2] from jsonTable"),
ctx.sql(
"select arrayOfBigInteger[0], arrayOfBigInteger[1], arrayOfBigInteger[2] from jsonTable"),
Row(new java.math.BigDecimal("922337203685477580700"),
new java.math.BigDecimal("-922337203685477580800"), null)
)
Expand Down Expand Up @@ -355,7 +356,8 @@ class JsonSuite extends QueryTest with SQLTestUtils with TestJsonData {

// Access elements of an array field of a struct.
checkAnswer(
ctx.sql("select structWithArrayFields.field1[1], structWithArrayFields.field2[3] from jsonTable"),
ctx.sql(
"select structWithArrayFields.field1[1], structWithArrayFields.field2[3] from jsonTable"),
Row(5, null)
)
}
Expand Down Expand Up @@ -590,7 +592,8 @@ class JsonSuite extends QueryTest with SQLTestUtils with TestJsonData {

val schema = StructType(StructField("a", LongType, true) :: Nil)
val logicalRelation =
_sqlContext.read.schema(schema).json(path).queryExecution.analyzed.asInstanceOf[LogicalRelation]
_sqlContext.read.schema(schema).json(path)
.queryExecution.analyzed.asInstanceOf[LogicalRelation]
val relationWithSchema = logicalRelation.relation.asInstanceOf[JSONRelation]
assert(relationWithSchema.paths === Array(path))
assert(relationWithSchema.schema === schema)
Expand Down Expand Up @@ -1021,7 +1024,8 @@ class JsonSuite extends QueryTest with SQLTestUtils with TestJsonData {

// Access an array field of a struct.
checkAnswer(
ctx.sql("select structWithArrayFields.field1, structWithArrayFields.field2 from complexTable"),
ctx.sql(
"select structWithArrayFields.field1, structWithArrayFields.field2 from complexTable"),
Row(Seq(4, 5, 6), Seq("str1", "str2"))
)

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -185,19 +185,22 @@ class JDBCSuite extends SparkFunSuite with BeforeAndAfter with SQLTestUtils {
}

test("SELECT * WHERE (quoted strings)") {
assert(ctx.sql("select * from foobar").where('NAME === "joe 'foo' \"bar\"").collect().size === 1)
assert(
ctx.sql("select * from foobar").where('NAME === "joe 'foo' \"bar\"").collect().size === 1)
}

test("SELECT first field") {
val names = ctx.sql("SELECT NAME FROM foobar").collect().map(x => x.getString(0)).sortWith(_ < _)
val names =
ctx.sql("SELECT NAME FROM foobar").collect().map(x => x.getString(0)).sortWith(_ < _)
assert(names.size === 3)
assert(names(0).equals("fred"))
assert(names(1).equals("joe 'foo' \"bar\""))
assert(names(2).equals("mary"))
}

test("SELECT first field when fetchSize is two") {
val names = ctx.sql("SELECT NAME FROM fetchtwo").collect().map(x => x.getString(0)).sortWith(_ < _)
val names =
ctx.sql("SELECT NAME FROM fetchtwo").collect().map(x => x.getString(0)).sortWith(_ < _)
assert(names.size === 3)
assert(names(0).equals("fred"))
assert(names(1).equals("joe 'foo' \"bar\""))
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -219,9 +219,10 @@ class InsertSuite extends DataSourceTest with BeforeAndAfterAll {
caseInsensitiveContext.sql("SELECT a * 2 FROM jsonTable"),
(1 to 10).map(i => Row(i * 2)).toSeq)

assertCached(caseInsensitiveContext.sql("SELECT x.a, y.a FROM jsonTable x JOIN jsonTable y ON x.a = y.a + 1"), 2)
checkAnswer(
caseInsensitiveContext.sql("SELECT x.a, y.a FROM jsonTable x JOIN jsonTable y ON x.a = y.a + 1"),
assertCached(caseInsensitiveContext.sql(
"SELECT x.a, y.a FROM jsonTable x JOIN jsonTable y ON x.a = y.a + 1"), 2)
checkAnswer(caseInsensitiveContext.sql(
"SELECT x.a, y.a FROM jsonTable x JOIN jsonTable y ON x.a = y.a + 1"),
(2 to 10).map(i => Row(i, i - 1)).toSeq)

// Insert overwrite and keep the same schema.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -301,9 +301,10 @@ class TableScanSuite extends DataSourceTest {
caseInsensitiveContext.sql("SELECT i * 2 FROM oneToTen"),
(1 to 10).map(i => Row(i * 2)).toSeq)

assertCached(caseInsensitiveContext.sql("SELECT a.i, b.i FROM oneToTen a JOIN oneToTen b ON a.i = b.i + 1"), 2)
checkAnswer(
caseInsensitiveContext.sql("SELECT a.i, b.i FROM oneToTen a JOIN oneToTen b ON a.i = b.i + 1"),
assertCached(caseInsensitiveContext.sql(
"SELECT a.i, b.i FROM oneToTen a JOIN oneToTen b ON a.i = b.i + 1"), 2)
checkAnswer(caseInsensitiveContext.sql(
"SELECT a.i, b.i FROM oneToTen a JOIN oneToTen b ON a.i = b.i + 1"),
(2 to 10).map(i => Row(i, i - 1)).toSeq)

// Verify uncaching
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@

/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,9 @@ class HiveExplainSuite extends QueryTest with HiveTestUtils {
"Limit",
"src")

checkExistence(ctx.sql("explain extended create table temp__b as select * from src limit 2"), true,
checkExistence(ctx.sql(
"explain extended create table temp__b as select * from src limit 2"),
true,
"== Parsed Logical Plan ==",
"== Analyzed Logical Plan ==",
"== Optimized Logical Plan ==",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -726,7 +726,8 @@ class HiveQuerySuite extends HiveComparisonTest with BeforeAndAfter {
.size == 5)

assert(
ctx.sql("select a, b from (select stack(2, key, value, key, value) as (a, b) from src) t limit 5")
ctx.sql(
"select a, b from (select stack(2, key, value, key, value) as (a, b) from src) t limit 5")
.collect()
.size == 5)
}
Expand Down Expand Up @@ -773,7 +774,8 @@ class HiveQuerySuite extends HiveComparisonTest with BeforeAndAfter {
}

test("DESCRIBE commands") {
ctx.sql(s"CREATE TABLE test_describe_commands1 (key INT, value STRING) PARTITIONED BY (dt STRING)")
ctx.sql(
s"CREATE TABLE test_describe_commands1 (key INT, value STRING) PARTITIONED BY (dt STRING)")

ctx.sql(
"""FROM src INSERT OVERWRITE TABLE test_describe_commands1 PARTITION (dt='2008-06-08')
Expand Down Expand Up @@ -935,7 +937,8 @@ class HiveQuerySuite extends HiveComparisonTest with BeforeAndAfter {

ignore("Dynamic partition folder layout") {
ctx.sql("DROP TABLE IF EXISTS dynamic_part_table")
ctx.sql("CREATE TABLE dynamic_part_table(intcol INT) PARTITIONED BY (partcol1 INT, partcol2 INT)")
ctx.sql(
"CREATE TABLE dynamic_part_table(intcol INT) PARTITIONED BY (partcol1 INT, partcol2 INT)")
ctx.sql("SET hive.exec.dynamic.partition.mode=nonstrict")

val data = Map(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -176,7 +176,8 @@ class HiveUDFSuite extends QueryTest with HiveTestUtils {
val testData = ctx.sparkContext.parallelize(StringCaseClass("") :: Nil).toDF()
testData.registerTempTable("inputTable")

ctx.sql(s"CREATE TEMPORARY FUNCTION testUDFToListString AS '${classOf[UDFToListString].getName}'")
ctx.sql(
s"CREATE TEMPORARY FUNCTION testUDFToListString AS '${classOf[UDFToListString].getName}'")
val errMsg = intercept[AnalysisException] {
ctx.sql("SELECT testUDFToListString(s) FROM inputTable")
}
Expand Down Expand Up @@ -270,7 +271,8 @@ class HiveUDFSuite extends QueryTest with HiveTestUtils {
StringCaseClass("world") :: StringCaseClass("goodbye") :: Nil).toDF()
testData.registerTempTable("stringTable")

ctx.sql(s"CREATE TEMPORARY FUNCTION testStringStringUDF AS '${classOf[UDFStringString].getName}'")
ctx.sql(
s"CREATE TEMPORARY FUNCTION testStringStringUDF AS '${classOf[UDFStringString].getName}'")
checkAnswer(
ctx.sql("SELECT testStringStringUDF(\"hello\", s) FROM stringTable"),
Seq(Row("hello world"), Row("hello goodbye")))
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,9 +28,9 @@ class PruningSuite extends HiveComparisonTest {
protected override def beforeAll(): Unit = {
super.beforeAll()
ctx.cacheTables = false
// Column/partition pruning is not implemented for `InMemoryColumnarTableScan` yet, need to reset
// the environment to ensure all referenced tables in this suites are not cached in-memory.
// Refer to https://issues.apache.org/jira/browse/SPARK-2283 for details.
// Column/partition pruning is not implemented for `InMemoryColumnarTableScan` yet, need
// to reset the environment to ensure all referenced tables in this suites are not cached
// in-memory. Refer to https://issues.apache.org/jira/browse/SPARK-2283 for details.
ctx.reset()
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -312,15 +312,16 @@ class SQLQuerySuite extends QueryTest with HiveTestUtils {
checkRelation("ctas1", true)
ctx.sql("DROP TABLE ctas1")

ctx.sql("CREATE TABLE ctas1 stored as rcfile AS SELECT key k, value FROM src ORDER BY k, value")
ctx.sql(
"CREATE TABLE ctas1 stored as rcfile AS SELECT key k, value FROM src ORDER BY k, value")
checkRelation("ctas1", false)
ctx.sql("DROP TABLE ctas1")

ctx.sql("CREATE TABLE ctas1 stored as orc AS SELECT key k, value FROM src ORDER BY k, value")
checkRelation("ctas1", false)
ctx.sql("DROP TABLE ctas1")

ctx.sql("CREATE TABLE ctas1 stored as parquet AS SELECT key k, value FROM src ORDER BY k, value")
ctx.sql(
"CREATE TABLE ctas1 stored as parquet AS SELECT key k, value FROM src ORDER BY k, value")
checkRelation("ctas1", false)
ctx.sql("DROP TABLE ctas1")
} finally {
Expand Down Expand Up @@ -636,7 +637,8 @@ class SQLQuerySuite extends QueryTest with HiveTestUtils {
}

test("SPARK-4296 Grouping field with Hive UDF as sub expression") {
val rdd = ctx.sparkContext.makeRDD( """{"a": "str", "b":"1", "c":"1970-01-01 00:00:00"}""" :: Nil)
val rdd = ctx.sparkContext.makeRDD(
"""{"a": "str", "b":"1", "c":"1970-01-01 00:00:00"}""" :: Nil)
ctx.read.json(rdd).registerTempTable("data")
checkAnswer(
ctx.sql("SELECT concat(a, '-', b), year(c) FROM data GROUP BY concat(a, '-', b), year(c)"),
Expand Down