Skip to content

[SPARK-19331][SQL][TESTS] Improve the test coverage of SQLViewSuite #16674

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
jiangxb1987 wants to merge 7 commits into from
Closed

[SPARK-19331][SQL][TESTS] Improve the test coverage of SQLViewSuite #16674

Changes from 1 commit
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
adde65f
update SQLViewSuite
jiangxb1987 Jan 22, 2017
2e201c2
move SQLViewSuite to sql/core.
jiangxb1987 Feb 8, 2017
664b53a
move some test cases.
jiangxb1987 Feb 10, 2017
e8fc8c7
code refactor.
jiangxb1987 Feb 11, 2017
2f00404
merge the beforeAll() and afterAll() logic.
jiangxb1987 Feb 12, 2017
4872918
place super.afterAll() before stop SparkContext in SharedSQLContext.a…
jiangxb1987 Feb 14, 2017
aa0cf0a
code refactor.
jiangxb1987 Feb 15, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Next Next commit
update SQLViewSuite
  • Loading branch information
@jiangxb1987
jiangxb1987 committed Feb 8, 2017
commit adde65f533b5fc62b95aa5c90d0cf9e06bbf11d6
260 changes: 117 additions & 143 deletions sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLViewSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -183,17 +183,26 @@ class SQLViewSuite extends QueryTest with SQLTestUtils with TestHiveSingleton {
}

test("error handling: fail if the view sql itself is invalid") {
// A database that does not exist
assertInvalidReference("CREATE OR REPLACE VIEW myabcdview AS SELECT * FROM db_not_exist234.jt")

// A table that does not exist
intercept[AnalysisException] {
sql("CREATE OR REPLACE VIEW myabcdview AS SELECT * FROM table_not_exist1345").collect()
}
assertInvalidReference("CREATE OR REPLACE VIEW myabcdview AS SELECT * FROM table_not_exist345")

// A column that does not exist
intercept[AnalysisException] {
sql("CREATE OR REPLACE VIEW myabcdview AS SELECT random1234 FROM jt").collect()
}
}

private def assertInvalidReference(query: String): Unit = {
val e = intercept[AnalysisException] {
sql(query)
}.getMessage
assert(e.contains("Table or view not found"))
}


test("error handling: fail if the temp view name contains the database prefix") {
// Fully qualified table name like "database.table" is not allowed for temporary view
val e = intercept[AnalysisException] {
Expand All @@ -210,10 +219,13 @@ class SQLViewSuite extends QueryTest with SQLTestUtils with TestHiveSingleton {
}

test("error handling: fail if the temp view sql itself is invalid") {
// A table that does not exist for temporary view
intercept[AnalysisException] {
sql("CREATE OR REPLACE TEMPORARY VIEW myabcdview AS SELECT * FROM table_not_exist1345")
}
// A database that does not exist
assertInvalidReference(
"CREATE OR REPLACE TEMPORARY VIEW myabcdview AS SELECT * FROM db_not_exist234.jt")

// A table that does not exist
assertInvalidReference(
"CREATE OR REPLACE TEMPORARY VIEW myabcdview AS SELECT * FROM table_not_exist1345")

// A column that does not exist, for temporary view
intercept[AnalysisException] {
Expand All @@ -233,6 +245,42 @@ class SQLViewSuite extends QueryTest with SQLTestUtils with TestHiveSingleton {
}
}

test("correctly parse a nested view") {
withTempDatabase { db =>
withView("view1", "view2", s"$db.view3") {
sql("CREATE VIEW view1(x, y) AS SELECT * FROM jt")

// Create a nested view in the same database.
sql("CREATE VIEW view2(id, id1) AS SELECT * FROM view1")
checkAnswer(sql("SELECT * FROM view2 ORDER BY id"), (1 to 9).map(i => Row(i, i)))

// Create a nested view in a different database.
activateDatabase(db) {
sql(s"CREATE VIEW $db.view3(id, id1) AS SELECT * FROM default.view1")
checkAnswer(sql("SELECT * FROM view3 ORDER BY id"), (1 to 9).map(i => Row(i, i)))
}
}
}
}

test("correctly parse a view with custom column names") {
withTable("testTable") {
spark.range(1, 10).selectExpr("id", "id + 1 id1").write.saveAsTable("testTable")
withView("testView1", "testView2") {
// Correctly create a view with custom column names
sql("CREATE VIEW testView1(x, y) AS SELECT * FROM testTable")
checkAnswer(sql("SELECT * FROM testView1 ORDER BY x"), (1 to 9).map(i => Row(i, i + 1)))

// Throw an AnalysisException if the number of columns don't match up.
val e = intercept[AnalysisException] {
sql("CREATE VIEW testView2(x, y, z) AS SELECT * FROM testTable")
}.getMessage
assert(e.contains("The number of columns produced by the SELECT clause (num: `2`) does " +
"not match the number of column names specified by CREATE VIEW (num: `3`)."))
}
}
}

test("correctly parse CREATE TEMPORARY VIEW statement") {
withView("testView") {
sql(
Expand Down Expand Up @@ -334,6 +382,25 @@ class SQLViewSuite extends QueryTest with SQLTestUtils with TestHiveSingleton {
}
}

test("correctly handle ALTER VIEW on a referenced view") {
withTable("jt2") {
val df = (1 until 10).map(i => (i, i + 1)).toDF("i", "j")
df.write.format("json").saveAsTable("jt2")

withView("view1", "view2") {
sql("CREATE VIEW view1(x, y) AS SELECT * FROM jt")

// Create a nested view.
sql("CREATE VIEW view2(id, id1) AS SELECT * FROM view1")
checkAnswer(sql("SELECT * FROM view2 ORDER BY id"), (1 to 9).map(i => Row(i, i)))

// Alter the referenced view.
sql("ALTER VIEW view1 AS SELECT i AS x, j AS y FROM jt2")
checkAnswer(sql("SELECT * FROM view2 ORDER BY id"), (1 to 9).map(i => Row(i, i + 1)))
}
}
}

test("should not allow ALTER VIEW AS when the view does not exist") {
assertNoSuchTable("ALTER VIEW testView AS SELECT 1, 2")
assertNoSuchTable("ALTER VIEW default.testView AS SELECT 1, 2")
Expand Down Expand Up @@ -457,14 +524,14 @@ class SQLViewSuite extends QueryTest with SQLTestUtils with TestHiveSingleton {
withTable("jt1", "jt2") {
spark.range(1, 10).toDF("id1").write.format("json").saveAsTable("jt1")
spark.range(1, 10).toDF("id2").write.format("json").saveAsTable("jt2")
sql("CREATE VIEW testView AS SELECT * FROM jt1 JOIN jt2 ON id1 == id2")
checkAnswer(sql("SELECT * FROM testView ORDER BY id1"), (1 to 9).map(i => Row(i, i)))

val df = (1 until 10).map(i => i -> i).toDF("id1", "newCol")
df.write.format("json").mode(SaveMode.Overwrite).saveAsTable("jt1")
checkAnswer(sql("SELECT * FROM testView ORDER BY id1"), (1 to 9).map(i => Row(i, i)))
withView("testView") {
sql("CREATE VIEW testView AS SELECT * FROM jt1 JOIN jt2 ON id1 == id2")
checkAnswer(sql("SELECT * FROM testView ORDER BY id1"), (1 to 9).map(i => Row(i, i)))

sql("DROP VIEW testView")
val df = (1 until 10).map(i => i -> i).toDF("id1", "newCol")
df.write.format("json").mode(SaveMode.Overwrite).saveAsTable("jt1")
checkAnswer(sql("SELECT * FROM testView ORDER BY id1"), (1 to 9).map(i => Row(i, i)))
}
}
}

Expand Down Expand Up @@ -548,122 +615,31 @@ class SQLViewSuite extends QueryTest with SQLTestUtils with TestHiveSingleton {
}
}

test("correctly resolve a nested view") {
withTempDatabase { db =>
withView(s"$db.view1", s"$db.view2") {
val view1 = CatalogTable(
identifier = TableIdentifier("view1", Some(db)),
tableType = CatalogTableType.VIEW,
storage = CatalogStorageFormat.empty,
schema = new StructType().add("x", "long").add("y", "long"),
viewText = Some("SELECT * FROM jt"),
properties = Map(CatalogTable.VIEW_DEFAULT_DATABASE -> "default",
CatalogTable.VIEW_QUERY_OUTPUT_NUM_COLUMNS -> "2",
s"${CatalogTable.VIEW_QUERY_OUTPUT_COLUMN_NAME_PREFIX}0" -> "id",
s"${CatalogTable.VIEW_QUERY_OUTPUT_COLUMN_NAME_PREFIX}1" -> "id1"))
val view2 = CatalogTable(
identifier = TableIdentifier("view2", Some(db)),
tableType = CatalogTableType.VIEW,
storage = CatalogStorageFormat.empty,
schema = new StructType().add("id", "long").add("id1", "long"),
viewText = Some("SELECT * FROM view1"),
properties = Map(CatalogTable.VIEW_DEFAULT_DATABASE -> db,
CatalogTable.VIEW_QUERY_OUTPUT_NUM_COLUMNS -> "2",
s"${CatalogTable.VIEW_QUERY_OUTPUT_COLUMN_NAME_PREFIX}0" -> "x",
s"${CatalogTable.VIEW_QUERY_OUTPUT_COLUMN_NAME_PREFIX}1" -> "y"))
activateDatabase(db) {
hiveContext.sessionState.catalog.createTable(view1, ignoreIfExists = false)
hiveContext.sessionState.catalog.createTable(view2, ignoreIfExists = false)
checkAnswer(sql("SELECT * FROM view2 ORDER BY id"), (1 to 9).map(i => Row(i, i)))
}
}
}
}

test("correctly resolve a view with CTE") {
withView("cte_view") {
val cte_view = CatalogTable(
identifier = TableIdentifier("cte_view"),
tableType = CatalogTableType.VIEW,
storage = CatalogStorageFormat.empty,
schema = new StructType().add("n", "int"),
viewText = Some("WITH w AS (SELECT 1 AS n) SELECT n FROM w"),
properties = Map(CatalogTable.VIEW_DEFAULT_DATABASE -> "default",
CatalogTable.VIEW_QUERY_OUTPUT_NUM_COLUMNS -> "1",
s"${CatalogTable.VIEW_QUERY_OUTPUT_COLUMN_NAME_PREFIX}0" -> "n"))
hiveContext.sessionState.catalog.createTable(cte_view, ignoreIfExists = false)
checkAnswer(sql("SELECT * FROM cte_view"), Row(1))
}
}

test("correctly resolve a view in a self join") {
withView("join_view") {
val join_view = CatalogTable(
identifier = TableIdentifier("join_view"),
tableType = CatalogTableType.VIEW,
storage = CatalogStorageFormat.empty,
schema = new StructType().add("id", "long").add("id1", "long"),
viewText = Some("SELECT * FROM jt"),
properties = Map(CatalogTable.VIEW_DEFAULT_DATABASE -> "default",
CatalogTable.VIEW_QUERY_OUTPUT_NUM_COLUMNS -> "2",
s"${CatalogTable.VIEW_QUERY_OUTPUT_COLUMN_NAME_PREFIX}0" -> "id",
s"${CatalogTable.VIEW_QUERY_OUTPUT_COLUMN_NAME_PREFIX}1" -> "id1"))
hiveContext.sessionState.catalog.createTable(join_view, ignoreIfExists = false)
checkAnswer(
sql("SELECT * FROM join_view t1 JOIN join_view t2 ON t1.id = t2.id ORDER BY t1.id"),
(1 to 9).map(i => Row(i, i, i, i)))
}
}

private def assertInvalidReference(query: String): Unit = {
val e = intercept[AnalysisException] {
sql(query)
}.getMessage
assert(e.contains("Table or view not found"))
}

test("error handling: fail if the referenced table or view is invalid") {
withView("view1", "view2", "view3") {
// Fail if the referenced table is defined in a invalid database.
val view1 = CatalogTable(
identifier = TableIdentifier("view1"),
tableType = CatalogTableType.VIEW,
storage = CatalogStorageFormat.empty,
schema = new StructType().add("id", "long").add("id1", "long"),
viewText = Some("SELECT * FROM invalid_db.jt"),
properties = Map(CatalogTable.VIEW_DEFAULT_DATABASE -> "default",
CatalogTable.VIEW_QUERY_OUTPUT_NUM_COLUMNS -> "2",
s"${CatalogTable.VIEW_QUERY_OUTPUT_COLUMN_NAME_PREFIX}0" -> "id",
s"${CatalogTable.VIEW_QUERY_OUTPUT_COLUMN_NAME_PREFIX}1" -> "id1"))
hiveContext.sessionState.catalog.createTable(view1, ignoreIfExists = false)
withTempDatabase { db =>
withTable(s"$db.table1") {
activateDatabase(db) {
sql("CREATE TABLE table1(a int, b string)")
sql("CREATE VIEW default.view1 AS SELECT * FROM table1")
}
}
}
assertInvalidReference("SELECT * FROM view1")

// Fail if the referenced table is invalid.
val view2 = CatalogTable(
identifier = TableIdentifier("view2"),
tableType = CatalogTableType.VIEW,
storage = CatalogStorageFormat.empty,
schema = new StructType().add("id", "long").add("id1", "long"),
viewText = Some("SELECT * FROM invalid_table"),
properties = Map(CatalogTable.VIEW_DEFAULT_DATABASE -> "default",
CatalogTable.VIEW_QUERY_OUTPUT_NUM_COLUMNS -> "2",
s"${CatalogTable.VIEW_QUERY_OUTPUT_COLUMN_NAME_PREFIX}0" -> "id",
s"${CatalogTable.VIEW_QUERY_OUTPUT_COLUMN_NAME_PREFIX}1" -> "id1"))
hiveContext.sessionState.catalog.createTable(view2, ignoreIfExists = false)
withTable("table2") {
sql("CREATE TABLE table2(a int, b string)")
sql("CREATE VIEW view2 AS SELECT * FROM table2")
}
assertInvalidReference("SELECT * FROM view2")

// Fail if the referenced view is invalid.
val view3 = CatalogTable(
identifier = TableIdentifier("view3"),
tableType = CatalogTableType.VIEW,
storage = CatalogStorageFormat.empty,
schema = new StructType().add("id", "long").add("id1", "long"),
viewText = Some("SELECT * FROM view2"),
properties = Map(CatalogTable.VIEW_DEFAULT_DATABASE -> "default",
CatalogTable.VIEW_QUERY_OUTPUT_NUM_COLUMNS -> "2",
s"${CatalogTable.VIEW_QUERY_OUTPUT_COLUMN_NAME_PREFIX}0" -> "id",
s"${CatalogTable.VIEW_QUERY_OUTPUT_COLUMN_NAME_PREFIX}1" -> "id1"))
hiveContext.sessionState.catalog.createTable(view3, ignoreIfExists = false)
withView("testView") {
sql("CREATE VIEW testView AS SELECT * FROM jt")
sql("CREATE VIEW view3 AS SELECT * FROM testView")
}
assertInvalidReference("SELECT * FROM view3")
}
}
Expand Down Expand Up @@ -694,21 +670,20 @@ class SQLViewSuite extends QueryTest with SQLTestUtils with TestHiveSingleton {
}
}

test("correctly resolve a view in a self join") {
withView("testView") {
sql("CREATE VIEW testView AS SELECT * FROM jt")
checkAnswer(
sql("SELECT * FROM testView t1 JOIN testView t2 ON t1.id = t2.id ORDER BY t1.id"),
(1 to 9).map(i => Row(i, i, i, i)))
}
}

test("resolve a view with custom column names") {
withTable("testTable") {
spark.range(1, 10).selectExpr("id", "id + 1 id1").write.saveAsTable("testTable")
withView("testView") {
val testView = CatalogTable(
identifier = TableIdentifier("testView"),
tableType = CatalogTableType.VIEW,
storage = CatalogStorageFormat.empty,
schema = new StructType().add("x", "long").add("y", "long"),
viewText = Some("SELECT * FROM testTable"),
properties = Map(CatalogTable.VIEW_DEFAULT_DATABASE -> "default",
CatalogTable.VIEW_QUERY_OUTPUT_NUM_COLUMNS -> "2",
s"${CatalogTable.VIEW_QUERY_OUTPUT_COLUMN_NAME_PREFIX}0" -> "id",
s"${CatalogTable.VIEW_QUERY_OUTPUT_COLUMN_NAME_PREFIX}1" -> "id1"))
hiveContext.sessionState.catalog.createTable(testView, ignoreIfExists = false)
sql("CREATE VIEW testView(x, y) AS SELECT * FROM testTable")

// Correctly resolve a view with custom column names.
checkAnswer(sql("SELECT * FROM testView ORDER BY x"), (1 to 9).map(i => Row(i, i + 1)))
Expand All @@ -730,17 +705,7 @@ class SQLViewSuite extends QueryTest with SQLTestUtils with TestHiveSingleton {
withTable("testTable") {
spark.range(1, 10).selectExpr("id", "id + 1 id1").write.saveAsTable("testTable")
withView("testView") {
val testView = CatalogTable(
identifier = TableIdentifier("testView"),
tableType = CatalogTableType.VIEW,
storage = CatalogStorageFormat.empty,
schema = new StructType().add("id", "long").add("id1", "long"),
viewText = Some("SELECT * FROM testTable"),
properties = Map(CatalogTable.VIEW_DEFAULT_DATABASE -> "default",
CatalogTable.VIEW_QUERY_OUTPUT_NUM_COLUMNS -> "2",
s"${CatalogTable.VIEW_QUERY_OUTPUT_COLUMN_NAME_PREFIX}0" -> "id",
s"${CatalogTable.VIEW_QUERY_OUTPUT_COLUMN_NAME_PREFIX}1" -> "id1"))
hiveContext.sessionState.catalog.createTable(testView, ignoreIfExists = false)
sql("CREATE VIEW testView AS SELECT * FROM testTable")
Copy link
Member

@gatorsmile gatorsmile Jan 22, 2017

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What is the reason?

Copy link
Member

@gatorsmile gatorsmile Jan 22, 2017

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

How about creating a new test case? First, create a view using a regular SQL statement, and then fetch the metadata and verify the fetched metadata is still matching what we expect?


// Allow casting from IntegerType to LongType
val df = (1 until 10).map(i => (i, i + 1)).toDF("id", "id1")
Expand All @@ -759,4 +724,13 @@ class SQLViewSuite extends QueryTest with SQLTestUtils with TestHiveSingleton {
}
}
}

// TODO: Check for cyclic view references on ALTER VIEW.
ignore("correctly handle a cyclic view reference") {
withView("view1", "view2") {
sql("CREATE VIEW view1 AS SELECT * FROM jt")
sql("CREATE VIEW view2 AS SELECT * FROM view1")
intercept[AnalysisException](sql("ALTER VIEW view1 AS SELECT * FROM view2"))
}
}
}