From 3d19ad9ad5885fa5cf7da0c733dca9152fb15655 Mon Sep 17 00:00:00 2001 From: baishuo Date: Sun, 7 Jun 2015 22:19:38 -0700 Subject: [PATCH 1/6] create table to specific database --- .../org/apache/spark/sql/hive/HiveContext.scala | 4 ++-- .../spark/sql/hive/HiveMetastoreCatalog.scala | 2 +- .../sql/hive/MetastoreDataSourcesSuite.scala | 17 +++++++++++++++++ 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveContext.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveContext.scala index b8f294c262af..f7800857fa7c 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveContext.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveContext.scala @@ -273,12 +273,12 @@ class HiveContext(sc: SparkContext) extends SQLContext(sc) { */ def refreshTable(tableName: String): Unit = { // TODO: Database support... - catalog.refreshTable("default", tableName) + catalog.refreshTable(catalog.client.currentDatabase, tableName) } protected[hive] def invalidateTable(tableName: String): Unit = { // TODO: Database support... - catalog.invalidateTable("default", tableName) + catalog.invalidateTable(catalog.client.currentDatabase, tableName) } /** diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveMetastoreCatalog.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveMetastoreCatalog.scala index 5a4651a887b7..62d9da38daf7 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveMetastoreCatalog.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveMetastoreCatalog.scala @@ -143,7 +143,7 @@ private[hive] class HiveMetastoreCatalog(val client: ClientInterface, hive: Hive provider: String, options: Map[String, String], isExternal: Boolean): Unit = { - val (dbName, tblName) = processDatabaseAndTableName("default", tableName) + val (dbName, tblName) = processDatabaseAndTableName(client.currentDatabase, tableName) val tableProperties = new scala.collection.mutable.HashMap[String, String] tableProperties.put("spark.sql.sources.provider", provider) diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala index af586712e323..c540f57ee97a 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala @@ -833,4 +833,21 @@ class MetastoreDataSourcesSuite extends QueryTest with SQLTestUtils with BeforeA (70 to 79).map(i => Row(i, s"str$i"))) } } + + test("SPARK-8516:create table to specific database by 'use dbname' ") { + + val df = (1 to 3).map(i => (i, s"val_$i", i * 2)).toDF("a", "b", "c") + sqlContext.sql("""create database if not exists testdb""") + sqlContext.sql("""use testdb""") + df.write + .format("parquet") + .mode(SaveMode.Overwrite) + .saveAsTable("ttt3") + + checkAnswer( + sqlContext.sql("show TABLES in testdb"), + Row("ttt3", false)) + + sqlContext.sql("""drop database if exists testdb CASCADE""") + } } From b742e69ad2df6f8c85afbcf353e4cee0a92242d1 Mon Sep 17 00:00:00 2001 From: baishuo Date: Mon, 8 Jun 2015 01:15:22 -0700 Subject: [PATCH 2/6] modify testcase --- .../org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala index c540f57ee97a..ef909f1f9b27 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala @@ -834,7 +834,7 @@ class MetastoreDataSourcesSuite extends QueryTest with SQLTestUtils with BeforeA } } - test("SPARK-8516:create table to specific database by 'use dbname' ") { + test("SPARK-8156:create table to specific database by 'use dbname' ") { val df = (1 to 3).map(i => (i, s"val_$i", i * 2)).toDF("a", "b", "c") sqlContext.sql("""create database if not exists testdb""") @@ -847,7 +847,7 @@ class MetastoreDataSourcesSuite extends QueryTest with SQLTestUtils with BeforeA checkAnswer( sqlContext.sql("show TABLES in testdb"), Row("ttt3", false)) - + sqlContext.sql("""use default""") sqlContext.sql("""drop database if exists testdb CASCADE""") } } From 4df48c7e4e1b5793d8c00c15b25ec9265cb8766d Mon Sep 17 00:00:00 2001 From: baishuo Date: Mon, 8 Jun 2015 07:44:04 -0700 Subject: [PATCH 3/6] modify testcase --- .../org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala index ef909f1f9b27..2f7d5ef5e542 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala @@ -834,7 +834,7 @@ class MetastoreDataSourcesSuite extends QueryTest with SQLTestUtils with BeforeA } } - test("SPARK-8156:create table to specific database by 'use dbname' ") { + test("SPARK-8157:create table to specific database by 'use dbname' ") { val df = (1 to 3).map(i => (i, s"val_$i", i * 2)).toDF("a", "b", "c") sqlContext.sql("""create database if not exists testdb""") @@ -845,7 +845,7 @@ class MetastoreDataSourcesSuite extends QueryTest with SQLTestUtils with BeforeA .saveAsTable("ttt3") checkAnswer( - sqlContext.sql("show TABLES in testdb"), + sqlContext.sql("show TABLES in testdb").filter("tableName = 'ttt3'"), Row("ttt3", false)) sqlContext.sql("""use default""") sqlContext.sql("""drop database if exists testdb CASCADE""") From 00a7a2d7a270b368f4d3b4254e7ed1143f00f0fa Mon Sep 17 00:00:00 2001 From: baishuo Date: Mon, 8 Jun 2015 07:50:50 -0700 Subject: [PATCH 4/6] modify testcase --- .../apache/spark/sql/hive/MetastoreDataSourcesSuite.scala | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala index 2f7d5ef5e542..d9d0a26df7a6 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala @@ -834,20 +834,20 @@ class MetastoreDataSourcesSuite extends QueryTest with SQLTestUtils with BeforeA } } - test("SPARK-8157:create table to specific database by 'use dbname' ") { + test("SPARK-8156:create table to specific database by 'use dbname' ") { val df = (1 to 3).map(i => (i, s"val_$i", i * 2)).toDF("a", "b", "c") sqlContext.sql("""create database if not exists testdb""") - sqlContext.sql("""use testdb""") + sqlContext.sql("""use testdb8516""") df.write .format("parquet") .mode(SaveMode.Overwrite) .saveAsTable("ttt3") checkAnswer( - sqlContext.sql("show TABLES in testdb").filter("tableName = 'ttt3'"), + sqlContext.sql("show TABLES in testdb8516").filter("tableName = 'ttt3'"), Row("ttt3", false)) sqlContext.sql("""use default""") - sqlContext.sql("""drop database if exists testdb CASCADE""") + sqlContext.sql("""drop database if exists testdb8516 CASCADE""") } } From cb9f0276129d275d6bf5bb8e7414a1151488d488 Mon Sep 17 00:00:00 2001 From: baishuo Date: Mon, 8 Jun 2015 07:57:17 -0700 Subject: [PATCH 5/6] modify testcase --- .../org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala index d9d0a26df7a6..49e96c851b9e 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala @@ -837,7 +837,7 @@ class MetastoreDataSourcesSuite extends QueryTest with SQLTestUtils with BeforeA test("SPARK-8156:create table to specific database by 'use dbname' ") { val df = (1 to 3).map(i => (i, s"val_$i", i * 2)).toDF("a", "b", "c") - sqlContext.sql("""create database if not exists testdb""") + sqlContext.sql("""create database if not exists testdb8516""") sqlContext.sql("""use testdb8516""") df.write .format("parquet") From 9e155f9b2ca51a13a3ac8e35498d2144c729f42d Mon Sep 17 00:00:00 2001 From: baishuo Date: Sat, 13 Jun 2015 00:57:52 -0700 Subject: [PATCH 6/6] remove no use comment --- .../scala/org/apache/spark/sql/hive/HiveContext.scala | 2 -- .../apache/spark/sql/hive/MetastoreDataSourcesSuite.scala | 8 ++++---- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveContext.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveContext.scala index f7800857fa7c..53ce50c5171c 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveContext.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveContext.scala @@ -272,12 +272,10 @@ class HiveContext(sc: SparkContext) extends SQLContext(sc) { * @since 1.3.0 */ def refreshTable(tableName: String): Unit = { - // TODO: Database support... catalog.refreshTable(catalog.client.currentDatabase, tableName) } protected[hive] def invalidateTable(tableName: String): Unit = { - // TODO: Database support... catalog.invalidateTable(catalog.client.currentDatabase, tableName) } diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala index 49e96c851b9e..79a85b24d2f6 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala @@ -837,17 +837,17 @@ class MetastoreDataSourcesSuite extends QueryTest with SQLTestUtils with BeforeA test("SPARK-8156:create table to specific database by 'use dbname' ") { val df = (1 to 3).map(i => (i, s"val_$i", i * 2)).toDF("a", "b", "c") - sqlContext.sql("""create database if not exists testdb8516""") - sqlContext.sql("""use testdb8516""") + sqlContext.sql("""create database if not exists testdb8156""") + sqlContext.sql("""use testdb8156""") df.write .format("parquet") .mode(SaveMode.Overwrite) .saveAsTable("ttt3") checkAnswer( - sqlContext.sql("show TABLES in testdb8516").filter("tableName = 'ttt3'"), + sqlContext.sql("show TABLES in testdb8156").filter("tableName = 'ttt3'"), Row("ttt3", false)) sqlContext.sql("""use default""") - sqlContext.sql("""drop database if exists testdb8516 CASCADE""") + sqlContext.sql("""drop database if exists testdb8156 CASCADE""") } }