From 7d5cc71e4753f26fed4563a0eef27aa9de173d57 Mon Sep 17 00:00:00 2001 From: Yuming Wang Date: Fri, 30 Mar 2018 18:41:42 +0800 Subject: [PATCH 1/3] Add org.apache.derby to IsolatedClientLoader --- .../apache/spark/sql/hive/client/IsolatedClientLoader.scala | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/IsolatedClientLoader.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/IsolatedClientLoader.scala index c2690ec32b9e..c0ec7e1e8487 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/IsolatedClientLoader.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/IsolatedClientLoader.scala @@ -188,6 +188,9 @@ private[hive] class IsolatedClientLoader( (name.startsWith("com.google") && !name.startsWith("com.google.cloud")) || name.startsWith("java.lang.") || name.startsWith("java.net") || + name.startsWith("com.sun.") || + name.startsWith("sun.reflect.") || + name.startsWith("org.apache.derby.") || sharedPrefixes.exists(name.startsWith) } From 1c801f1e673b3d6f9e94eeade08d5b309a105061 Mon Sep 17 00:00:00 2001 From: Yuming Wang Date: Mon, 9 Apr 2018 14:39:35 +0800 Subject: [PATCH 2/3] Remove com.sun. and sun.reflect. --- .../apache/spark/sql/hive/client/IsolatedClientLoader.scala | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/IsolatedClientLoader.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/IsolatedClientLoader.scala index c0ec7e1e8487..4570288f8972 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/IsolatedClientLoader.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/IsolatedClientLoader.scala @@ -182,15 +182,13 @@ private[hive] class IsolatedClientLoader( name.startsWith("org.slf4j") || name.startsWith("org.apache.log4j") || // log4j1.x name.startsWith("org.apache.logging.log4j") || // log4j2 + name.startsWith("org.apache.derby.") || name.startsWith("org.apache.spark.") || (sharesHadoopClasses && isHadoopClass) || name.startsWith("scala.") || (name.startsWith("com.google") && !name.startsWith("com.google.cloud")) || name.startsWith("java.lang.") || name.startsWith("java.net") || - name.startsWith("com.sun.") || - name.startsWith("sun.reflect.") || - name.startsWith("org.apache.derby.") || sharedPrefixes.exists(name.startsWith) } From 7133d7aac3c1aaffbd8a9c4433060c4a9d488035 Mon Sep 17 00:00:00 2001 From: Yuming Wang Date: Thu, 21 Jun 2018 22:00:06 +0800 Subject: [PATCH 3/3] Add test case --- .../apache/spark/sql/hive/HiveExternalCatalogSuite.scala | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogSuite.scala index 0a522b6a11c8..1de258f06094 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogSuite.scala @@ -113,4 +113,10 @@ class HiveExternalCatalogSuite extends ExternalCatalogSuite { catalog.createDatabase(newDb("dbWithNullDesc").copy(description = null), ignoreIfExists = false) assert(catalog.getDatabase("dbWithNullDesc").description == "") } + + test("SPARK-23831: Add org.apache.derby to IsolatedClientLoader") { + val client1 = HiveUtils.newClientForMetadata(new SparkConf, new Configuration) + val client2 = HiveUtils.newClientForMetadata(new SparkConf, new Configuration) + assert(!client1.equals(client2)) + } }