From c7819e60ff674e929d47b4c7247dea0f85349ac5 Mon Sep 17 00:00:00 2001 From: Maxim Gekk Date: Sun, 8 Jul 2018 20:59:18 +0200 Subject: [PATCH 1/5] [SQL] Is a config modifiable at runtime In the PR, I propose to extend `RuntimeConfig` by new method `isModifiable()` which returns `true` if a config parameter can be modified at runtime (in notebooks). For static SQL and core parameters, the method returns `false`. --- python/pyspark/sql/conf.py | 6 ++++++ .../scala/org/apache/spark/sql/internal/SQLConf.scala | 4 ++++ .../scala/org/apache/spark/sql/RuntimeConfig.scala | 7 +++++++ .../org/apache/spark/sql/RuntimeConfigSuite.scala | 11 +++++++++++ 4 files changed, 28 insertions(+) diff --git a/python/pyspark/sql/conf.py b/python/pyspark/sql/conf.py index db49040e17b6..c4780e038d14 100644 --- a/python/pyspark/sql/conf.py +++ b/python/pyspark/sql/conf.py @@ -63,6 +63,12 @@ def _checkType(self, obj, identifier): raise TypeError("expected %s '%s' to be a string (was '%s')" % (identifier, obj, type(obj).__name__)) + @ignore_unicode_prefix + @since(2.4) + def isModifiable(self, key): + """Is the configuration property modifiable or not.""" + return self._jconf.isModifiable(key) + def _test(): import os diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala index 50965c1abc68..722e1943b344 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala @@ -1892,4 +1892,8 @@ class SQLConf extends Serializable with Logging { } cloned } + + def isModifiable(key: String): Boolean = { + sqlConfEntries.containsKey(key) && !staticConfKeys.contains(key) + } } diff --git a/sql/core/src/main/scala/org/apache/spark/sql/RuntimeConfig.scala b/sql/core/src/main/scala/org/apache/spark/sql/RuntimeConfig.scala index b352e332bc7e..3f1dd31efe2a 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/RuntimeConfig.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/RuntimeConfig.scala @@ -132,6 +132,13 @@ class RuntimeConfig private[sql](sqlConf: SQLConf = new SQLConf) { sqlConf.unsetConf(key) } + /** + * Can the configuration property be modified at runtime. + * + * @since 2.4.0 + */ + def isModifiable(key: String): Boolean = sqlConf.isModifiable(key) + /** * Returns whether a particular key is set. */ diff --git a/sql/core/src/test/scala/org/apache/spark/sql/RuntimeConfigSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/RuntimeConfigSuite.scala index cfe2e9f2dbc4..1d12d557b012 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/RuntimeConfigSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/RuntimeConfigSuite.scala @@ -54,4 +54,15 @@ class RuntimeConfigSuite extends SparkFunSuite { conf.get("k1") } } + + test("is a config parameter modifiable") { + val conf = newConf() + + // SQL configs + assert(!conf.isModifiable("spark.sql.sources.schemaStringLengthThreshold")) + assert(conf.isModifiable("spark.sql.streaming.checkpointLocation")) + // Core configs + assert(!conf.isModifiable("spark.task.cpus")) + assert(!conf.isModifiable("spark.executor.cores")) + } } From b90dbf834f7d8a7dc14fcee87968997f80fd52ac Mon Sep 17 00:00:00 2001 From: Maxim Gekk Date: Sun, 8 Jul 2018 21:32:08 +0200 Subject: [PATCH 2/5] Adding ticket number to test's title --- .../test/scala/org/apache/spark/sql/RuntimeConfigSuite.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/RuntimeConfigSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/RuntimeConfigSuite.scala index 1d12d557b012..7dccd402206f 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/RuntimeConfigSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/RuntimeConfigSuite.scala @@ -55,7 +55,7 @@ class RuntimeConfigSuite extends SparkFunSuite { } } - test("is a config parameter modifiable") { + test("SPARK-24761: is a config parameter modifiable") { val conf = newConf() // SQL configs From 465a5dd85a643e1cbc2b87f9d896e6053a51073d Mon Sep 17 00:00:00 2001 From: Maxim Gekk Date: Sun, 8 Jul 2018 22:35:40 +0200 Subject: [PATCH 3/5] Addressing Sim's review comment --- python/pyspark/sql/conf.py | 4 +++- .../src/main/scala/org/apache/spark/sql/RuntimeConfig.scala | 3 ++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/python/pyspark/sql/conf.py b/python/pyspark/sql/conf.py index c4780e038d14..f80bf598c221 100644 --- a/python/pyspark/sql/conf.py +++ b/python/pyspark/sql/conf.py @@ -66,7 +66,9 @@ def _checkType(self, obj, identifier): @ignore_unicode_prefix @since(2.4) def isModifiable(self, key): - """Is the configuration property modifiable or not.""" + """Indicates whether the configuration property with the given key + is modifiable in the current session. + """ return self._jconf.isModifiable(key) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/RuntimeConfig.scala b/sql/core/src/main/scala/org/apache/spark/sql/RuntimeConfig.scala index 3f1dd31efe2a..137ac1ff8c89 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/RuntimeConfig.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/RuntimeConfig.scala @@ -133,7 +133,8 @@ class RuntimeConfig private[sql](sqlConf: SQLConf = new SQLConf) { } /** - * Can the configuration property be modified at runtime. + * Indicates whether the configuration property with the given key + * is modifiable in the current session. * * @since 2.4.0 */ From 183c743782dd04a1f4ee2c889ea95f1b6fcf855f Mon Sep 17 00:00:00 2001 From: Maxim Gekk Date: Wed, 11 Jul 2018 21:42:44 +0200 Subject: [PATCH 4/5] Improving the comment for new method --- .../src/main/scala/org/apache/spark/sql/RuntimeConfig.scala | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/RuntimeConfig.scala b/sql/core/src/main/scala/org/apache/spark/sql/RuntimeConfig.scala index 137ac1ff8c89..3c39579149ff 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/RuntimeConfig.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/RuntimeConfig.scala @@ -136,6 +136,9 @@ class RuntimeConfig private[sql](sqlConf: SQLConf = new SQLConf) { * Indicates whether the configuration property with the given key * is modifiable in the current session. * + * @return `true` if the configuration property is modifiable. For static SQL, Spark Core, + * invalid (not existing) and other non-modifiable configuration properties, + * the returned value is `false`. * @since 2.4.0 */ def isModifiable(key: String): Boolean = sqlConf.isModifiable(key) From d0d4620aa56f4d97a7899102dfeea601c2667e17 Mon Sep 17 00:00:00 2001 From: Maxim Gekk Date: Wed, 11 Jul 2018 21:43:01 +0200 Subject: [PATCH 5/5] Added tests for invalid params --- .../test/scala/org/apache/spark/sql/RuntimeConfigSuite.scala | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/RuntimeConfigSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/RuntimeConfigSuite.scala index 7dccd402206f..cdcea09ad975 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/RuntimeConfigSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/RuntimeConfigSuite.scala @@ -64,5 +64,8 @@ class RuntimeConfigSuite extends SparkFunSuite { // Core configs assert(!conf.isModifiable("spark.task.cpus")) assert(!conf.isModifiable("spark.executor.cores")) + // Invalid config parameters + assert(!conf.isModifiable("")) + assert(!conf.isModifiable("invalid config parameter")) } }