change property to use to move derby.log, clean start in test, update…

… test

R/pkg/R/sparkR.R

@@ -390,8 +390,8 @@ sparkR.session <- function(
     deployMode <- sparkConfigMap[["spark.submit.deployMode"]]
   }
 
-  if (!exists("spark.r.sql.default.derby.dir", envir = sparkConfigMap)) {
-    sparkConfigMap[["spark.r.sql.default.derby.dir"]] <- tempdir()
+  if (!exists("spark.r.sql.derby.temp.dir", envir = sparkConfigMap)) {
+    sparkConfigMap[["spark.r.sql.derby.temp.dir"]] <- tempdir()
   }
 
   if (!exists(".sparkRjsc", envir = .sparkREnv)) {

R/pkg/inst/tests/testthat/test_sparkSQL.R

@@ -60,7 +60,7 @@ unsetHiveContext <- function() {
 
 # Tests for SparkSQL functions in SparkR
 
-filesBefore <- list.files(path = file.path(Sys.getenv("SPARK_HOME"), "R"), all.files = TRUE)
+filesBefore <- list.files(path = sparkRDir, all.files = TRUE)
 sparkSession <- sparkR.session()
 sc <- callJStatic("org.apache.spark.sql.api.r.SQLUtils", "getJavaSparkContext", sparkSession)
 
@@ -2923,12 +2923,15 @@ compare_list <- function(list1, list2) {
 test_that("No extra files are created in SPARK_HOME by starting session and making calls", {
   # Check that it is not creating any extra file.
   # Does not check the tempdir which would be cleaned up after.
-  filesAfter <- list.files(path = file.path(Sys.getenv("SPARK_HOME"), "R"), all.files = TRUE)
-
-  expect_true(length(sparkHomeFileBefore) > 0)
-  compare_list(sparkHomeFileBefore, filesBefore)
-
-  compare_list(filesBefore, filesAfter)
+  filesAfter <- list.files(path = sparkRDir, all.files = TRUE)
+
+  expect_true(length(sparkRFilesBefore) > 0)
+  # first, ensure derby.log is not there
+  expect_false("derby.log" %in% filesAfter)
+  # second, ensure only spark-warehouse is created when calling SparkSession, enableHiveSupport = F
+  compare_list(sparkRFilesBefore, setdiff(filesBefore, sparkRWhitelistSQLDirs[[1]]))
+  # third, ensure only spark-warehouse and metastore_db are created when enableHiveSupport = T
+  compare_list(sparkRFilesBefore, setdiff(filesAfter, sparkRWhitelistSQLDirs))
 })
 
 unlink(parquetPath)

R/pkg/tests/run-all.R

@@ -22,7 +22,12 @@ library(SparkR)
 options("warn" = 2)
 
 # Setup global test environment
-sparkHomeFileBefore <- list.files(path = file.path(Sys.getenv("SPARK_HOME"), "R"), all.files = TRUE)
+sparkRDir <- file.path(Sys.getenv("SPARK_HOME"), "R")
+sparkRFilesBefore <- list.files(path = sparkRDir, all.files = TRUE)
+sparkRWhitelistSQLDirs <- c("spark-warehouse", "metastore_db")
+invisible(lapply(sparkRWhitelistSQLDirs,
+                 function(x) { unlink(file.path(sparkRDir, x), recursive = TRUE, force = TRUE)}))
+
 install.spark()
 
 test_package("SparkR")

core/src/main/scala/org/apache/spark/api/r/RRDD.scala

@@ -18,6 +18,7 @@
 package org.apache.spark.api.r
 
 import java.util.{Map => JMap}
+import java.io.File
 
 import scala.collection.JavaConverters._
 import scala.reflect.ClassTag
@@ -127,11 +128,12 @@ private[r] object RRDD {
       sparkConf.setExecutorEnv(name.toString, value.toString)
     }
 
-    if (sparkEnvirMap.containsKey("spark.r.sql.default.derby.dir") &&
-        System.getProperty("derby.system.home") == null) {
+    if (sparkEnvirMap.containsKey("spark.r.sql.derby.temp.dir") &&
+        System.getProperty("derby.stream.error.file") == null) {
       // This must be set before SparkContext is instantiated.
-      System.setProperty("derby.system.home",
-                         sparkEnvirMap.get("spark.r.sql.default.derby.dir").toString)
+      System.setProperty("derby.stream.error.file",
+                         Seq(sparkEnvirMap.get("spark.r.sql.derby.temp.dir").toString, "derby.log")
+                         .mkString(File.separator))
     }
 
     val jsc = new JavaSparkContext(sparkConf)