From b142a119c12a3bb909a1d0547997e450f80d48b2 Mon Sep 17 00:00:00 2001 From: HyukjinKwon Date: Mon, 27 Jul 2020 11:26:59 +0900 Subject: [PATCH 1/4] [SPARK-32452][R][SQL] Bump up the minimum Arrow version as 1.0.0 in SparkR --- R/pkg/DESCRIPTION | 2 +- docs/sparkr.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/R/pkg/DESCRIPTION b/R/pkg/DESCRIPTION index c5c08bd7a063..2047f0d75ca1 100644 --- a/R/pkg/DESCRIPTION +++ b/R/pkg/DESCRIPTION @@ -23,7 +23,7 @@ Suggests: testthat, e1071, survival, - arrow (>= 0.15.1) + arrow (>= 1.0.0) Collate: 'schema.R' 'generics.R' diff --git a/docs/sparkr.md b/docs/sparkr.md index d86fa86c8985..fa1bb1b85181 100644 --- a/docs/sparkr.md +++ b/docs/sparkr.md @@ -674,7 +674,7 @@ Rscript -e 'install.packages("arrow", repos="https://cloud.r-project.org/")' Please refer [the official documentation of Apache Arrow](https://arrow.apache.org/docs/r/) for more detials. Note that you must ensure that Arrow R package is installed and available on all cluster nodes. -The current supported minimum version is 0.15.1; however, this might change between the minor releases since Arrow optimization in SparkR is experimental. +The current supported minimum version is 1.0.0; however, this might change between the minor releases since Arrow optimization in SparkR is experimental. ## Enabling for Conversion to/from R DataFrame, `dapply` and `gapply` From e79df80865783060784716558930dfbeb881de0d Mon Sep 17 00:00:00 2001 From: HyukjinKwon Date: Mon, 27 Jul 2020 12:14:17 +0900 Subject: [PATCH 2/4] Set a default python --- appveyor.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/appveyor.yml b/appveyor.yml index 1fd91daae901..cf64b497036a 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -57,6 +57,8 @@ environment: # "(converted from warning) unable to identify current timezone 'C':" for an unknown reason. # This environment variable works around to test SparkR against a higher version. R_REMOTES_NO_ERRORS_FROM_WARNINGS: true + # AppVeyor does not have python3 yet which is used by default. + PYSPARK_PYTHON: python test_script: - cmd: .\bin\spark-submit2.cmd --driver-java-options "-Dlog4j.configuration=file:///%CD:\=/%/R/log4j.properties" --conf spark.hadoop.fs.defaultFS="file:///" R\pkg\tests\run-all.R From 1af484d78baa0355d888759fa46fbf01b5716bd9 Mon Sep 17 00:00:00 2001 From: HyukjinKwon Date: Mon, 27 Jul 2020 13:01:08 +0900 Subject: [PATCH 3/4] Workaround --- appveyor.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/appveyor.yml b/appveyor.yml index cf64b497036a..cb843c19748d 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -59,6 +59,8 @@ environment: R_REMOTES_NO_ERRORS_FROM_WARNINGS: true # AppVeyor does not have python3 yet which is used by default. PYSPARK_PYTHON: python + # TODO(SPARK-32453): Remove SPARK_SCALA_VERSION environment and let load-spark-env scripts detect it. + SPARK_SCALA_VERSION: 2.12 test_script: - cmd: .\bin\spark-submit2.cmd --driver-java-options "-Dlog4j.configuration=file:///%CD:\=/%/R/log4j.properties" --conf spark.hadoop.fs.defaultFS="file:///" R\pkg\tests\run-all.R From 039d000e1e6ea021b7dc38b2849c6560dffbceaa Mon Sep 17 00:00:00 2001 From: HyukjinKwon Date: Mon, 27 Jul 2020 14:19:37 +0900 Subject: [PATCH 4/4] Revert AppVeyor changes back --- appveyor.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index cb843c19748d..1fd91daae901 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -57,10 +57,6 @@ environment: # "(converted from warning) unable to identify current timezone 'C':" for an unknown reason. # This environment variable works around to test SparkR against a higher version. R_REMOTES_NO_ERRORS_FROM_WARNINGS: true - # AppVeyor does not have python3 yet which is used by default. - PYSPARK_PYTHON: python - # TODO(SPARK-32453): Remove SPARK_SCALA_VERSION environment and let load-spark-env scripts detect it. - SPARK_SCALA_VERSION: 2.12 test_script: - cmd: .\bin\spark-submit2.cmd --driver-java-options "-Dlog4j.configuration=file:///%CD:\=/%/R/log4j.properties" --conf spark.hadoop.fs.defaultFS="file:///" R\pkg\tests\run-all.R