From 7722de654121915ec55e3f0885602f895710167a Mon Sep 17 00:00:00 2001 From: HyukjinKwon Date: Thu, 30 Jul 2020 21:01:31 +0900 Subject: [PATCH 1/4] Installs qpdf package for CRAN check in GitHub Actions --- .github/workflows/master.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 881e11d5ea4f..009ebe90ddf5 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -164,7 +164,8 @@ jobs: - name: Install R packages if: contains(matrix.modules, 'sparkr') run: | - sudo apt-get install -y libcurl4-openssl-dev + # qpdf is required to reduce the size of PDFs to make CRAN check pass. See SPARK-32497. + sudo apt-get install -y libcurl4-openssl-dev qpdf sudo Rscript -e "install.packages(c('knitr', 'rmarkdown', 'testthat', 'devtools', 'e1071', 'survival', 'arrow', 'roxygen2'), repos='https://cloud.r-project.org/')" # Show installed packages in R. sudo Rscript -e 'pkg_list <- as.data.frame(installed.packages()[, c(1,3:4)]); pkg_list[is.na(pkg_list$Priority), 1:2, drop = FALSE]' From 21d6bfc318240f5031908789eb20d5f30c839ab3 Mon Sep 17 00:00:00 2001 From: HyukjinKwon Date: Thu, 30 Jul 2020 23:32:15 +0900 Subject: [PATCH 2/4] Tweak run-tests.py script --- dev/run-tests.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/dev/run-tests.py b/dev/run-tests.py index 5d8b7a075826..6aae3bdaefaf 100755 --- a/dev/run-tests.py +++ b/dev/run-tests.py @@ -610,8 +610,14 @@ def main(): # Install SparkR should_only_test_modules = opts.modules is not None - if not should_only_test_modules: + test_modules = [] + if should_only_test_modules: + str_test_modules = [m.strip() for m in opts.modules.split(",")] + test_modules = [m for m in modules.all_modules if m.name in str_test_modules] + + if not should_only_test_modules or modules.sparkr in test_modules: # If tests modules are specified, we will not run R linter. + # SparkR needs the manual SparkR installation. if which("R"): run_cmd([os.path.join(SPARK_HOME, "R", "install-dev.sh")]) else: @@ -642,15 +648,11 @@ def main(): "and Hive profile", hive_version, "under environment", test_env) extra_profiles = get_hadoop_profiles(hadoop_version) + get_hive_profiles(hive_version) - changed_modules = None - test_modules = None - changed_files = None + changed_modules = [] + changed_files = [] included_tags = [] excluded_tags = [] if should_only_test_modules: - str_test_modules = [m.strip() for m in opts.modules.split(",")] - test_modules = [m for m in modules.all_modules if m.name in str_test_modules] - # If we're running the tests in Github Actions, attempt to detect and test # only the affected modules. if test_env == "github_actions": From 772f7102a5f5ddfd62154a84ca1151518660f4fd Mon Sep 17 00:00:00 2001 From: HyukjinKwon Date: Thu, 30 Jul 2020 23:53:52 +0900 Subject: [PATCH 3/4] Show qpdf still is the cause --- .github/workflows/master.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 009ebe90ddf5..881e11d5ea4f 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -164,8 +164,7 @@ jobs: - name: Install R packages if: contains(matrix.modules, 'sparkr') run: | - # qpdf is required to reduce the size of PDFs to make CRAN check pass. See SPARK-32497. - sudo apt-get install -y libcurl4-openssl-dev qpdf + sudo apt-get install -y libcurl4-openssl-dev sudo Rscript -e "install.packages(c('knitr', 'rmarkdown', 'testthat', 'devtools', 'e1071', 'survival', 'arrow', 'roxygen2'), repos='https://cloud.r-project.org/')" # Show installed packages in R. sudo Rscript -e 'pkg_list <- as.data.frame(installed.packages()[, c(1,3:4)]); pkg_list[is.na(pkg_list$Priority), 1:2, drop = FALSE]' From 8ff181e0fd0433fbc721c99e6f5aefbf2de58d6d Mon Sep 17 00:00:00 2001 From: HyukjinKwon Date: Thu, 30 Jul 2020 23:54:32 +0900 Subject: [PATCH 4/4] Revert "Show qpdf still is the cause" This reverts commit 772f7102a5f5ddfd62154a84ca1151518660f4fd. --- .github/workflows/master.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 881e11d5ea4f..009ebe90ddf5 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -164,7 +164,8 @@ jobs: - name: Install R packages if: contains(matrix.modules, 'sparkr') run: | - sudo apt-get install -y libcurl4-openssl-dev + # qpdf is required to reduce the size of PDFs to make CRAN check pass. See SPARK-32497. + sudo apt-get install -y libcurl4-openssl-dev qpdf sudo Rscript -e "install.packages(c('knitr', 'rmarkdown', 'testthat', 'devtools', 'e1071', 'survival', 'arrow', 'roxygen2'), repos='https://cloud.r-project.org/')" # Show installed packages in R. sudo Rscript -e 'pkg_list <- as.data.frame(installed.packages()[, c(1,3:4)]); pkg_list[is.na(pkg_list$Priority), 1:2, drop = FALSE]'