diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 29f98ed36e2e..f81299d43a9d 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -27,20 +27,6 @@ jobs: # Kinesis tests depends on external Amazon kinesis service. # Note that the modules below are from sparktestsupport/modules.py. modules: - - |- - core, unsafe, kvstore, avro, - network-common, network-shuffle, repl, launcher, - examples, sketch, graphx - - |- - catalyst, hive-thriftserver - - |- - streaming, sql-kafka-0-10, streaming-kafka-0-10, - mllib-local, mllib, - yarn, mesos, kubernetes, hadoop-cloud, spark-ganglia-lgpl - - |- - pyspark-sql, pyspark-mllib, pyspark-resource - - |- - pyspark-core, pyspark-streaming, pyspark-ml - |- sparkr # Here, we split Hive and SQL tests into some of slow ones and the rest of them. @@ -48,19 +34,6 @@ jobs: excluded-tags: [""] comment: [""] include: - # Hive tests - - modules: hive - java: 1.8 - hadoop: hadoop3.2 - hive: hive2.3 - included-tags: org.apache.spark.tags.SlowHiveTest - comment: "- slow tests" - - modules: hive - java: 1.8 - hadoop: hadoop3.2 - hive: hive2.3 - excluded-tags: org.apache.spark.tags.SlowHiveTest - comment: "- other tests" # SQL tests - modules: sql java: 1.8 @@ -68,12 +41,6 @@ jobs: hive: hive2.3 included-tags: org.apache.spark.tags.ExtendedSQLTest comment: "- slow tests" - - modules: sql - java: 1.8 - hadoop: hadoop3.2 - hive: hive2.3 - excluded-tags: org.apache.spark.tags.ExtendedSQLTest - comment: "- other tests" env: MODULES_TO_TEST: ${{ matrix.modules }} EXCLUDED_TAGS: ${{ matrix.excluded-tags }} @@ -174,70 +141,3 @@ jobs: mkdir -p ~/.m2 ./dev/run-tests --parallelism 2 --modules "$MODULES_TO_TEST" --included-tags "$INCLUDED_TAGS" --excluded-tags "$EXCLUDED_TAGS" rm -rf ~/.m2/repository/org/apache/spark - - # Static analysis, and documentation build - lint: - name: Linters, licenses, dependencies and documentation generation - runs-on: ubuntu-latest - steps: - - name: Checkout Spark repository - uses: actions/checkout@v2 - - name: Cache Maven local repository - uses: actions/cache@v2 - with: - path: ~/.m2/repository - key: docs-maven-repo-${{ hashFiles('**/pom.xml') }} - restore-keys: | - docs-maven- - - name: Install JDK 1.8 - uses: actions/setup-java@v1 - with: - java-version: 1.8 - - name: Install Python 3.6 - uses: actions/setup-python@v2 - with: - python-version: 3.6 - architecture: x64 - - name: Install Python linter dependencies - run: | - # TODO(SPARK-32407): Sphinx 3.1+ does not correctly index nested classes. - # See also https://github.com/sphinx-doc/sphinx/issues/7551. - pip3 install flake8 'sphinx<3.1.0' numpy pydata_sphinx_theme - - name: Install R 3.6 - uses: r-lib/actions/setup-r@v1 - with: - r-version: 3.6 - - name: Install R linter dependencies and SparkR - run: | - sudo apt-get install -y libcurl4-openssl-dev - sudo Rscript -e "install.packages(c('devtools'), repos='https://cloud.r-project.org/')" - sudo Rscript -e "devtools::install_github('jimhester/lintr@v2.0.0')" - ./R/install-dev.sh - - name: Install Ruby 2.7 for documentation generation - uses: actions/setup-ruby@v1 - with: - ruby-version: 2.7 - - name: Install dependencies for documentation generation - run: | - sudo apt-get install -y libcurl4-openssl-dev pandoc - # TODO(SPARK-32407): Sphinx 3.1+ does not correctly index nested classes. - # See also https://github.com/sphinx-doc/sphinx/issues/7551. - pip install 'sphinx<3.1.0' mkdocs numpy pydata_sphinx_theme - gem install jekyll jekyll-redirect-from rouge - sudo Rscript -e "install.packages(c('devtools', 'testthat', 'knitr', 'rmarkdown', 'roxygen2'), repos='https://cloud.r-project.org/')" - - name: Scala linter - run: ./dev/lint-scala - - name: Java linter - run: ./dev/lint-java - - name: Python linter - run: ./dev/lint-python - - name: R linter - run: ./dev/lint-r - - name: License test - run: ./dev/check-license - - name: Dependencies test - run: ./dev/test-dependencies.sh - - name: Run documentation build - run: | - cd docs - jekyll build diff --git a/core/src/main/scala/org/apache/spark/TestUtils.scala b/core/src/main/scala/org/apache/spark/TestUtils.scala index d459627930f4..7ee09f80c191 100644 --- a/core/src/main/scala/org/apache/spark/TestUtils.scala +++ b/core/src/main/scala/org/apache/spark/TestUtils.scala @@ -236,8 +236,33 @@ private[spark] object TestUtils { * Test if a command is available. */ def testCommandAvailable(command: String): Boolean = { - val attempt = Try(Process(command).run(ProcessLogger(_ => ())).exitValue()) - attempt.isSuccess && attempt.get == 0 + val messages0 = ArrayBuffer[String]() + val messages1 = ArrayBuffer[String]() + // scalastyle:off println + val logger = ProcessLogger( + (o: String) => messages0.append(o), + (e: String) => messages1.append(e)) + // scalastyle:on println + + val attempt = if (Utils.isWindows) { + Try(Process(s"WHERE $command").run(ProcessLogger(_ => ())).exitValue()) + } else { + Try(Process(Seq( + "sh", "-c", s"command -v $command")).run(logger).exitValue()) + } + val ret = attempt.isSuccess && attempt.get == 0 + val anotherAttempt = + Try(Process(command).run(ProcessLogger(_ => ())).exitValue()) + val anotherRet = anotherAttempt.isSuccess && anotherAttempt.get == 0 + if (ret != anotherRet) { + throw new IllegalArgumentException( + s"command -v $command, stdout: $messages0, stderr: $messages1" + + s"anotherAttempt.isSuccess: ${anotherAttempt.isSuccess}, " + + s"anotherAttempt.get: ${anotherAttempt.get}" + + s"attempt.isSuccess: ${attempt.isSuccess}" + + s"attempt.get: ${attempt.get}") + } + ret } /** diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala index 920f6385f8e1..62a411a56159 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala @@ -92,7 +92,8 @@ abstract class SQLQuerySuiteBase extends QueryTest with SQLTestUtils with TestHi test("script") { withTempView("script_table") { assume(TestUtils.testCommandAvailable("/bin/bash")) - assume(TestUtils.testCommandAvailable("echo | sed")) + assume(TestUtils.testCommandAvailable("echo")) + assume(TestUtils.testCommandAvailable("sed")) val scriptFilePath = getTestResourcePath("test_script.sh") val df = Seq(("x1", "y1", "z1"), ("x2", "y2", "z2")).toDF("c1", "c2", "c3") df.createOrReplaceTempView("script_table")