diff --git a/common/tags/src/test/java/org/apache/spark/tags/ChromeUITest.java b/common/tags/src/test/java/org/apache/spark/tags/ChromeUITest.java new file mode 100644 index 000000000000..e3fed3d656d2 --- /dev/null +++ b/common/tags/src/test/java/org/apache/spark/tags/ChromeUITest.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.tags; + +import java.lang.annotation.*; + +import org.scalatest.TagAnnotation; + +@TagAnnotation +@Retention(RetentionPolicy.RUNTIME) +@Target({ElementType.METHOD, ElementType.TYPE}) +public @interface ChromeUITest { } diff --git a/core/src/test/scala/org/apache/spark/ui/ChromeUISeleniumSuite.scala b/core/src/test/scala/org/apache/spark/ui/ChromeUISeleniumSuite.scala new file mode 100644 index 000000000000..9ba705c4abd7 --- /dev/null +++ b/core/src/test/scala/org/apache/spark/ui/ChromeUISeleniumSuite.scala @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.ui + +import org.openqa.selenium.WebDriver +import org.openqa.selenium.chrome.{ChromeDriver, ChromeOptions} + +import org.apache.spark.tags.ChromeUITest + +/** + * Selenium tests for the Spark Web UI with Chrome. + */ +@ChromeUITest +class ChromeUISeleniumSuite extends RealBrowserUISeleniumSuite("webdriver.chrome.driver") { + + override var webDriver: WebDriver = _ + + override def beforeAll(): Unit = { + super.beforeAll() + val chromeOptions = new ChromeOptions + chromeOptions.addArguments("--headless", "--disable-gpu") + webDriver = new ChromeDriver(chromeOptions) + } + + override def afterAll(): Unit = { + try { + if (webDriver != null) { + webDriver.quit() + } + } finally { + super.afterAll() + } + } +} diff --git a/core/src/test/scala/org/apache/spark/ui/RealBrowserUISeleniumSuite.scala b/core/src/test/scala/org/apache/spark/ui/RealBrowserUISeleniumSuite.scala new file mode 100644 index 000000000000..84f888267857 --- /dev/null +++ b/core/src/test/scala/org/apache/spark/ui/RealBrowserUISeleniumSuite.scala @@ -0,0 +1,109 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.ui + +import org.openqa.selenium.{By, WebDriver} +import org.scalatest._ +import org.scalatest.concurrent.Eventually._ +import org.scalatest.time.SpanSugar._ +import org.scalatestplus.selenium.WebBrowser + +import org.apache.spark._ +import org.apache.spark.LocalSparkContext.withSpark +import org.apache.spark.internal.config.MEMORY_OFFHEAP_SIZE +import org.apache.spark.internal.config.UI.{UI_ENABLED, UI_KILL_ENABLED, UI_PORT} +import org.apache.spark.util.CallSite + +/** + * Selenium tests for the Spark Web UI with real web browsers. + */ +abstract class RealBrowserUISeleniumSuite(val driverProp: String) + extends SparkFunSuite with WebBrowser with Matchers with BeforeAndAfterAll { + + implicit var webDriver: WebDriver + private val driverPropPrefix = "spark.test." + + override def beforeAll(): Unit = { + super.beforeAll() + assume( + sys.props(driverPropPrefix + driverProp) !== null, + "System property " + driverPropPrefix + driverProp + + " should be set to the corresponding driver path.") + sys.props(driverProp) = sys.props(driverPropPrefix + driverProp) + } + + override def afterAll(): Unit = { + sys.props.remove(driverProp) + super.afterAll() + } + + test("SPARK-31534: text for tooltip should be escaped") { + withSpark(newSparkContext()) { sc => + sc.setLocalProperty(CallSite.LONG_FORM, "collect at :25") + sc.setLocalProperty(CallSite.SHORT_FORM, "collect at :25") + sc.parallelize(1 to 10).collect + + eventually(timeout(10.seconds), interval(50.milliseconds)) { + goToUi(sc, "/jobs") + + val jobDesc = + webDriver.findElement(By.cssSelector("div[class='application-timeline-content']")) + jobDesc.getAttribute("data-title") should include ("collect at <console>:25") + + goToUi(sc, "/jobs/job/?id=0") + webDriver.get(sc.ui.get.webUrl.stripSuffix("/") + "/jobs/job/?id=0") + val stageDesc = webDriver.findElement(By.cssSelector("div[class='job-timeline-content']")) + stageDesc.getAttribute("data-title") should include ("collect at <console>:25") + + // Open DAG Viz. + webDriver.findElement(By.id("job-dag-viz")).click() + val nodeDesc = webDriver.findElement(By.cssSelector("g[class='node_0 node']")) + nodeDesc.getAttribute("name") should include ("collect at <console>:25") + } + } + } + + /** + * Create a test SparkContext with the SparkUI enabled. + * It is safe to `get` the SparkUI directly from the SparkContext returned here. + */ + private def newSparkContext( + killEnabled: Boolean = true, + master: String = "local", + additionalConfs: Map[String, String] = Map.empty): SparkContext = { + val conf = new SparkConf() + .setMaster(master) + .setAppName("test") + .set(UI_ENABLED, true) + .set(UI_PORT, 0) + .set(UI_KILL_ENABLED, killEnabled) + .set(MEMORY_OFFHEAP_SIZE.key, "64m") + additionalConfs.foreach { case (k, v) => conf.set(k, v) } + val sc = new SparkContext(conf) + assert(sc.ui.isDefined) + sc + } + + def goToUi(sc: SparkContext, path: String): Unit = { + goToUi(sc.ui.get, path) + } + + def goToUi(ui: SparkUI, path: String): Unit = { + go to (ui.webUrl.stripSuffix("/") + path) + } +} diff --git a/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala b/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala index 3ec938511640..909056eab8c5 100644 --- a/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala +++ b/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala @@ -773,33 +773,6 @@ class UISeleniumSuite extends SparkFunSuite with WebBrowser with Matchers with B } } - test("SPARK-31534: text for tooltip should be escaped") { - withSpark(newSparkContext()) { sc => - sc.setLocalProperty(CallSite.LONG_FORM, "collect at :25") - sc.setLocalProperty(CallSite.SHORT_FORM, "collect at :25") - sc.parallelize(1 to 10).collect - - val driver = webDriver.asInstanceOf[HtmlUnitDriver] - driver.setJavascriptEnabled(true) - - eventually(timeout(10.seconds), interval(50.milliseconds)) { - goToUi(sc, "/jobs") - val jobDesc = - driver.findElement(By.cssSelector("div[class='application-timeline-content']")) - jobDesc.getAttribute("data-title") should include ("collect at <console>:25") - - goToUi(sc, "/jobs/job/?id=0") - val stageDesc = driver.findElement(By.cssSelector("div[class='job-timeline-content']")) - stageDesc.getAttribute("data-title") should include ("collect at <console>:25") - - // Open DAG Viz. - driver.findElement(By.id("job-dag-viz")).click() - val nodeDesc = driver.findElement(By.cssSelector("g[class='node_0 node']")) - nodeDesc.getAttribute("name") should include ("collect at <console>:25") - } - } - } - def goToUi(sc: SparkContext, path: String): Unit = { goToUi(sc.ui.get, path) } diff --git a/pom.xml b/pom.xml index 0231f8fd0791..d8d559550a5b 100644 --- a/pom.xml +++ b/pom.xml @@ -204,6 +204,9 @@ org.fusesource.leveldbjni ${java.home} + + + org.apache.spark.tags.ChromeUITest @@ -243,6 +246,7 @@ things breaking. --> ${session.executionRootDirectory} + 1g @@ -2504,10 +2508,11 @@ false false true + ${spark.test.webdriver.chrome.driver} __not_used__ - ${test.exclude.tags} + ${test.exclude.tags},${test.default.exclude.tags} ${test.include.tags} diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index 7bd92b3f15b5..c9521ea26857 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -966,6 +966,9 @@ object TestSettings { "2.12" } */ + + private val defaultExcludedTags = Seq("org.apache.spark.tags.ChromeUITest") + lazy val settings = Seq ( // Fork new JVMs for tests and set Java options for those fork := true, @@ -1003,6 +1006,10 @@ object TestSettings { sys.props.get("test.exclude.tags").map { tags => tags.split(",").flatMap { tag => Seq("-l", tag) }.toSeq }.getOrElse(Nil): _*), + testOptions in Test += Tests.Argument(TestFrameworks.ScalaTest, + sys.props.get("test.default.exclude.tags").map(tags => tags.split(",").toSeq) + .map(tags => tags.filter(!_.trim.isEmpty)).getOrElse(defaultExcludedTags) + .flatMap(tag => Seq("-l", tag)): _*), testOptions in Test += Tests.Argument(TestFrameworks.JUnit, sys.props.get("test.exclude.tags").map { tags => Seq("--exclude-categories=" + tags)