diff --git a/dev/deps/spark-deps-hadoop-palantir b/dev/deps/spark-deps-hadoop-palantir index 3c0f239d24ca..e1a8bbae9302 100644 --- a/dev/deps/spark-deps-hadoop-palantir +++ b/dev/deps/spark-deps-hadoop-palantir @@ -63,26 +63,26 @@ gson-2.2.4.jar guava-14.0.1.jar guice-3.0.jar guice-servlet-3.0.jar -hadoop-annotations-2.9.2-palantir.4.jar -hadoop-auth-2.9.2-palantir.4.jar -hadoop-aws-2.9.2-palantir.4.jar -hadoop-azure-2.9.2-palantir.4.jar -hadoop-azure-datalake-2.9.2-palantir.4.jar -hadoop-client-2.9.2-palantir.4.jar -hadoop-common-2.9.2-palantir.4.jar -hadoop-hdfs-client-2.9.2-palantir.4.jar -hadoop-mapreduce-client-app-2.9.2-palantir.4.jar -hadoop-mapreduce-client-common-2.9.2-palantir.4.jar -hadoop-mapreduce-client-core-2.9.2-palantir.4.jar -hadoop-mapreduce-client-jobclient-2.9.2-palantir.4.jar -hadoop-mapreduce-client-shuffle-2.9.2-palantir.4.jar -hadoop-openstack-2.9.2-palantir.4.jar -hadoop-yarn-api-2.9.2-palantir.4.jar -hadoop-yarn-client-2.9.2-palantir.4.jar -hadoop-yarn-common-2.9.2-palantir.4.jar -hadoop-yarn-registry-2.9.2-palantir.4.jar -hadoop-yarn-server-common-2.9.2-palantir.4.jar -hadoop-yarn-server-web-proxy-2.9.2-palantir.4.jar +hadoop-annotations-2.9.2-palantir.5.jar +hadoop-auth-2.9.2-palantir.5.jar +hadoop-aws-2.9.2-palantir.5.jar +hadoop-azure-2.9.2-palantir.5.jar +hadoop-azure-datalake-2.9.2-palantir.5.jar +hadoop-client-2.9.2-palantir.5.jar +hadoop-common-2.9.2-palantir.5.jar +hadoop-hdfs-client-2.9.2-palantir.5.jar +hadoop-mapreduce-client-app-2.9.2-palantir.5.jar +hadoop-mapreduce-client-common-2.9.2-palantir.5.jar +hadoop-mapreduce-client-core-2.9.2-palantir.5.jar +hadoop-mapreduce-client-jobclient-2.9.2-palantir.5.jar +hadoop-mapreduce-client-shuffle-2.9.2-palantir.5.jar +hadoop-openstack-2.9.2-palantir.5.jar +hadoop-yarn-api-2.9.2-palantir.5.jar +hadoop-yarn-client-2.9.2-palantir.5.jar +hadoop-yarn-common-2.9.2-palantir.5.jar +hadoop-yarn-registry-2.9.2-palantir.5.jar +hadoop-yarn-server-common-2.9.2-palantir.5.jar +hadoop-yarn-server-web-proxy-2.9.2-palantir.5.jar hibernate-validator-5.2.4.Final.jar hk2-api-2.5.0-b32.jar hk2-locator-2.5.0-b32.jar @@ -192,7 +192,7 @@ shapeless_2.11-2.3.2.jar slf4j-api-1.7.25.jar slf4j-log4j12-1.7.25.jar snakeyaml-1.23.jar -snappy-java-1.1.7.2.jar +snappy-java-1.1.7.3.jar spire-macros_2.11-0.13.0.jar spire_2.11-0.13.0.jar stax-api-1.0-2.jar diff --git a/dists/hadoop-palantir-bom/pom.xml b/dists/hadoop-palantir-bom/pom.xml index 57559945f726..fcee9d7cc9e1 100644 --- a/dists/hadoop-palantir-bom/pom.xml +++ b/dists/hadoop-palantir-bom/pom.xml @@ -34,7 +34,7 @@ spark-dist-hadoop-palantir-bom none none - 2.9.2-palantir.4 + 2.9.2-palantir.5 2.7.1 diff --git a/pom.xml b/pom.xml index 05c7d9e1a2c3..17b5e5fc1d2b 100644 --- a/pom.xml +++ b/pom.xml @@ -127,7 +127,7 @@ spark 1.7.25 1.2.17 - 2.9.2-palantir.4 + 2.9.2-palantir.5 2.5.0 ${hadoop.version} 3.4.7 @@ -180,7 +180,7 @@ true 1.9.13 2.9.7 - 1.1.7.2 + 1.1.7.3 1.1.2 1.2.0-incubating 1.11 diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala index 92553b7dae2b..bf9dffd96a35 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala @@ -798,6 +798,7 @@ object CollapseWindow extends Rule[LogicalPlan] { def apply(plan: LogicalPlan): LogicalPlan = plan transformUp { case w1 @ Window(we1, ps1, os1, w2 @ Window(we2, ps2, os2, grandChild)) if ps1 == ps2 && os1 == os2 && w1.references.intersect(w2.windowOutputSet).isEmpty && + we1.nonEmpty && we2.nonEmpty && // This assumes Window contains the same type of window expressions. This is ensured // by ExtractWindowFunctions. WindowFunctionType.functionType(we1.head) == WindowFunctionType.functionType(we2.head) => diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/CollapseWindowSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/CollapseWindowSuite.scala index 52054c2f8bd8..3b3b4907eea8 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/CollapseWindowSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/CollapseWindowSuite.scala @@ -89,4 +89,15 @@ class CollapseWindowSuite extends PlanTest { val optimized = Optimize.execute(query.analyze) comparePlans(optimized, expected) } + + test("Skip windows with empty window expressions") { + val query = testRelation + .window(Seq(), partitionSpec1, orderSpec1) + .window(Seq(sum(a).as('sum_a)), partitionSpec1, orderSpec1) + + val optimized = Optimize.execute(query.analyze) + val correctAnswer = query.analyze + + comparePlans(optimized, correctAnswer) + } }