Skip to content

Commit 1a923f5

Browse files
maropucloud-fan
authored andcommitted
[SPARK-35479][SQL] Format PartitionFilters IN strings in scan nodes
### What changes were proposed in this pull request? This PR proposes to format strings correctly for `PushedFilters`. For example, `explain()` for a query below prints `v in (array('a'))` as `PushedFilters: [In(v, [WrappedArray(a)])]`; ``` scala> sql("create table t (v array<string>) using parquet") scala> sql("select * from t where v in (array('a'), null)").explain() == Physical Plan == *(1) Filter v#4 IN ([a],null) +- FileScan parquet default.t[v#4] Batched: false, DataFilters: [v#4 IN ([a],null)], Format: Parquet, Location: InMemoryFileIndex[file:/Users/maropu/Repositories/spark/spark-3.1.1-bin-hadoop2.7/spark-warehouse/t], PartitionFilters: [], PushedFilters: [In(v, [WrappedArray(a),null])], ReadSchema: struct<v:array<string>> ``` This PR makes `explain()` print it as `PushedFilters: [In(v, [[a]])]`; ``` scala> sql("select * from t where v in (array('a'), null)").explain() == Physical Plan == *(1) Filter v#4 IN ([a],null) +- FileScan parquet default.t[v#4] Batched: false, DataFilters: [v#4 IN ([a],null)], Format: Parquet, Location: InMemoryFileIndex[file:/Users/maropu/Repositories/spark/spark-3.1.1-bin-hadoop2.7/spark-warehouse/t], PartitionFilters: [], PushedFilters: [In(v, [[a],null])], ReadSchema: struct<v:array<string>> ``` NOTE: This PR includes a bugfix caused by apache#32577 (See the cloud-fan comment: https://github.com/apache/spark/pull/32577/files#r636108150). ### Why are the changes needed? To improve explain strings. ### Does this PR introduce _any_ user-facing change? Yes, this PR improves the explain strings for pushed-down filters. ### How was this patch tested? Added tests in `SQLQueryTestSuite`. Closes apache#32615 from maropu/ExplainPartitionFilters. Authored-by: Takeshi Yamamuro <[email protected]> Signed-off-by: Wenchen Fan <[email protected]>
1 parent e1296ea commit 1a923f5

File tree

4 files changed

+65
-3
lines changed

4 files changed

+65
-3
lines changed

sql/catalyst/src/main/scala/org/apache/spark/sql/sources/filters.scala

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -173,9 +173,14 @@ case class In(attribute: String, values: Array[Any]) extends Filter {
173173
a == attribute && vs.length == values.length && vs.zip(values).forall(x => x._1 == x._2)
174174
case _ => false
175175
}
176+
private def formatValue(v: Any): String = v match {
177+
case null => "null"
178+
case ar: Seq[Any] => ar.map(formatValue).mkString("[", ", ", "]")
179+
case _ => v.toString
180+
}
176181
override def toString: String = {
177182
// Sort elements for deterministic behaviours
178-
s"In($attribute, [${values.map(_.toString).sorted.mkString(",")}])"
183+
s"In($attribute, [${values.map(formatValue).sorted.mkString(",")}])"
179184
}
180185

181186
override def references: Array[String] = Array(attribute) ++ values.flatMap(findReferences)

sql/core/src/test/resources/sql-tests/inputs/explain.sql

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -124,3 +124,8 @@ DROP TABLE explain_temp1;
124124
DROP TABLE explain_temp2;
125125
DROP TABLE explain_temp3;
126126
DROP TABLE explain_temp4;
127+
128+
-- SPARK-35479: Format PartitionFilters IN strings in scan nodes
129+
CREATE table t(v array<string>) USING PARQUET;
130+
EXPLAIN SELECT * FROM t WHERE v IN (array('a'), null);
131+
DROP TABLE t;

sql/core/src/test/resources/sql-tests/results/explain-aqe.sql.out

Lines changed: 27 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
-- Automatically generated by SQLQueryTestSuite
2-
-- Number of queries: 24
2+
-- Number of queries: 27
33

44

55
-- !query
@@ -1113,3 +1113,29 @@ DROP TABLE explain_temp4
11131113
struct<>
11141114
-- !query output
11151115

1116+
1117+
1118+
-- !query
1119+
CREATE table t(v array<string>) USING PARQUET
1120+
-- !query schema
1121+
struct<>
1122+
-- !query output
1123+
1124+
1125+
1126+
-- !query
1127+
EXPLAIN SELECT * FROM t WHERE v IN (array('a'), null)
1128+
-- !query schema
1129+
struct<plan:string>
1130+
-- !query output
1131+
== Physical Plan ==
1132+
*Filter v#x IN ([a],null)
1133+
+- FileScan parquet default.t[v#x] Batched: false, DataFilters: [v#x IN ([a],null)], Format: Parquet, Location [not included in comparison]/{warehouse_dir}/t], PartitionFilters: [], PushedFilters: [In(v, [[a],null])], ReadSchema: struct<v:array<string>>
1134+
1135+
1136+
-- !query
1137+
DROP TABLE t
1138+
-- !query schema
1139+
struct<>
1140+
-- !query output
1141+

sql/core/src/test/resources/sql-tests/results/explain.sql.out

Lines changed: 27 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
-- Automatically generated by SQLQueryTestSuite
2-
-- Number of queries: 24
2+
-- Number of queries: 27
33

44

55
-- !query
@@ -1055,3 +1055,29 @@ DROP TABLE explain_temp4
10551055
struct<>
10561056
-- !query output
10571057

1058+
1059+
1060+
-- !query
1061+
CREATE table t(v array<string>) USING PARQUET
1062+
-- !query schema
1063+
struct<>
1064+
-- !query output
1065+
1066+
1067+
1068+
-- !query
1069+
EXPLAIN SELECT * FROM t WHERE v IN (array('a'), null)
1070+
-- !query schema
1071+
struct<plan:string>
1072+
-- !query output
1073+
== Physical Plan ==
1074+
*Filter v#x IN ([a],null)
1075+
+- FileScan parquet default.t[v#x] Batched: false, DataFilters: [v#x IN ([a],null)], Format: Parquet, Location [not included in comparison]/{warehouse_dir}/t], PartitionFilters: [], PushedFilters: [In(v, [[a],null])], ReadSchema: struct<v:array<string>>
1076+
1077+
1078+
-- !query
1079+
DROP TABLE t
1080+
-- !query schema
1081+
struct<>
1082+
-- !query output
1083+

0 commit comments

Comments
 (0)