Skip to content

Commit 286a376

Browse files
ulysses-youyaooqinn
authored andcommitted
[SPARK-37098][SQL] Alter table properties should invalidate cache
### What changes were proposed in this pull request? Invalidate the table cache after alter table properties (set and unset). ### Why are the changes needed? The table properties can change the behavior of wriing. e.g. the parquet table with `parquet.compression`. If you execute the following SQL, we will get the file with snappy compression rather than zstd. ``` CREATE TABLE t (c int) STORED AS PARQUET; // cache table metadata SELECT * FROM t; ALTER TABLE t SET TBLPROPERTIES('parquet.compression'='zstd'); INSERT INTO TABLE t values(1); ``` So we should invalidate the table cache after alter table properties. ### Does this PR introduce _any_ user-facing change? yes, bug fix ### How was this patch tested? Add test Closes #34365 from ulysses-you/SPARK-37098. Authored-by: ulysses-you <ulyssesyou18@gmail.com> Signed-off-by: Kent Yao <yao@apache.org> (cherry picked from commit 02d3b3b) Signed-off-by: Kent Yao <yao@apache.org>
1 parent c23698f commit 286a376

File tree

2 files changed

+26
-0
lines changed

2 files changed

+26
-0
lines changed

sql/core/src/main/scala/org/apache/spark/sql/execution/command/ddl.scala

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -277,6 +277,7 @@ case class AlterTableSetPropertiesCommand(
277277
properties = table.properties ++ properties,
278278
comment = properties.get(TableCatalog.PROP_COMMENT).orElse(table.comment))
279279
catalog.alterTable(newTable)
280+
catalog.invalidateCachedTable(tableName)
280281
Seq.empty[Row]
281282
}
282283

@@ -315,6 +316,7 @@ case class AlterTableUnsetPropertiesCommand(
315316
val newProperties = table.properties.filter { case (k, _) => !propKeys.contains(k) }
316317
val newTable = table.copy(properties = newProperties, comment = tableComment)
317318
catalog.alterTable(newTable)
319+
catalog.invalidateCachedTable(tableName)
318320
Seq.empty[Row]
319321
}
320322

sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveParquetSuite.scala

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -123,4 +123,28 @@ class HiveParquetSuite extends QueryTest with ParquetTest with TestHiveSingleton
123123
assert(msg.contains("cannot resolve '`c3`' given input columns"))
124124
}
125125
}
126+
127+
test("SPARK-37098: Alter table properties should invalidate cache") {
128+
// specify the compression in case we change it in future
129+
withSQLConf(SQLConf.PARQUET_COMPRESSION.key -> "snappy") {
130+
withTempPath { dir =>
131+
withTable("t") {
132+
sql(s"CREATE TABLE t (c int) STORED AS PARQUET LOCATION '${dir.getCanonicalPath}'")
133+
// cache table metadata
134+
sql("SELECT * FROM t")
135+
sql("ALTER TABLE t SET TBLPROPERTIES('parquet.compression'='zstd')")
136+
sql("INSERT INTO TABLE t values(1)")
137+
val files1 = dir.listFiles().filter(_.getName.endsWith("zstd.parquet"))
138+
assert(files1.length == 1)
139+
140+
// cache table metadata again
141+
sql("SELECT * FROM t")
142+
sql("ALTER TABLE t UNSET TBLPROPERTIES('parquet.compression')")
143+
sql("INSERT INTO TABLE t values(1)")
144+
val files2 = dir.listFiles().filter(_.getName.endsWith("snappy.parquet"))
145+
assert(files2.length == 1)
146+
}
147+
}
148+
}
149+
}
126150
}

0 commit comments

Comments
 (0)