Skip to content

Commit f0a3380

Browse files
AngersZhuuuucatalinii
authored andcommitted
[SPARK-37196][SQL] HiveDecimal enforcePrecisionScale failed return null
### What changes were proposed in this pull request? For case ``` withTempDir { dir => withSQLConf(HiveUtils.CONVERT_METASTORE_PARQUET.key -> "false") { withTable("test_precision") { val df = sql("SELECT 'dummy' AS name, 1000000000000000000010.7000000000000010 AS value") df.write.mode("Overwrite").parquet(dir.getAbsolutePath) sql( s""" |CREATE EXTERNAL TABLE test_precision(name STRING, value DECIMAL(18,6)) |STORED AS PARQUET LOCATION '${dir.getAbsolutePath}' |""".stripMargin) checkAnswer(sql("SELECT * FROM test_precision"), Row("dummy", null)) } } } ``` We write a data with schema It's caused by you create a df with ``` root |-- name: string (nullable = false) |-- value: decimal(38,16) (nullable = false) ``` but create table schema ``` root |-- name: string (nullable = false) |-- value: decimal(18,6) (nullable = false) ``` This will cause enforcePrecisionScale return `null` ``` public HiveDecimal getPrimitiveJavaObject(Object o) { return o == null ? null : this.enforcePrecisionScale(((HiveDecimalWritable)o).getHiveDecimal()); } ``` Then throw NPE when call `toCatalystDecimal ` We should judge if the return value is `null` to avoid throw NPE ### Why are the changes needed? Fix bug ### Does this PR introduce _any_ user-facing change? No ### How was this patch tested? Added UT Closes apache#34519 from AngersZhuuuu/SPARK-37196. Authored-by: Angerszhuuuu <[email protected]> Signed-off-by: Dongjoon Hyun <[email protected]> (cherry picked from commit a4f8ffb) Signed-off-by: Dongjoon Hyun <[email protected]>
1 parent 7d86651 commit f0a3380

File tree

2 files changed

+29
-3
lines changed

2 files changed

+29
-3
lines changed

sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveShim.scala

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -101,10 +101,19 @@ private[hive] object HiveShim {
101101

102102
def toCatalystDecimal(hdoi: HiveDecimalObjectInspector, data: Any): Decimal = {
103103
if (hdoi.preferWritable()) {
104-
Decimal(hdoi.getPrimitiveWritableObject(data).getHiveDecimal().bigDecimalValue,
105-
hdoi.precision(), hdoi.scale())
104+
val value = hdoi.getPrimitiveWritableObject(data)
105+
if (value == null) {
106+
null
107+
} else {
108+
Decimal(value.getHiveDecimal().bigDecimalValue, hdoi.precision(), hdoi.scale())
109+
}
106110
} else {
107-
Decimal(hdoi.getPrimitiveJavaObject(data).bigDecimalValue(), hdoi.precision(), hdoi.scale())
111+
val value = hdoi.getPrimitiveJavaObject(data)
112+
if (value == null) {
113+
null
114+
} else {
115+
Decimal(value.bigDecimalValue(), hdoi.precision(), hdoi.scale())
116+
}
108117
}
109118
}
110119

sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2642,6 +2642,23 @@ abstract class SQLQuerySuiteBase extends QueryTest with SQLTestUtils with TestHi
26422642
}
26432643
}
26442644
}
2645+
2646+
test("SPARK-37196: HiveDecimal Precision Scale match failed should return null") {
2647+
withTempDir { dir =>
2648+
withSQLConf(HiveUtils.CONVERT_METASTORE_PARQUET.key -> "false") {
2649+
withTable("test_precision") {
2650+
val df = sql(s"SELECT 'dummy' AS name, ${"1" * 20}.${"2" * 18} AS value")
2651+
df.write.mode("Overwrite").parquet(dir.getAbsolutePath)
2652+
sql(
2653+
s"""
2654+
|CREATE EXTERNAL TABLE test_precision(name STRING, value DECIMAL(18,6))
2655+
|STORED AS PARQUET LOCATION '${dir.getAbsolutePath}'
2656+
|""".stripMargin)
2657+
checkAnswer(sql("SELECT * FROM test_precision"), Row("dummy", null))
2658+
}
2659+
}
2660+
}
2661+
}
26452662
}
26462663

26472664
@SlowHiveTest

0 commit comments

Comments
 (0)