Skip to content

Commit aa4ef0e

Browse files
AngersZhuuuudongjoon-hyun
authored andcommitted
[SPARK-37196][SQL] HiveDecimal enforcePrecisionScale failed return null
For case ``` withTempDir { dir => withSQLConf(HiveUtils.CONVERT_METASTORE_PARQUET.key -> "false") { withTable("test_precision") { val df = sql("SELECT 'dummy' AS name, 1000000000000000000010.7000000000000010 AS value") df.write.mode("Overwrite").parquet(dir.getAbsolutePath) sql( s""" |CREATE EXTERNAL TABLE test_precision(name STRING, value DECIMAL(18,6)) |STORED AS PARQUET LOCATION '${dir.getAbsolutePath}' |""".stripMargin) checkAnswer(sql("SELECT * FROM test_precision"), Row("dummy", null)) } } } ``` We write a data with schema It's caused by you create a df with ``` root |-- name: string (nullable = false) |-- value: decimal(38,16) (nullable = false) ``` but create table schema ``` root |-- name: string (nullable = false) |-- value: decimal(18,6) (nullable = false) ``` This will cause enforcePrecisionScale return `null` ``` public HiveDecimal getPrimitiveJavaObject(Object o) { return o == null ? null : this.enforcePrecisionScale(((HiveDecimalWritable)o).getHiveDecimal()); } ``` Then throw NPE when call `toCatalystDecimal ` We should judge if the return value is `null` to avoid throw NPE Fix bug No Added UT Closes #34519 from AngersZhuuuu/SPARK-37196. Authored-by: Angerszhuuuu <[email protected]> Signed-off-by: Dongjoon Hyun <[email protected]> (cherry picked from commit a4f8ffb) Signed-off-by: Dongjoon Hyun <[email protected]>
1 parent 615e525 commit aa4ef0e

File tree

2 files changed

+29
-3
lines changed

2 files changed

+29
-3
lines changed

sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveShim.scala

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -103,10 +103,19 @@ private[hive] object HiveShim {
103103

104104
def toCatalystDecimal(hdoi: HiveDecimalObjectInspector, data: Any): Decimal = {
105105
if (hdoi.preferWritable()) {
106-
Decimal(hdoi.getPrimitiveWritableObject(data).getHiveDecimal().bigDecimalValue,
107-
hdoi.precision(), hdoi.scale())
106+
val value = hdoi.getPrimitiveWritableObject(data)
107+
if (value == null) {
108+
null
109+
} else {
110+
Decimal(value.getHiveDecimal().bigDecimalValue, hdoi.precision(), hdoi.scale())
111+
}
108112
} else {
109-
Decimal(hdoi.getPrimitiveJavaObject(data).bigDecimalValue(), hdoi.precision(), hdoi.scale())
113+
val value = hdoi.getPrimitiveJavaObject(data)
114+
if (value == null) {
115+
null
116+
} else {
117+
Decimal(value.bigDecimalValue(), hdoi.precision(), hdoi.scale())
118+
}
110119
}
111120
}
112121

sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2558,6 +2558,23 @@ abstract class SQLQuerySuiteBase extends QueryTest with SQLTestUtils with TestHi
25582558
}
25592559
}
25602560
}
2561+
2562+
test("SPARK-37196: HiveDecimal Precision Scale match failed should return null") {
2563+
withTempDir { dir =>
2564+
withSQLConf(HiveUtils.CONVERT_METASTORE_PARQUET.key -> "false") {
2565+
withTable("test_precision") {
2566+
val df = sql(s"SELECT 'dummy' AS name, ${"1" * 20}.${"2" * 18} AS value")
2567+
df.write.mode("Overwrite").parquet(dir.getAbsolutePath)
2568+
sql(
2569+
s"""
2570+
|CREATE EXTERNAL TABLE test_precision(name STRING, value DECIMAL(18,6))
2571+
|STORED AS PARQUET LOCATION '${dir.getAbsolutePath}'
2572+
|""".stripMargin)
2573+
checkAnswer(sql("SELECT * FROM test_precision"), Row("dummy", null))
2574+
}
2575+
}
2576+
}
2577+
}
25612578
}
25622579

25632580
@SlowHiveTest

0 commit comments

Comments
 (0)