diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonGenerator.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonGenerator.scala index eb06e4f304f0..56fe9a01f0c6 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonGenerator.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonGenerator.scala @@ -171,8 +171,10 @@ private[sql] class JacksonGenerator( var i = 0 while (i < row.numFields) { val field = schema(i) - if (!row.isNullAt(i)) { - gen.writeFieldName(field.name) + gen.writeFieldName(field.name) + if (row.isNullAt(i)) { + gen.writeNull() + } else { fieldWriters(i).apply(row, i) } i += 1 diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/JsonExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/JsonExpressionsSuite.scala index 7812319756ea..d3ae11f42e7f 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/JsonExpressionsSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/JsonExpressionsSuite.scala @@ -556,7 +556,7 @@ class JsonExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper with test("to_json - array with single empty row") { val inputSchema = ArrayType(StructType(StructField("a", IntegerType) :: Nil)) val input = new GenericArrayData(InternalRow(null) :: Nil) - val output = """[{}]""" + val output = """[{"a":null}]""" checkEvaluation( StructsToJson(Map.empty, Literal.create(input, inputSchema), gmtId), output) diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/json/JacksonGeneratorSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/json/JacksonGeneratorSuite.scala index 9b27490ed0e3..d2293a4c1b6c 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/json/JacksonGeneratorSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/json/JacksonGeneratorSuite.scala @@ -56,7 +56,7 @@ class JacksonGeneratorSuite extends SparkFunSuite { val gen = new JacksonGenerator(dataType, writer, option) gen.write(input) gen.flush() - assert(writer.toString === """[{}]""") + assert(writer.toString === """[{"a":null}]""") } test("initial with StructType and write out an empty array") { diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala index 8c8d41ebf115..bb12197fa481 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala @@ -1229,7 +1229,7 @@ class JsonSuite extends QueryTest with SharedSQLContext with TestJsonData { val df2 = df1.toDF val result = df2.toJSON.collect() // scalastyle:off - assert(result(0) === "{\"f1\":1,\"f2\":\"A1\",\"f3\":true,\"f4\":[\"1\",\" A1\",\" true\",\" null\"]}") + assert(result(0) === "{\"f1\":1,\"f2\":\"A1\",\"f3\":true,\"f4\":[\"1\",\" A1\",\" true\",\" null\"],\"f5\":null}") assert(result(3) === "{\"f1\":4,\"f2\":\"D4\",\"f3\":true,\"f4\":[\"4\",\" D4\",\" true\",\" 2147483644\"],\"f5\":2147483644}") // scalastyle:on @@ -1265,6 +1265,7 @@ class JsonSuite extends QueryTest with SharedSQLContext with TestJsonData { 1.7976931348623157E308, 10, 21474836470L, + null, "this is a simple string.") ) @@ -2063,4 +2064,11 @@ class JsonSuite extends QueryTest with SharedSQLContext with TestJsonData { ) } } + + test("SPARK-23773: JacksonGenerator does not include keys " + + "that have null value for StructTypes") { + val df = sql("select NAMED_STRUCT('f1', 10, 'f2', null, 'f3', 15) as my_struct") + val result = df.toJSON.collect()(0) + assert(result === "{\"my_struct\":{\"f1\":10,\"f2\":null,\"f3\":15}}") + } }