Skip to content

[SPARK-18937][SQL] Timezone support in CSV/JSON parsing #16750

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
ueshin wants to merge 11 commits into from
Closed

[SPARK-18937][SQL] Timezone support in CSV/JSON parsing #16750

Changes from 1 commit
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
aa052f4
Add timeZone option to JSONOptions.
ueshin Jan 5, 2017
890879e
Apply timeZone option to JSON datasources.
ueshin Jan 6, 2017
f08b78c
Apply timeZone option to CSV datasources.
ueshin Jan 6, 2017
551cff9
Modify python files.
ueshin Jan 6, 2017
bdad5b7
Move new arguments to the end of the argument list.
ueshin Feb 1, 2017
d5ab37c
Modify JSONOptions/CSVOptions to take default timezone id argument.
ueshin Feb 1, 2017
9b30a9c
Merge branch 'master' into issues/SPARK-18937
ueshin Feb 9, 2017
dec84e0
Use `InternalRow.apply()` instead of `InternalRow.fromSeq()`.
ueshin Feb 9, 2017
ffc4912
Add a comment to explain the test result.
ueshin Feb 9, 2017
a455f4f
Merge branch 'master' into issues/SPARK-18937
ueshin Feb 13, 2017
ae6397d
Merge branch 'master' into issues/SPARK-18937
ueshin Feb 15, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Use InternalRow.apply() instead of InternalRow.fromSeq().
  • Loading branch information
@ueshin
ueshin committed Feb 9, 2017
commit dec84e05dc6a1dde778248f362891faafdd51c34
24 changes: 12 additions & 12 deletions ...alyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/JsonExpressionsSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -307,43 +307,43 @@ class JsonExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
test("json_tuple - hive key 4 - null json") {
checkJsonTuple(
JsonTuple(Literal(null) :: jsonTupleQuery),
InternalRow.fromSeq(Seq(null, null, null, null, null)))
InternalRow(null, null, null, null, null))
}

test("json_tuple - hive key 5 - null and empty fields") {
checkJsonTuple(
JsonTuple(Literal("""{"f1": "", "f5": null}""") :: jsonTupleQuery),
InternalRow.fromSeq(Seq(UTF8String.fromString(""), null, null, null, null)))
InternalRow(UTF8String.fromString(""), null, null, null, null))
}

test("json_tuple - hive key 6 - invalid json (array)") {
checkJsonTuple(
JsonTuple(Literal("[invalid JSON string]") :: jsonTupleQuery),
InternalRow.fromSeq(Seq(null, null, null, null, null)))
InternalRow(null, null, null, null, null))
}

test("json_tuple - invalid json (object start only)") {
checkJsonTuple(
JsonTuple(Literal("{") :: jsonTupleQuery),
InternalRow.fromSeq(Seq(null, null, null, null, null)))
InternalRow(null, null, null, null, null))
}

test("json_tuple - invalid json (no object end)") {
checkJsonTuple(
JsonTuple(Literal("""{"foo": "bar"""") :: jsonTupleQuery),
InternalRow.fromSeq(Seq(null, null, null, null, null)))
InternalRow(null, null, null, null, null))
}

test("json_tuple - invalid json (invalid json)") {
checkJsonTuple(
JsonTuple(Literal("\\") :: jsonTupleQuery),
InternalRow.fromSeq(Seq(null, null, null, null, null)))
InternalRow(null, null, null, null, null))
}

test("json_tuple - preserve newlines") {
checkJsonTuple(
JsonTuple(Literal("{\"a\":\"b\nc\"}") :: Literal("a") :: Nil),
InternalRow.fromSeq(Seq(UTF8String.fromString("b\nc"))))
InternalRow(UTF8String.fromString("b\nc")))
}

val gmtId = Option(DateTimeUtils.TimeZoneGMT.getID)
Expand All @@ -353,7 +353,7 @@ class JsonExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
val schema = StructType(StructField("a", IntegerType) :: Nil)
checkEvaluation(
JsonToStruct(schema, Map.empty, Literal(jsonData), gmtId),
InternalRow.fromSeq(1 :: Nil)
InternalRow(1)
)
}

Expand Down Expand Up @@ -389,11 +389,11 @@ class JsonExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
c.set(Calendar.MILLISECOND, 123)
checkEvaluation(
JsonToStruct(schema, Map.empty, Literal(jsonData1), gmtId),
InternalRow.fromSeq(c.getTimeInMillis * 1000L :: Nil)
InternalRow(c.getTimeInMillis * 1000L)
)
checkEvaluation(
JsonToStruct(schema, Map.empty, Literal(jsonData1), Option("PST")),
InternalRow.fromSeq(c.getTimeInMillis * 1000L :: Nil)
InternalRow(c.getTimeInMillis * 1000L)
)

val jsonData2 = """{"t": "2016-01-01T00:00:00"}"""
Expand All @@ -407,15 +407,15 @@ class JsonExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
Map("timestampFormat" -> "yyyy-MM-dd'T'HH:mm:ss"),
Literal(jsonData2),
Option(tz.getID)),
InternalRow.fromSeq(c.getTimeInMillis * 1000L :: Nil)
InternalRow(c.getTimeInMillis * 1000L)
)
checkEvaluation(
JsonToStruct(
schema,
Map("timestampFormat" -> "yyyy-MM-dd'T'HH:mm:ss", "timeZone" -> tz.getID),
Literal(jsonData2),
gmtId),
InternalRow.fromSeq(c.getTimeInMillis * 1000L :: Nil)
InternalRow(c.getTimeInMillis * 1000L)
)
}
}
Expand Down