Skip to content
Closed
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Next Next commit
[SPARK-47719] LEGACY_TIME_PARSER_POLICY_KEY default to CORRECTED
  • Loading branch information
srielau committed Apr 3, 2024
commit 9fdf8df25beef3bde460e5aa16fd16dc02c419b6
Original file line number Diff line number Diff line change
Expand Up @@ -4016,13 +4016,13 @@ object SQLConf {
.doc("When LEGACY, java.text.SimpleDateFormat is used for formatting and parsing " +
"dates/timestamps in a locale-sensitive manner, which is the approach before Spark 3.0. " +
"When set to CORRECTED, classes from java.time.* packages are used for the same purpose. " +
"The default value is EXCEPTION, RuntimeException is thrown when we will get different " +
"results.")
"When set to EXCEPTION, RuntimeException is thrown when we will get different " +
"results. The default is CORRECTED.")
.version("3.0.0")
.stringConf
.transform(_.toUpperCase(Locale.ROOT))
.checkValues(LegacyBehaviorPolicy.values.map(_.toString))
.createWithDefault(LegacyBehaviorPolicy.EXCEPTION.toString)
.createWithDefault(LegacyBehaviorPolicy.CORRECTED.toString)

val LEGACY_ARRAY_EXISTS_FOLLOWS_THREE_VALUED_LOGIC =
buildConf("spark.sql.legacy.followThreeValuedLogicInArrayExists")
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
--SET spark.sql.legacy.timeParserPolicy=EXCEPTION
--IMPORT date.sql
--IMPORT timestamp.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
--SET spark.sql.legacy.timeParserPolicy=EXCEPTION
--IMPORT datetime-formatting.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
--SET spark.sql.legacy.timeParserPolicy=EXCEPTION
--IMPORT datetime-parsing.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,207 @@
-- Automatically generated by SQLQueryTestSuite
-- !query
select to_timestamp('1', 'y')
-- !query schema
struct<to_timestamp(1, y):timestamp>
-- !query output
0001-01-01 00:00:00


-- !query
select to_timestamp('009999', 'y')
-- !query schema
struct<to_timestamp(009999, y):timestamp>
-- !query output
9999-01-01 00:00:00


-- !query
select to_timestamp('00', 'yy')
-- !query schema
struct<to_timestamp(00, yy):timestamp>
-- !query output
2000-01-01 00:00:00


-- !query
select to_timestamp('99', 'yy')
-- !query schema
struct<to_timestamp(99, yy):timestamp>
-- !query output
2099-01-01 00:00:00


-- !query
select to_timestamp('001', 'yyy')
-- !query schema
struct<to_timestamp(001, yyy):timestamp>
-- !query output
0001-01-01 00:00:00


-- !query
select to_timestamp('009999', 'yyy')
-- !query schema
struct<to_timestamp(009999, yyy):timestamp>
-- !query output
9999-01-01 00:00:00


-- !query
select to_timestamp('0001', 'yyyy')
-- !query schema
struct<to_timestamp(0001, yyyy):timestamp>
-- !query output
0001-01-01 00:00:00


-- !query
select to_timestamp('9999', 'yyyy')
-- !query schema
struct<to_timestamp(9999, yyyy):timestamp>
-- !query output
9999-01-01 00:00:00


-- !query
select to_timestamp('00001', 'yyyyy')
-- !query schema
struct<to_timestamp(00001, yyyyy):timestamp>
-- !query output
0001-01-01 00:00:00


-- !query
select to_timestamp('09999', 'yyyyy')
-- !query schema
struct<to_timestamp(09999, yyyyy):timestamp>
-- !query output
9999-01-01 00:00:00


-- !query
select to_timestamp('000001', 'yyyyyy')
-- !query schema
struct<to_timestamp(000001, yyyyyy):timestamp>
-- !query output
0001-01-01 00:00:00


-- !query
select to_timestamp('009999', 'yyyyyy')
-- !query schema
struct<to_timestamp(009999, yyyyyy):timestamp>
-- !query output
9999-01-01 00:00:00


-- !query
select to_timestamp('9', 'D')
-- !query schema
struct<to_timestamp(9, D):timestamp>
-- !query output
1970-01-09 00:00:00


-- !query
select to_timestamp('300', 'D')
-- !query schema
struct<to_timestamp(300, D):timestamp>
-- !query output
1970-10-27 00:00:00


-- !query
select to_timestamp('09', 'DD')
-- !query schema
struct<to_timestamp(09, DD):timestamp>
-- !query output
1970-01-09 00:00:00


-- !query
select to_timestamp('99', 'DD')
-- !query schema
struct<to_timestamp(99, DD):timestamp>
-- !query output
1970-04-09 00:00:00


-- !query
select to_timestamp('100', 'DD')
-- !query schema
struct<to_timestamp(100, DD):timestamp>
-- !query output
1970-04-10 00:00:00


-- !query
select to_timestamp('009', 'DDD')
-- !query schema
struct<to_timestamp(009, DDD):timestamp>
-- !query output
1970-01-09 00:00:00


-- !query
select to_timestamp('365', 'DDD')
-- !query schema
struct<to_timestamp(365, DDD):timestamp>
-- !query output
1970-12-31 00:00:00


-- !query
select to_timestamp('31-365', 'dd-DDD')
-- !query schema
struct<to_timestamp(31-365, dd-DDD):timestamp>
-- !query output
1970-12-31 00:00:00


-- !query
select to_timestamp('12-365', 'MM-DDD')
-- !query schema
struct<to_timestamp(12-365, MM-DDD):timestamp>
-- !query output
1970-12-31 00:00:00


-- !query
select to_timestamp('2020-365', 'yyyy-DDD')
-- !query schema
struct<to_timestamp(2020-365, yyyy-DDD):timestamp>
-- !query output
2020-12-30 00:00:00


-- !query
select to_timestamp('12-31-365', 'MM-dd-DDD')
-- !query schema
struct<to_timestamp(12-31-365, MM-dd-DDD):timestamp>
-- !query output
1970-12-31 00:00:00


-- !query
select to_timestamp('2020-30-365', 'yyyy-dd-DDD')
-- !query schema
struct<to_timestamp(2020-30-365, yyyy-dd-DDD):timestamp>
-- !query output
2020-12-30 00:00:00


-- !query
select to_timestamp('2020-12-350', 'yyyy-MM-DDD')
-- !query schema
struct<to_timestamp(2020-12-350, yyyy-MM-DDD):timestamp>
-- !query output
2020-12-15 00:00:00


-- !query
select to_timestamp('2020-12-31-366', 'yyyy-MM-dd-DDD')
-- !query schema
struct<to_timestamp(2020-12-31-366, yyyy-MM-dd-DDD):timestamp>
-- !query output
2020-12-31 00:00:00