Skip to content

Commit cb126a9

Browse files
MaxGekkMaxGekk
committed
Make legacy date formatters independent from default JVM time zone
1 parent b806fc4 commit cb126a9

File tree

3 files changed

+90
-63
lines changed

3 files changed

+90
-63
lines changed

‎sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateFormatter.scala‎

Lines changed: 20 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -19,8 +19,7 @@ package org.apache.spark.sql.catalyst.util
1919

2020
import java.text.SimpleDateFormat
2121
import java.time.{LocalDate, ZoneId}
22-
import java.time.format.DateTimeFormatter
23-
import java.util.{Date, Locale}
22+
import java.util.{Date, Locale, TimeZone}
2423

2524
import org.apache.commons.lang3.time.FastDateFormat
2625

@@ -83,31 +82,42 @@ class Iso8601DateFormatter(
8382

8483
trait LegacyDateFormatter extends DateFormatter {
8584
def parseToDate(s: String): Date
85+
def zoneId: ZoneId
8686

8787
override def parse(s: String): Int = {
88-
fromJavaDate(new java.sql.Date(parseToDate(s).getTime))
88+
fromJavaDate(new java.sql.Date(parseToDate(s).getTime), TimeZone.getTimeZone(zoneId))
8989
}
9090

9191
override def format(days: Int): String = {
92-
format(DateTimeUtils.toJavaDate(days))
92+
format(DateTimeUtils.toJavaDate(days, TimeZone.getTimeZone(zoneId)))
9393
}
9494

9595
override def format(localDate: LocalDate): String = {
9696
format(localDateToDays(localDate))
9797
}
9898
}
9999

100-
class LegacyFastDateFormatter(pattern: String, locale: Locale) extends LegacyDateFormatter {
100+
class LegacyFastDateFormatter(
101+
pattern: String,
102+
override val zoneId: ZoneId,
103+
locale: Locale) extends LegacyDateFormatter {
101104
@transient
102-
private lazy val fdf = FastDateFormat.getInstance(pattern, locale)
105+
private lazy val fdf = FastDateFormat.getInstance(pattern, TimeZone.getTimeZone(zoneId), locale)
103106
override def parseToDate(s: String): Date = fdf.parse(s)
104107
override def format(d: Date): String = fdf.format(d)
105108
override def validatePatternString(): Unit = fdf
106109
}
107110

108-
class LegacySimpleDateFormatter(pattern: String, locale: Locale) extends LegacyDateFormatter {
111+
class LegacySimpleDateFormatter(
112+
pattern: String,
113+
override val zoneId: ZoneId,
114+
locale: Locale) extends LegacyDateFormatter {
109115
@transient
110-
private lazy val sdf = new SimpleDateFormat(pattern, locale)
116+
private lazy val sdf = {
117+
val formatter = new SimpleDateFormat(pattern, locale)
118+
formatter.setTimeZone(TimeZone.getTimeZone(zoneId))
119+
formatter
120+
}
111121
override def parseToDate(s: String): Date = sdf.parse(s)
112122
override def format(d: Date): String = sdf.format(d)
113123
override def validatePatternString(): Unit = sdf
@@ -143,9 +153,9 @@ object DateFormatter {
143153
legacyFormat: LegacyDateFormat): DateFormatter = {
144154
legacyFormat match {
145155
case FAST_DATE_FORMAT =>
146-
new LegacyFastDateFormatter(pattern, locale)
156+
new LegacyFastDateFormatter(pattern, zoneId, locale)
147157
case SIMPLE_DATE_FORMAT | LENIENT_SIMPLE_DATE_FORMAT =>
148-
new LegacySimpleDateFormatter(pattern, locale)
158+
new LegacySimpleDateFormatter(pattern, zoneId, locale)
149159
}
150160
}
151161

‎sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -103,8 +103,12 @@ object DateTimeUtils {
103103
* @return The number of days since epoch from java.sql.Date.
104104
*/
105105
def fromJavaDate(date: Date): SQLDate = {
106+
fromJavaDate(date, TimeZone.getDefault)
107+
}
108+
109+
def fromJavaDate(date: Date, timeZone: TimeZone): SQLDate = {
106110
val millisUtc = date.getTime
107-
val millisLocal = millisUtc + TimeZone.getDefault.getOffset(millisUtc)
111+
val millisLocal = millisUtc + timeZone.getOffset(millisUtc)
108112
val julianDays = Math.toIntExact(Math.floorDiv(millisLocal, MILLIS_PER_DAY))
109113
rebaseJulianToGregorianDays(julianDays)
110114
}
@@ -123,9 +127,13 @@ object DateTimeUtils {
123127
* @return A `java.sql.Date` from number of days since epoch.
124128
*/
125129
def toJavaDate(daysSinceEpoch: SQLDate): Date = {
130+
toJavaDate(daysSinceEpoch, TimeZone.getDefault)
131+
}
132+
133+
def toJavaDate(daysSinceEpoch: SQLDate, timeZone: TimeZone): Date = {
126134
val rebasedDays = rebaseGregorianToJulianDays(daysSinceEpoch)
127135
val localMillis = Math.multiplyExact(rebasedDays, MILLIS_PER_DAY)
128-
val timeZoneOffset = TimeZone.getDefault match {
136+
val timeZoneOffset = timeZone match {
129137
case zoneInfo: ZoneInfo => zoneInfo.getOffsetsByWall(localMillis, null)
130138
case timeZone: TimeZone => timeZone.getOffset(localMillis - timeZone.getRawOffset)
131139
}

‎sql/catalyst/src/test/scala/org/apache/spark/sql/util/DateFormatterSuite.scala‎

Lines changed: 60 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,8 @@
1717

1818
package org.apache.spark.sql.util
1919

20-
import java.time.{DateTimeException, LocalDate}
20+
import java.time.{DateTimeException, LocalDate, ZoneId}
21+
import java.util.{Calendar, TimeZone}
2122

2223
import org.apache.spark.{SparkFunSuite, SparkUpgradeException}
2324
import org.apache.spark.sql.catalyst.plans.SQLHelper
@@ -28,26 +29,35 @@ import org.apache.spark.sql.internal.SQLConf
2829
import org.apache.spark.sql.internal.SQLConf.LegacyBehaviorPolicy
2930

3031
class DateFormatterSuite extends SparkFunSuite with SQLHelper {
31-
test("parsing dates") {
32-
outstandingTimezonesIds.foreach { timeZone =>
33-
withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> timeZone) {
34-
val formatter = DateFormatter(getZoneId(timeZone))
35-
val daysSinceEpoch = formatter.parse("2018-12-02")
36-
assert(daysSinceEpoch === 17867)
32+
private def withOutstandingZoneIds(f: ZoneId => Unit): Unit = {
33+
for {
34+
jvmZoneId <- outstandingZoneIds
35+
sessionZoneId <- outstandingZoneIds
36+
} {
37+
withDefaultTimeZone(jvmZoneId) {
38+
withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> sessionZoneId.getId) {
39+
f(sessionZoneId)
40+
}
3741
}
3842
}
3943
}
4044

45+
test("parsing dates") {
46+
withOutstandingZoneIds { zoneId =>
47+
val formatter = DateFormatter(zoneId)
48+
val daysSinceEpoch = formatter.parse("2018-12-02")
49+
assert(daysSinceEpoch === 17867)
50+
}
51+
}
52+
4153
test("format dates") {
42-
outstandingTimezonesIds.foreach { timeZone =>
43-
withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> timeZone) {
44-
val formatter = DateFormatter(getZoneId(timeZone))
45-
val (days, expected) = (17867, "2018-12-02")
46-
val date = formatter.format(days)
47-
assert(date === expected)
48-
assert(formatter.format(daysToLocalDate(days)) === expected)
49-
assert(formatter.format(toJavaDate(days)) === expected)
50-
}
54+
withOutstandingZoneIds { zoneId =>
55+
val formatter = DateFormatter(zoneId)
56+
val (days, expected) = (17867, "2018-12-02")
57+
val date = formatter.format(days)
58+
assert(date === expected)
59+
assert(formatter.format(daysToLocalDate(days)) === expected)
60+
assert(formatter.format(toJavaDate(days, TimeZone.getTimeZone(zoneId))) === expected)
5161
}
5262
}
5363

@@ -66,18 +76,16 @@ class DateFormatterSuite extends SparkFunSuite with SQLHelper {
6676
"2018-12-12",
6777
"2038-01-01",
6878
"5010-11-17").foreach { date =>
69-
outstandingTimezonesIds.foreach { timeZone =>
70-
withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> timeZone) {
71-
val formatter = DateFormatter(
72-
DateFormatter.defaultPattern,
73-
getZoneId(timeZone),
74-
DateFormatter.defaultLocale,
75-
legacyFormat)
76-
val days = formatter.parse(date)
77-
assert(date === formatter.format(days))
78-
assert(date === formatter.format(daysToLocalDate(days)))
79-
assert(date === formatter.format(toJavaDate(days)))
80-
}
79+
withOutstandingZoneIds { zoneId =>
80+
val formatter = DateFormatter(
81+
DateFormatter.defaultPattern,
82+
zoneId,
83+
DateFormatter.defaultLocale,
84+
legacyFormat)
85+
val days = formatter.parse(date)
86+
assert(date === formatter.format(days))
87+
assert(date === formatter.format(daysToLocalDate(days)))
88+
assert(date === formatter.format(toJavaDate(days, TimeZone.getTimeZone(zoneId))))
8189
}
8290
}
8391
}
@@ -100,17 +108,15 @@ class DateFormatterSuite extends SparkFunSuite with SQLHelper {
100108
17877,
101109
24837,
102110
1110657).foreach { days =>
103-
outstandingTimezonesIds.foreach { timeZone =>
104-
withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> timeZone) {
105-
val formatter = DateFormatter(
106-
DateFormatter.defaultPattern,
107-
getZoneId(timeZone),
108-
DateFormatter.defaultLocale,
109-
legacyFormat)
110-
val date = formatter.format(days)
111-
val parsed = formatter.parse(date)
112-
assert(days === parsed)
113-
}
111+
withOutstandingZoneIds { zoneId =>
112+
val formatter = DateFormatter(
113+
DateFormatter.defaultPattern,
114+
zoneId,
115+
DateFormatter.defaultLocale,
116+
legacyFormat)
117+
val date = formatter.format(days)
118+
val parsed = formatter.parse(date)
119+
assert(days === parsed)
114120
}
115121
}
116122
}
@@ -167,18 +173,21 @@ class DateFormatterSuite extends SparkFunSuite with SQLHelper {
167173
test("SPARK-31557: rebasing in legacy formatters/parsers") {
168174
withSQLConf(SQLConf.LEGACY_TIME_PARSER_POLICY.key -> LegacyBehaviorPolicy.LEGACY.toString) {
169175
LegacyDateFormats.values.foreach { legacyFormat =>
170-
outstandingTimezonesIds.foreach { timeZone =>
171-
withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> timeZone) {
172-
val formatter = DateFormatter(
173-
DateFormatter.defaultPattern,
174-
getZoneId(timeZone),
175-
DateFormatter.defaultLocale,
176-
legacyFormat)
177-
assert(LocalDate.ofEpochDay(formatter.parse("1000-01-01")) === LocalDate.of(1000, 1, 1))
178-
assert(formatter.format(LocalDate.of(1000, 1, 1)) === "1000-01-01")
179-
assert(formatter.format(localDateToDays(LocalDate.of(1000, 1, 1))) === "1000-01-01")
180-
assert(formatter.format(java.sql.Date.valueOf("1000-01-01")) === "1000-01-01")
181-
}
176+
withOutstandingZoneIds { zoneId =>
177+
val formatter = DateFormatter(
178+
DateFormatter.defaultPattern,
179+
zoneId,
180+
DateFormatter.defaultLocale,
181+
legacyFormat)
182+
assert(LocalDate.ofEpochDay(formatter.parse("1000-01-01")) === LocalDate.of(1000, 1, 1))
183+
assert(formatter.format(LocalDate.of(1000, 1, 1)) === "1000-01-01")
184+
assert(formatter.format(localDateToDays(LocalDate.of(1000, 1, 1))) === "1000-01-01")
185+
val cal = new Calendar.Builder()
186+
.setCalendarType("gregory")
187+
.setTimeZone(TimeZone.getTimeZone(zoneId))
188+
.setDate(1000, 0, 1)
189+
.build()
190+
assert(formatter.format(cal.getTime) === "1000-01-01")
182191
}
183192
}
184193
}

0 commit comments

Comments
 (0)