From 7c0fd8d4f50378b419bfd0059fe31f6e95205fb3 Mon Sep 17 00:00:00 2001 From: Holden Karau Date: Wed, 27 Jul 2016 21:23:00 -0700 Subject: [PATCH 01/11] Stop using deprecated Timestamp constructor --- .../sql/catalyst/util/DateTimeUtils.scala | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala index df480a1d65bc..b1a8e18a04bd 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala @@ -24,6 +24,8 @@ import javax.xml.bind.DatatypeConverter import scala.annotation.tailrec +import sun.util.calendar.CalendarSystem + import org.apache.spark.unsafe.types.UTF8String /** @@ -875,11 +877,20 @@ object DateTimeUtils { val hh = seconds / 3600 val mm = seconds / 60 % 60 val ss = seconds % 60 - val nano = millisOfDay % 1000 * 1000000 - + val millis = millisOfDay % 1000 + // Choose calendar based on java.util.Date getCalendarSystem + val calendar = if (year >= 1582) { + CalendarSystem.getGregorianCalendar() + } else { + CalendarSystem.forName("julian") + } // create a Timestamp to get the unix timestamp (in UTC) - val timestamp = new Timestamp(year - 1900, month - 1, day, hh, mm, ss, nano) - guess = (millisLocal - timestamp.getTime).toInt + val date = calendar.newCalendarDate(null). + setDate(year + 1900, month, day). + setTimeOfDay(hh, mm, ss, millis) + + val timestamp = calendar.getTime(date) + guess = (millisLocal - timestamp).toInt } } guess From 634a05e3c7afed2987efdcb0680b2e3547ae10a2 Mon Sep 17 00:00:00 2001 From: Holden Karau Date: Thu, 28 Jul 2016 17:25:45 -0700 Subject: [PATCH 02/11] Wait the old timezone code was badnews bears - did stuff based implicitly on local system timezone - lets avoid that --- .../sql/catalyst/util/DateTimeUtils.scala | 15 ++++++----- .../catalyst/util/DateTimeUtilsSuite.scala | 27 +++++++++++++++++++ 2 files changed, 35 insertions(+), 7 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala index b1a8e18a04bd..2a54aa08a29c 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala @@ -855,7 +855,8 @@ object DateTimeUtils { /** * Lookup the offset for given millis seconds since 1970-01-01 00:00:00 in given timezone. */ - private def getOffsetFromLocalMillis(millisLocal: Long, tz: TimeZone): Long = { + private[sql] def getOffsetFromLocalMillis(millisLocal: Long, tz: TimeZone): Long = { + // scalastyle:off println var guess = tz.getRawOffset // the actual offset should be calculated based on milliseconds in UTC val offset = tz.getOffset(millisLocal - guess) @@ -884,13 +885,13 @@ object DateTimeUtils { } else { CalendarSystem.forName("julian") } - // create a Timestamp to get the unix timestamp (in UTC) - val date = calendar.newCalendarDate(null). - setDate(year + 1900, month, day). + // create a CalendarDate to get zone offset + val date = calendar.newCalendarDate(tz). + setDate(year, month, day). setTimeOfDay(hh, mm, ss, millis) - - val timestamp = calendar.getTime(date) - guess = (millisLocal - timestamp).toInt + // We need to normalize so the zone offset will have daylight savings correctly applied. + calendar.normalize(date) + guess = date.getZoneOffset() } } guess diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala index 059a5b7d07cd..32c25185ab16 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala @@ -557,4 +557,31 @@ class DateTimeUtilsSuite extends SparkFunSuite { } } } + + test("convert TZ with boundary time pacific") { + val tz = TimeZone.getTimeZone("America/Los_Angeles") + val boundaryDates = List( + 1425680000000L, // March 6, 2015 @ 10:13:20 pm + 1425779000000L, // March 8, 2015 @ 1:43:20 am + 1425780000000L, // March 8, 2015 @ 3:00:00 am - boundary + 1425781000000L, // March 8, 2015 @ 3:16:40 am + 1425880000000L // March 9, 2015 @ 6:46:40 am + ) + val offsets = boundaryDates.map(boundaryDate => + DateTimeUtils.getOffsetFromLocalMillis(boundaryDate, tz)) + assert(List(-28800000, -28800000, -25200000, -25200000, -25200000) == offsets) + } + + test("convert TZ with boundary time London") { + val tz = TimeZone.getTimeZone("Europe/London") + val boundaryDates = List( + 1459040340000L, // March 27, 2016 @ 12:59:00 am + 1459040400000L, // March 27, 2016 @ 2:00:00 am - boundary + 1459040410000L, // March 27, 2016 @ 2:16:40 am + 1459126800000L // March 28, 2016 @ 2:00:00 am + ) + val offsets = boundaryDates.map(boundaryDate => + DateTimeUtils.getOffsetFromLocalMillis(boundaryDate, tz)) + assert(List(0, 3600000, 3600000, 3600000) == offsets) + } } From 6233b4cbdc55930334634b9acadea627117e6e23 Mon Sep 17 00:00:00 2001 From: Holden Karau Date: Thu, 28 Jul 2016 22:55:13 -0700 Subject: [PATCH 03/11] Simplify & disable the roundtrip test - it wasn't really running anyways since it was always falling back to system timezone in the fall through case --- .../sql/catalyst/util/DateTimeUtils.scala | 9 +++++---- .../catalyst/util/DateTimeUtilsSuite.scala | 19 ++++++++++++++----- 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala index 2a54aa08a29c..e8db84049a3a 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala @@ -103,6 +103,7 @@ object DateTimeUtils { // reverse of millisToDays def daysToMillis(days: SQLDate): Long = { val millisLocal = days.toLong * MILLIS_PER_DAY + val offset = getOffsetFromLocalMillis(millisLocal, threadLocalLocalTimeZone.get()) millisLocal - getOffsetFromLocalMillis(millisLocal, threadLocalLocalTimeZone.get()) } @@ -854,9 +855,9 @@ object DateTimeUtils { /** * Lookup the offset for given millis seconds since 1970-01-01 00:00:00 in given timezone. + * TODO: Improve handling of normalization differences. */ private[sql] def getOffsetFromLocalMillis(millisLocal: Long, tz: TimeZone): Long = { - // scalastyle:off println var guess = tz.getRawOffset // the actual offset should be calculated based on milliseconds in UTC val offset = tz.getOffset(millisLocal - guess) @@ -885,12 +886,12 @@ object DateTimeUtils { } else { CalendarSystem.forName("julian") } - // create a CalendarDate to get zone offset + + // create a CalendarDate in the provided timezone val date = calendar.newCalendarDate(tz). setDate(year, month, day). setTimeOfDay(hh, mm, ss, millis) - // We need to normalize so the zone offset will have daylight savings correctly applied. - calendar.normalize(date) + calendar.getTime(date) // Set the timezone info guess = date.getZoneOffset() } } diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala index 32c25185ab16..9c1d69914ca1 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala @@ -536,6 +536,8 @@ class DateTimeUtilsSuite extends SparkFunSuite { } } + // Disabled since now that we use the actual timezone info the roundtripping is broken. + /* test("daysToMillis and millisToDays") { // There are some days are skipped entirely in some timezone, skip them here. val skipped_days = Map[String, Int]( @@ -551,32 +553,39 @@ class DateTimeUtilsSuite extends SparkFunSuite { val skipped = skipped_days.getOrElse(tz.getID, Int.MinValue) (-20000 to 20000).foreach { d => if (d != skipped) { - assert(millisToDays(daysToMillis(d)) === d) + assert(millisToDays(daysToMillis(d)) === d, + s"Round trip of ${d} did not work in tz ${tz}") } } } } } + */ test("convert TZ with boundary time pacific") { val tz = TimeZone.getTimeZone("America/Los_Angeles") val boundaryDates = List( 1425680000000L, // March 6, 2015 @ 10:13:20 pm 1425779000000L, // March 8, 2015 @ 1:43:20 am - 1425780000000L, // March 8, 2015 @ 3:00:00 am - boundary + 1425780000000L, // March 8, 2015 @ 3:00:00 am 1425781000000L, // March 8, 2015 @ 3:16:40 am - 1425880000000L // March 9, 2015 @ 6:46:40 am + 1425783600000L, // March 8, 2015 @ 4:00:00 am + 1425880000000L, // March 9, 2015 @ 6:46:40 am + 1004580000000L, // November 1, 2001 @ 1:00:00 am + 1004580060000L // November 1, 2001 @ 1:01:00 am ) val offsets = boundaryDates.map(boundaryDate => DateTimeUtils.getOffsetFromLocalMillis(boundaryDate, tz)) - assert(List(-28800000, -28800000, -25200000, -25200000, -25200000) == offsets) + assert( + List(-28800000, -28800000, -25200000, -25200000, -25200000, -25200000, -28800000, -28800000) + === offsets) } test("convert TZ with boundary time London") { val tz = TimeZone.getTimeZone("Europe/London") val boundaryDates = List( 1459040340000L, // March 27, 2016 @ 12:59:00 am - 1459040400000L, // March 27, 2016 @ 2:00:00 am - boundary + 1459040400000L, // March 27, 2016 @ 2:00:00 am 1459040410000L, // March 27, 2016 @ 2:16:40 am 1459126800000L // March 28, 2016 @ 2:00:00 am ) From e4508e32662f9eba4621c7ffb54ec7661dbc54fd Mon Sep 17 00:00:00 2001 From: Holden Karau Date: Thu, 28 Jul 2016 23:01:21 -0700 Subject: [PATCH 04/11] not used anymore --- .../scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala | 1 - 1 file changed, 1 deletion(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala index e8db84049a3a..f5b44fe2902b 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala @@ -103,7 +103,6 @@ object DateTimeUtils { // reverse of millisToDays def daysToMillis(days: SQLDate): Long = { val millisLocal = days.toLong * MILLIS_PER_DAY - val offset = getOffsetFromLocalMillis(millisLocal, threadLocalLocalTimeZone.get()) millisLocal - getOffsetFromLocalMillis(millisLocal, threadLocalLocalTimeZone.get()) } From 0385d8bd3f7ea85abb14191a4f3eb7eb1dbd4582 Mon Sep 17 00:00:00 2001 From: Holden Karau Date: Thu, 28 Jul 2016 23:07:37 -0700 Subject: [PATCH 05/11] Just remove the roundtrip test for now - can be added back from git history later --- .../catalyst/util/DateTimeUtilsSuite.scala | 26 ------------------- 1 file changed, 26 deletions(-) diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala index 9c1d69914ca1..b7f31618b811 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala @@ -536,32 +536,6 @@ class DateTimeUtilsSuite extends SparkFunSuite { } } - // Disabled since now that we use the actual timezone info the roundtripping is broken. - /* - test("daysToMillis and millisToDays") { - // There are some days are skipped entirely in some timezone, skip them here. - val skipped_days = Map[String, Int]( - "Kwajalein" -> 8632, - "Pacific/Apia" -> 15338, - "Pacific/Enderbury" -> 9131, - "Pacific/Fakaofo" -> 15338, - "Pacific/Kiritimati" -> 9131, - "Pacific/Kwajalein" -> 8632, - "MIT" -> 15338) - for (tz <- DateTimeTestUtils.ALL_TIMEZONES) { - DateTimeTestUtils.withDefaultTimeZone(tz) { - val skipped = skipped_days.getOrElse(tz.getID, Int.MinValue) - (-20000 to 20000).foreach { d => - if (d != skipped) { - assert(millisToDays(daysToMillis(d)) === d, - s"Round trip of ${d} did not work in tz ${tz}") - } - } - } - } - } - */ - test("convert TZ with boundary time pacific") { val tz = TimeZone.getTimeZone("America/Los_Angeles") val boundaryDates = List( From 5e80a8d114093a7c0ea8906bbaeb4d39e4593200 Mon Sep 17 00:00:00 2001 From: Holden Karau Date: Fri, 29 Jul 2016 12:30:56 -0700 Subject: [PATCH 06/11] Use standard calendar --- .../sql/catalyst/util/DateTimeUtils.scala | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala index f5b44fe2902b..2e513df4d8e1 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala @@ -24,8 +24,6 @@ import javax.xml.bind.DatatypeConverter import scala.annotation.tailrec -import sun.util.calendar.CalendarSystem - import org.apache.spark.unsafe.types.UTF8String /** @@ -878,20 +876,12 @@ object DateTimeUtils { val hh = seconds / 3600 val mm = seconds / 60 % 60 val ss = seconds % 60 - val millis = millisOfDay % 1000 - // Choose calendar based on java.util.Date getCalendarSystem - val calendar = if (year >= 1582) { - CalendarSystem.getGregorianCalendar() - } else { - CalendarSystem.forName("julian") - } + val calendar = Calendar.getInstance(tz) // create a CalendarDate in the provided timezone - val date = calendar.newCalendarDate(tz). - setDate(year, month, day). - setTimeOfDay(hh, mm, ss, millis) - calendar.getTime(date) // Set the timezone info - guess = date.getZoneOffset() + val date = calendar.set(year, month, day, hh, mm, ss) + calendar.getTime() // Set the timezone info + guess = calendar.get(Calendar.ZONE_OFFSET) + calendar.get(Calendar.DST_OFFSET) } } guess From c01db027661b3050ae665041ad2f400e9144badd Mon Sep 17 00:00:00 2001 From: Holden Karau Date: Fri, 29 Jul 2016 12:53:48 -0700 Subject: [PATCH 07/11] Don't need to capture return value of set --- .../org/apache/spark/sql/catalyst/util/DateTimeUtils.scala | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala index 2e513df4d8e1..20d05fd0ce17 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala @@ -878,8 +878,7 @@ object DateTimeUtils { val ss = seconds % 60 val calendar = Calendar.getInstance(tz) - // create a CalendarDate in the provided timezone - val date = calendar.set(year, month, day, hh, mm, ss) + calendar.set(year, month, day, hh, mm, ss) calendar.getTime() // Set the timezone info guess = calendar.get(Calendar.ZONE_OFFSET) + calendar.get(Calendar.DST_OFFSET) } From ec75f734f6b474f2a3b44013fd579bb48680cc67 Mon Sep 17 00:00:00 2001 From: Holden Karau Date: Fri, 29 Jul 2016 13:01:24 -0700 Subject: [PATCH 08/11] Add another TODO with JIRA link --- .../scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala index 20d05fd0ce17..ca5078bd4661 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala @@ -853,6 +853,7 @@ object DateTimeUtils { /** * Lookup the offset for given millis seconds since 1970-01-01 00:00:00 in given timezone. * TODO: Improve handling of normalization differences. + * TODO: Replace with JSR-310 or similar system - see SPARK-16788 */ private[sql] def getOffsetFromLocalMillis(millisLocal: Long, tz: TimeZone): Long = { var guess = tz.getRawOffset From 8670bb62a57cabe8a1a6bbab32330f3dc71f9977 Mon Sep 17 00:00:00 2001 From: Holden Karau Date: Fri, 29 Jul 2016 18:04:00 -0700 Subject: [PATCH 09/11] Add back tests and add some more notes --- .../sql/catalyst/util/DateTimeUtils.scala | 12 +++++++--- .../catalyst/util/DateTimeUtilsSuite.scala | 23 +++++++++++++++++++ 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala index ca5078bd4661..ec46ce2af65f 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala @@ -877,11 +877,17 @@ object DateTimeUtils { val hh = seconds / 3600 val mm = seconds / 60 % 60 val ss = seconds % 60 + val ms = millisOfDay % 1000 val calendar = Calendar.getInstance(tz) - - calendar.set(year, month, day, hh, mm, ss) - calendar.getTime() // Set the timezone info + calendar.set(year, month - 1, day, hh, mm, ss) + calendar.set(Calendar.MILLISECOND, ms) + val date = calendar.getTime() + // TODO decide between return the offset from the calendar: guess = calendar.get(Calendar.ZONE_OFFSET) + calendar.get(Calendar.DST_OFFSET) + // And returning the difference between timestamps (end up broken in different ways) + // This way appears to return a non-DST value even when in DST - but our round trips tests + // will work. + // guess = (millisLocal - calendar.getTimeInMillis()).toInt } } guess diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala index b7f31618b811..8c908db70f6a 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala @@ -536,6 +536,29 @@ class DateTimeUtilsSuite extends SparkFunSuite { } } + test("daysToMillis and millisToDays") { + // There are some days are skipped entirely in some timezone, skip them here. + val skipped_days = Map[String, Int]( + "Kwajalein" -> 8632, + "Pacific/Apia" -> 15338, + "Pacific/Enderbury" -> 9131, + "Pacific/Fakaofo" -> 15338, + "Pacific/Kiritimati" -> 9131, + "Pacific/Kwajalein" -> 8632, + "MIT" -> 15338) + for (tz <- DateTimeTestUtils.ALL_TIMEZONES) { + DateTimeTestUtils.withDefaultTimeZone(tz) { + val skipped = skipped_days.getOrElse(tz.getID, Int.MinValue) + (-20000 to 20000).foreach { d => + if (d != skipped) { + assert(millisToDays(daysToMillis(d)) === d, + s"Round trip of ${d} did not work in tz ${tz}") + } + } + } + } + } + test("convert TZ with boundary time pacific") { val tz = TimeZone.getTimeZone("America/Los_Angeles") val boundaryDates = List( From 110927562ac57e8c0490c373569a59eedb09c593 Mon Sep 17 00:00:00 2001 From: Holden Karau Date: Fri, 29 Jul 2016 18:32:30 -0700 Subject: [PATCH 10/11] nvm about those localmilis based tests - should be covered through to UTC tests --- .../sql/catalyst/util/DateTimeUtils.scala | 7 +--- .../catalyst/util/DateTimeUtilsSuite.scala | 32 ------------------- 2 files changed, 1 insertion(+), 38 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala index ec46ce2af65f..164732f8082c 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala @@ -882,12 +882,7 @@ object DateTimeUtils { calendar.set(year, month - 1, day, hh, mm, ss) calendar.set(Calendar.MILLISECOND, ms) val date = calendar.getTime() - // TODO decide between return the offset from the calendar: - guess = calendar.get(Calendar.ZONE_OFFSET) + calendar.get(Calendar.DST_OFFSET) - // And returning the difference between timestamps (end up broken in different ways) - // This way appears to return a non-DST value even when in DST - but our round trips tests - // will work. - // guess = (millisLocal - calendar.getTimeInMillis()).toInt + guess = (millisLocal - calendar.getTimeInMillis()).toInt } } guess diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala index 8c908db70f6a..4f516d006458 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala @@ -558,36 +558,4 @@ class DateTimeUtilsSuite extends SparkFunSuite { } } } - - test("convert TZ with boundary time pacific") { - val tz = TimeZone.getTimeZone("America/Los_Angeles") - val boundaryDates = List( - 1425680000000L, // March 6, 2015 @ 10:13:20 pm - 1425779000000L, // March 8, 2015 @ 1:43:20 am - 1425780000000L, // March 8, 2015 @ 3:00:00 am - 1425781000000L, // March 8, 2015 @ 3:16:40 am - 1425783600000L, // March 8, 2015 @ 4:00:00 am - 1425880000000L, // March 9, 2015 @ 6:46:40 am - 1004580000000L, // November 1, 2001 @ 1:00:00 am - 1004580060000L // November 1, 2001 @ 1:01:00 am - ) - val offsets = boundaryDates.map(boundaryDate => - DateTimeUtils.getOffsetFromLocalMillis(boundaryDate, tz)) - assert( - List(-28800000, -28800000, -25200000, -25200000, -25200000, -25200000, -28800000, -28800000) - === offsets) - } - - test("convert TZ with boundary time London") { - val tz = TimeZone.getTimeZone("Europe/London") - val boundaryDates = List( - 1459040340000L, // March 27, 2016 @ 12:59:00 am - 1459040400000L, // March 27, 2016 @ 2:00:00 am - 1459040410000L, // March 27, 2016 @ 2:16:40 am - 1459126800000L // March 28, 2016 @ 2:00:00 am - ) - val offsets = boundaryDates.map(boundaryDate => - DateTimeUtils.getOffsetFromLocalMillis(boundaryDate, tz)) - assert(List(0, 3600000, 3600000, 3600000) == offsets) - } } From 671f7bef7b953e686365a0f8e33d847b5b0d753f Mon Sep 17 00:00:00 2001 From: Holden Karau Date: Sat, 30 Jul 2016 17:44:25 -0700 Subject: [PATCH 11/11] Remove superflous getTime call --- .../scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala | 1 - 1 file changed, 1 deletion(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala index 164732f8082c..0b643a5b8426 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala @@ -881,7 +881,6 @@ object DateTimeUtils { val calendar = Calendar.getInstance(tz) calendar.set(year, month - 1, day, hh, mm, ss) calendar.set(Calendar.MILLISECOND, ms) - val date = calendar.getTime() guess = (millisLocal - calendar.getTimeInMillis()).toInt } }