Skip to content
Closed
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
29 commits
Select commit Hold shift + click to select a range
b22f243
Generate benchmark results for JDK 8
MaxGekk Oct 24, 2019
c33710f
Generate benchmark results for JDK 11
MaxGekk Oct 24, 2019
ffa1bee
Initial implementation
MaxGekk Oct 25, 2019
a4c16a1
Add tests to IntervalUtilsSuite
MaxGekk Oct 25, 2019
f249f49
Optimize unit name matching
MaxGekk Oct 25, 2019
d077c87
Catch ArithmeticException
MaxGekk Oct 25, 2019
80f5a06
Generate benchmark results for JDK 8
MaxGekk Oct 25, 2019
479d5bd
Generate benchmark results for JDK 11
MaxGekk Oct 25, 2019
d68f41e
Merge remote-tracking branch 'remotes/origin/master' into string-to-i…
MaxGekk Oct 29, 2019
8e8e539
Merge remote-tracking branch 'remotes/origin/master' into string-to-i…
MaxGekk Nov 2, 2019
9dfb45d
Rebase on interval with days
MaxGekk Nov 2, 2019
8c3fb28
Merge remote-tracking branch 'remotes/origin/master' into string-to-i…
MaxGekk Nov 6, 2019
bc006a2
Refactor IntervalUtilsSuite to check fromString and stringToInterval
MaxGekk Nov 6, 2019
7515981
Bug fix: expect space after interval
MaxGekk Nov 6, 2019
78a2e8e
Use checkFromInvalidString
MaxGekk Nov 6, 2019
f61e6f8
Improve the multiple units test
MaxGekk Nov 6, 2019
dd8f2d1
Add a few units to the test
MaxGekk Nov 6, 2019
1204656
Return CalendarInterval from stringToInterval()
MaxGekk Nov 6, 2019
a2d91c3
Support fraction of seconds
MaxGekk Nov 6, 2019
94bd39b
Fix for negative values
MaxGekk Nov 6, 2019
98dd44f
Checks only second can have fractions
MaxGekk Nov 6, 2019
0cd7e88
Regen benchmark results for JDK 8
MaxGekk Nov 6, 2019
107d16c
Regen benchmark results for JDK 11
MaxGekk Nov 6, 2019
8dd9518
Address Wenchen's comments + change behavior for fractions
MaxGekk Nov 6, 2019
464eacc
Improve comments
MaxGekk Nov 6, 2019
dbad971
Unify structure
MaxGekk Nov 6, 2019
527b00e
Remove exact functions in fraction parsing
MaxGekk Nov 6, 2019
5e96a0e
Replace magic numbers
MaxGekk Nov 6, 2019
2222f13
Replace 100000000 by NANOS_PER_SECOND / 10
MaxGekk Nov 6, 2019
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Merge remote-tracking branch 'remotes/origin/master' into string-to-i…
…nterval

# Conflicts:
#	sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala
#	sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/IntervalUtilsSuite.scala
  • Loading branch information
MaxGekk committed Nov 2, 2019
commit 8e8e5397cc4431cd366c1eb3edf22ecf84c2bf75
Original file line number Diff line number Diff line change
Expand Up @@ -325,6 +325,47 @@ object IntervalUtils {
}
}

/**
* Gets interval duration
*
* @param interval The interval to get duration
* @param targetUnit Time units of the result
* @param daysPerMonth The number of days per one month. The default value is 31 days
* per month. This value was taken as the default because it is used
* in Structured Streaming for watermark calculations. Having 31 days
* per month, we can guarantee that events are not dropped before
* the end of any month (February with 29 days or January with 31 days).
* @return Duration in the specified time units
*/
def getDuration(
interval: CalendarInterval,
targetUnit: TimeUnit,
daysPerMonth: Int = 31): Long = {
val monthsDuration = Math.multiplyExact(
daysPerMonth * DateTimeUtils.MICROS_PER_DAY,
interval.months)
val daysDuration = Math.multiplyExact(
DateTimeUtils.MICROS_PER_DAY,
interval.days)
val result = Math.addExact(interval.microseconds, Math.addExact(daysDuration, monthsDuration))
targetUnit.convert(result, TimeUnit.MICROSECONDS)
}

/**
* Checks the interval is negative
*
* @param interval The checked interval
* @param daysPerMonth The number of days per one month. The default value is 31 days
* per month. This value was taken as the default because it is used
* in Structured Streaming for watermark calculations. Having 31 days
* per month, we can guarantee that events are not dropped before
* the end of any month (February with 29 days or January with 31 days).
* @return true if duration of the given interval is less than 0 otherwise false
*/
def isNegative(interval: CalendarInterval, daysPerMonth: Int = 31): Boolean = {
getDuration(interval, TimeUnit.MICROSECONDS, daysPerMonth) < 0
}

private object ParseState extends Enumeration {
val PREFIX,
BEGIN_VALUE,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -153,6 +153,40 @@ class IntervalUtilsSuite extends SparkFunSuite {
}
}

test("interval duration") {
def duration(s: String, unit: TimeUnit, daysPerMonth: Int): Long = {
IntervalUtils.getDuration(fromString(s), unit, daysPerMonth)
}

assert(duration("0 seconds", TimeUnit.MILLISECONDS, 31) === 0)
assert(duration("1 month", TimeUnit.DAYS, 31) === 31)
assert(duration("1 microsecond", TimeUnit.MICROSECONDS, 30) === 1)
assert(duration("1 month -30 days", TimeUnit.DAYS, 31) === 1)

try {
duration(Integer.MAX_VALUE + " month", TimeUnit.SECONDS, 31)
fail("Expected to throw an exception for the invalid input")
} catch {
case e: ArithmeticException =>
assert(e.getMessage.contains("overflow"))
}
}

test("negative interval") {
def isNegative(s: String, daysPerMonth: Int): Boolean = {
IntervalUtils.isNegative(fromString(s), daysPerMonth)
}

assert(isNegative("-1 months", 28))
assert(isNegative("-1 microsecond", 30))
assert(isNegative("-1 month 30 days", 31))
assert(isNegative("2 months -61 days", 30))
assert(isNegative("-1 year -2 seconds", 30))
assert(!isNegative("0 months", 28))
assert(!isNegative("1 year -360 days", 31))
assert(!isNegative("-1 year 380 days", 31))
}

test("string to interval") {
Copy link
Contributor

@cloud-fan cloud-fan Nov 5, 2019

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

can we always test stringToInterval and safeFromString together to make sure they have the same behavior?

Seq(
"1 day",
Expand Down
You are viewing a condensed version of this merge commit. You can view the full changes here.