-
Notifications
You must be signed in to change notification settings - Fork 29k
[SPARK-28420][SQL] Support the INTERVAL type in date_part()
#25981
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Closed
Closed
Changes from 1 commit
Commits
Show all changes
43 commits
Select commit
Hold shift + click to select a range
b75a80f
Add Millennium
MaxGekk b233c90
Add Century
MaxGekk 6f017a6
Add Decade
MaxGekk ac1c3a8
Add Year
MaxGekk 7265508
Add Quarter
MaxGekk 554df71
Move MONTHS_PER_QUARTER up
MaxGekk d0f89f4
Eliminate a warning
MaxGekk 8174fd5
Extend with ExpectsInputTypes
MaxGekk 6378b95
Remove blank lines
MaxGekk 8e4ca7d
Add Month
MaxGekk 8ed01c7
Week is not supported by PostgreSQL
MaxGekk f1eea12
Remove not-supported fields
MaxGekk 561f789
Week of interval is not supported
MaxGekk 0a671a9
Add Day
MaxGekk 894a6c7
Add Hour
MaxGekk f86f4f5
Change Month type to ByteType
MaxGekk 283fd99
Change Quarter type to ByteType
MaxGekk 5e189ca
Put common code to IntervalPart
MaxGekk 62c21b9
Run scalafmt
MaxGekk a4fbb5e
Add Minute
MaxGekk e1c9415
Add Second
MaxGekk 7f4100f
Run scalafmt
MaxGekk f3cf7f0
Refactoring
MaxGekk 9262f9d
Add a test for overflow
MaxGekk b9890ec
Add Milliseconds
MaxGekk bea3faf
Add Microseconds
MaxGekk 77e0fb3
Add Epoch
MaxGekk 58017a7
Support intervals by date_part
MaxGekk f5620b3
Update comments for DatePart
MaxGekk dcaf5b2
Regenerate results of date_part.sql
MaxGekk f202b15
Add tests for intervals to date_part.sql
MaxGekk dca29e5
Remove wrong test
MaxGekk f8a2385
Make Dongjoon and Scala style checker happy
MaxGekk 7b1663e
Merge remote-tracking branch 'remotes/origin/master' into extract-fro…
MaxGekk 8a494a2
Improve an example
MaxGekk f08531b
Precise epoch calculation from intervals
MaxGekk e8a61c8
Fix expected results in IntervalExpressionsSuite
MaxGekk f8a45b3
Revert "Precise epoch calculation from intervals"
MaxGekk a496d73
Precise calculation micros per month
MaxGekk 47a0290
Revert "Precise calculation micros per month"
MaxGekk 2099a91
Fix expected results in IntervalExpressionsSuite
MaxGekk d4375b5
Add the Extract prefix to all classes
MaxGekk 5620472
Change indentation for extends to 2 spaces
MaxGekk File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Revert "Precise calculation micros per month"
This reverts commit a496d73.
- Loading branch information
commit 47a0290fd882bad78f6c848a5547d9aba7b75319
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -21,24 +21,17 @@ import org.apache.spark.sql.types.Decimal | |
| import org.apache.spark.unsafe.types.CalendarInterval | ||
|
|
||
| object IntervalUtils { | ||
| final val MONTHS_PER_YEAR: Int = 12 | ||
| final val MONTHS_PER_QUARTER: Byte = 3 | ||
| final val YEARS_PER_MILLENNIUM: Int = 1000 | ||
| final val YEARS_PER_CENTURY: Int = 100 | ||
| final val YEARS_PER_DECADE: Int = 10 | ||
| final val MICROS_PER_HOUR: Long = DateTimeUtils.MILLIS_PER_HOUR * DateTimeUtils.MICROS_PER_MILLIS | ||
| final val MICROS_PER_MINUTE: Long = { | ||
| DateTimeUtils.MILLIS_PER_MINUTE * DateTimeUtils.MICROS_PER_MILLIS | ||
| } | ||
| // The average year of the Gregorian calendar 365.2425 days long, see | ||
| // https://en.wikipedia.org/wiki/Gregorian_calendar | ||
| // Leap year occurs every 4 years, except for years that are divisible by 100 | ||
| // and not divisible by 400. So, the mean length of of the Gregorian calendar year is: | ||
| // 1 mean year = (365 + 1/4 - 1/100 + 1/400) days = 365.2425 days | ||
| // The mean year length in seconds is: | ||
| // 60 * 60 * 24 * 365.2425 = 31556952.0 = 12 * 2629746 | ||
| final val SECONDS_PER_MONTH: Int = 2629746 | ||
| final val MICROS_PER_MONTH: Long = SECONDS_PER_MONTH * DateTimeUtils.MICROS_PER_SECOND | ||
| val MONTHS_PER_YEAR: Int = 12 | ||
| val MONTHS_PER_QUARTER: Byte = 3 | ||
| val YEARS_PER_MILLENNIUM: Int = 1000 | ||
| val YEARS_PER_CENTURY: Int = 100 | ||
| val YEARS_PER_DECADE: Int = 10 | ||
| val MICROS_PER_HOUR: Long = DateTimeUtils.MILLIS_PER_HOUR * DateTimeUtils.MICROS_PER_MILLIS | ||
| val MICROS_PER_MINUTE: Long = DateTimeUtils.MILLIS_PER_MINUTE * DateTimeUtils.MICROS_PER_MILLIS | ||
| val DAYS_PER_MONTH: Byte = 30 | ||
| val MICROS_PER_MONTH: Long = DAYS_PER_MONTH * DateTimeUtils.SECONDS_PER_DAY | ||
| /* 365.25 days per year assumes leap year every four years */ | ||
| val MICROS_PER_YEAR: Long = (36525L * DateTimeUtils.MICROS_PER_DAY) / 100 | ||
|
|
||
| def getYears(interval: CalendarInterval): Int = { | ||
| interval.months / MONTHS_PER_YEAR | ||
|
|
@@ -90,8 +83,9 @@ object IntervalUtils { | |
|
|
||
| // Returns total number of seconds with microseconds fractional part in the given interval. | ||
| def getEpoch(interval: CalendarInterval): Decimal = { | ||
|
Member
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. |
||
| val monthsDurationUs = Math.multiplyExact(interval.months, MICROS_PER_MONTH) | ||
| val result = Math.addExact(interval.microseconds, monthsDurationUs) | ||
| var result = interval.microseconds | ||
| result += MICROS_PER_YEAR * (interval.months / MONTHS_PER_YEAR) | ||
| result += MICROS_PER_MONTH * (interval.months % MONTHS_PER_YEAR) | ||
| Decimal(result, 18, 6) | ||
| } | ||
| } | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.