Skip to content

[SPARK-31296][SQL][TESTS] Benchmark date-time rebasing in Parquet datasource #28057

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
MaxGekk wants to merge 14 commits into from
Closed

[SPARK-31296][SQL][TESTS] Benchmark date-time rebasing in Parquet datasource #28057

Changes from 1 commit
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
e217139
Add benchmarks for rebasing date-time in parquet
MaxGekk Mar 27, 2020
24476a3
Minor change: `after` -> `before`
MaxGekk Mar 27, 2020
f3c30b7
Benchmarks for dates
MaxGekk Mar 28, 2020
0734e46
Remove LocalDate
MaxGekk Mar 28, 2020
417001b
Re-gen results of DateTimeRebaseBenchmark on JDK 8
MaxGekk Mar 28, 2020
50247e5
Re-gen results of DateTimeRebaseBenchmark on JDK 11
MaxGekk Mar 28, 2020
8703579
Merge branch 'rebase-bechmark-run' into rebase-bechmark
MaxGekk Mar 28, 2020
27a75a6
Minor change of load bench name
MaxGekk Mar 28, 2020
69bb4ed
Fix date generation
MaxGekk Mar 28, 2020
10382e2
Re-gen results of DateTimeRebaseBenchmark on JDK 11
MaxGekk Mar 28, 2020
912dee4
Re-gen results of DateTimeRebaseBenchmark on JDK 8
MaxGekk Mar 28, 2020
c2cc385
Regen benchmark results on Linux JDK 8
MaxGekk Mar 29, 2020
c89f2c9
Regen benchmark results on Linux JDK 11
MaxGekk Mar 29, 2020
e0aedf5
Rename benchmark
MaxGekk Mar 30, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Fix date generation
  • Loading branch information
@MaxGekk
MaxGekk committed Mar 28, 2020
commit 69bb4ed39e55ebe11f230205ff070ab9e4e17410
19 changes: 16 additions & 3 deletions ...ore/src/test/scala/org/apache/spark/sql/execution/benchmark/DateTimeRebaseBenchmark.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,11 @@

package org.apache.spark.sql.execution.benchmark

import java.time.{LocalDateTime, ZoneOffset}
import java.time.{LocalDate, LocalDateTime, LocalTime, ZoneOffset}

import org.apache.spark.benchmark.Benchmark
import org.apache.spark.sql.DataFrame
import org.apache.spark.sql.catalyst.util.DateTimeConstants.SECONDS_PER_DAY
import org.apache.spark.sql.internal.SQLConf

/**
Expand Down Expand Up @@ -56,15 +57,27 @@ object DateTimeRebaseBenchmark extends SqlBasedBenchmark {
val start = LocalDateTime.of(10, 1, 1, 0, 0, 0)
val end = LocalDateTime.of(1580, 1, 1, 0, 0, 0)
genTs(cardinality, start, end)
}

private def genDate(cardinality: Int, start: LocalDate, end: LocalDate): DataFrame = {
val startSec = LocalDateTime.of(start, LocalTime.MIDNIGHT).toEpochSecond(ZoneOffset.UTC)
val endSec = LocalDateTime.of(end, LocalTime.MIDNIGHT).toEpochSecond(ZoneOffset.UTC)
spark.range(0, cardinality * SECONDS_PER_DAY, SECONDS_PER_DAY, 1)
.select((($"id" % (endSec - startSec)) + startSec).as("seconds"))
.select($"seconds".cast("timestamp").as("ts"))
.select($"ts".cast("date").as("date"))
}

private def genDateAfter1582(cardinality: Int): DataFrame = {
genTsAfter1582(cardinality).select($"ts".cast("date").as("date"))
val start = LocalDate.of(1582, 10, 15)
val end = LocalDate.of(3000, 1, 1)
genDate(cardinality, start, end)
}

private def genDateBefore1582(cardinality: Int): DataFrame = {
genTsBefore1582(cardinality).select($"ts".cast("date").as("date"))
val start = LocalDate.of(10, 1, 1)
val end = LocalDate.of(1580, 1, 1)
genDate(cardinality, start, end)
}

private def genDF(cardinality: Int, dateTime: String, after1582: Boolean): DataFrame = {
Expand Down