Skip to content
Closed
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Update code
  • Loading branch information
chenzhx committed Jul 5, 2022
commit ea0e7c2649aa489fd3daa3f8e57158718b54a19c
146 changes: 71 additions & 75 deletions sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCV2Suite.scala
Original file line number Diff line number Diff line change
Expand Up @@ -1047,81 +1047,77 @@ class JDBCV2Suite extends QueryTest with SharedSparkSession with ExplainSuiteHel
}

test("scan with filter push-down with date time functions") {
Seq(false, true).foreach { ansiMode =>
withSQLConf(SQLConf.ANSI_ENABLED.key -> ansiMode.toString,
SQLConf.MAX_METADATA_STRING_LENGTH.key -> "200") {

val df1 = sql("SELECT name FROM h2.test.datetime WHERE " +
"dayofyear(date1) > 100 AND dayofmonth(date1) > 10 ")
checkFiltersRemoved(df1)
val expectedPlanFragment1 =
"PushedFilters: [DATE1 IS NOT NULL, EXTRACT(DAY_OF_YEAR FROM DATE1) > 100, " +
"EXTRACT(DAY FROM DATE1) > 10]"
checkPushedInfo(df1, expectedPlanFragment1)
checkAnswer(df1, Seq(Row("amy"), Row("alex")))

val df2 = sql("SELECT name FROM h2.test.datetime WHERE " +
"year(date1) = 2022 AND quarter(date1) = 2")
checkFiltersRemoved(df2)
val expectedPlanFragment2 =
"[DATE1 IS NOT NULL, EXTRACT(YEAR FROM DATE1) = 2022, " +
"EXTRACT(QUARTER FROM DATE1) = 2]"
checkPushedInfo(df2, expectedPlanFragment2)
checkAnswer(df2, Seq(Row("amy"), Row("alex")))

val df3 = sql("SELECT name FROM h2.test.datetime WHERE " +
"second(time1) = 0 AND month(date1) = 5")
checkFiltersRemoved(df3)
val expectedPlanFragment3 =
"PushedFilters: [TIME1 IS NOT NULL, DATE1 IS NOT NULL, " +
"EXTRACT(SECOND FROM TIME1) = 0, EXTRACT(MONTH FROM DATE1) = 5]"
checkPushedInfo(df3, expectedPlanFragment3)
checkAnswer(df3, Seq(Row("amy"), Row("alex")))

val df4 = sql("SELECT name FROM h2.test.datetime WHERE " +
"hour(time1) = 0 AND minute(time1) = 0")
checkFiltersRemoved(df4)
val expectedPlanFragment4 =
"PushedFilters: [TIME1 IS NOT NULL, EXTRACT(HOUR FROM TIME1) = 0, " +
"EXTRACT(MINUTE FROM TIME1) = 0]"
checkPushedInfo(df4, expectedPlanFragment4)
checkAnswer(df4, Seq(Row("amy"), Row("alex")))

val df5 = sql("SELECT name FROM h2.test.datetime WHERE " +
"extract(WEEk from date1) > 10 AND extract(YEAROFWEEK from date1) = 2022")
checkFiltersRemoved(df5)
val expectedPlanFragment5 =
"PushedFilters: [DATE1 IS NOT NULL, EXTRACT(WEEK FROM DATE1) > 10, " +
"EXTRACT(YEAR_OF_WEEK FROM DATE1) = 2022]"
checkPushedInfo(df5, expectedPlanFragment5)
checkAnswer(df5, Seq(Row("alex"), Row("amy")))

// H2 does not support
val df6 = sql("SELECT name FROM h2.test.datetime WHERE " +
"trunc(date1, 'week') = date'2022-05-16' AND date_add(date1, 1) = date'2022-05-20' " +
"AND datediff(date1, '2022-05-10') > 0")
checkFiltersRemoved(df6, false)
val expectedPlanFragment6 =
"PushedFilters: [DATE1 IS NOT NULL]"
checkPushedInfo(df6, expectedPlanFragment6)
checkAnswer(df6, Seq(Row("amy")))

val df7 = sql("SELECT name FROM h2.test.datetime WHERE " +
"weekday(date1) = 2")
checkFiltersRemoved(df7)
val expectedPlanFragment7 =
"PushedFilters: [DATE1 IS NOT NULL, (EXTRACT(DAY_OF_WEEK FROM DATE1) - 1) = 2]"
checkPushedInfo(df7, expectedPlanFragment7)
checkAnswer(df7, Seq(Row("alex")))

val df8 = sql("SELECT name FROM h2.test.datetime WHERE " +
"dayofweek(date1) = 4")
checkFiltersRemoved(df8)
val expectedPlanFragment8 =
"PushedFilters: [DATE1 IS NOT NULL, ((EXTRACT(DAY_OF_WEEK FROM DATE1) % 7) + 1) = 4]"
checkPushedInfo(df8, expectedPlanFragment8)
checkAnswer(df8, Seq(Row("alex")))
}
withSQLConf(SQLConf.MAX_METADATA_STRING_LENGTH.key -> "200") {
val df1 = sql("SELECT name FROM h2.test.datetime WHERE " +
"dayofyear(date1) > 100 AND dayofmonth(date1) > 10 ")
checkFiltersRemoved(df1)
val expectedPlanFragment1 =
"PushedFilters: [DATE1 IS NOT NULL, EXTRACT(DAY_OF_YEAR FROM DATE1) > 100, " +
"EXTRACT(DAY FROM DATE1) > 10]"
checkPushedInfo(df1, expectedPlanFragment1)
checkAnswer(df1, Seq(Row("amy"), Row("alex")))

val df2 = sql("SELECT name FROM h2.test.datetime WHERE " +
"year(date1) = 2022 AND quarter(date1) = 2")
checkFiltersRemoved(df2)
val expectedPlanFragment2 =
"[DATE1 IS NOT NULL, EXTRACT(YEAR FROM DATE1) = 2022, " +
"EXTRACT(QUARTER FROM DATE1) = 2]"
checkPushedInfo(df2, expectedPlanFragment2)
checkAnswer(df2, Seq(Row("amy"), Row("alex")))

val df3 = sql("SELECT name FROM h2.test.datetime WHERE " +
"second(time1) = 0 AND month(date1) = 5")
checkFiltersRemoved(df3)
val expectedPlanFragment3 =
"PushedFilters: [TIME1 IS NOT NULL, DATE1 IS NOT NULL, " +
"EXTRACT(SECOND FROM TIME1) = 0, EXTRACT(MONTH FROM DATE1) = 5]"
checkPushedInfo(df3, expectedPlanFragment3)
checkAnswer(df3, Seq(Row("amy"), Row("alex")))

val df4 = sql("SELECT name FROM h2.test.datetime WHERE " +
"hour(time1) = 0 AND minute(time1) = 0")
checkFiltersRemoved(df4)
val expectedPlanFragment4 =
"PushedFilters: [TIME1 IS NOT NULL, EXTRACT(HOUR FROM TIME1) = 0, " +
"EXTRACT(MINUTE FROM TIME1) = 0]"
checkPushedInfo(df4, expectedPlanFragment4)
checkAnswer(df4, Seq(Row("amy"), Row("alex")))

val df5 = sql("SELECT name FROM h2.test.datetime WHERE " +
"extract(WEEk from date1) > 10 AND extract(YEAROFWEEK from date1) = 2022")
checkFiltersRemoved(df5)
val expectedPlanFragment5 =
"PushedFilters: [DATE1 IS NOT NULL, EXTRACT(WEEK FROM DATE1) > 10, " +
"EXTRACT(YEAR_OF_WEEK FROM DATE1) = 2022]"
checkPushedInfo(df5, expectedPlanFragment5)
checkAnswer(df5, Seq(Row("alex"), Row("amy")))

// H2 does not support
val df6 = sql("SELECT name FROM h2.test.datetime WHERE " +
"trunc(date1, 'week') = date'2022-05-16' AND date_add(date1, 1) = date'2022-05-20' " +
"AND datediff(date1, '2022-05-10') > 0")
checkFiltersRemoved(df6, false)
val expectedPlanFragment6 =
"PushedFilters: [DATE1 IS NOT NULL]"
checkPushedInfo(df6, expectedPlanFragment6)
checkAnswer(df6, Seq(Row("amy")))

val df7 = sql("SELECT name FROM h2.test.datetime WHERE " +
"weekday(date1) = 2")
checkFiltersRemoved(df7)
val expectedPlanFragment7 =
"PushedFilters: [DATE1 IS NOT NULL, (EXTRACT(DAY_OF_WEEK FROM DATE1) - 1) = 2]"
checkPushedInfo(df7, expectedPlanFragment7)
checkAnswer(df7, Seq(Row("alex")))

val df8 = sql("SELECT name FROM h2.test.datetime WHERE " +
"dayofweek(date1) = 4")
checkFiltersRemoved(df8)
val expectedPlanFragment8 =
"PushedFilters: [DATE1 IS NOT NULL, ((EXTRACT(DAY_OF_WEEK FROM DATE1) % 7) + 1) = 4]"
checkPushedInfo(df8, expectedPlanFragment8)
checkAnswer(df8, Seq(Row("alex")))
}
}

Expand Down