Merge remote-tracking branch 'origin/master' into interval-mul-div

sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/intervalExpressions.scala

@@ -17,10 +17,99 @@
 
 package org.apache.spark.sql.catalyst.expressions
 
+import java.util.Locale
+
 import org.apache.spark.sql.catalyst.expressions.codegen.{CodegenContext, ExprCode}
-import org.apache.spark.sql.types.{AbstractDataType, CalendarIntervalType, DataType, DoubleType}
+import org.apache.spark.sql.catalyst.util.IntervalUtils
+import org.apache.spark.sql.catalyst.util.IntervalUtils._
+import org.apache.spark.sql.types._
 import org.apache.spark.unsafe.types.CalendarInterval
 
+abstract class ExtractIntervalPart(
+    child: Expression,
+    val dataType: DataType,
+    func: CalendarInterval => Any,
+    funcName: String)
+  extends UnaryExpression with ExpectsInputTypes with Serializable {
+
+  override def inputTypes: Seq[AbstractDataType] = Seq(CalendarIntervalType)
+
+  override protected def nullSafeEval(interval: Any): Any = {
+    func(interval.asInstanceOf[CalendarInterval])
+  }
+
+  override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
+    val iu = IntervalUtils.getClass.getName.stripSuffix("$")
+    defineCodeGen(ctx, ev, c => s"$iu.$funcName($c)")
+  }
+}
+
+case class ExtractIntervalMillenniums(child: Expression)
+  extends ExtractIntervalPart(child, IntegerType, getMillenniums, "getMillenniums")
+
+case class ExtractIntervalCenturies(child: Expression)
+  extends ExtractIntervalPart(child, IntegerType, getCenturies, "getCenturies")
+
+case class ExtractIntervalDecades(child: Expression)
+  extends ExtractIntervalPart(child, IntegerType, getDecades, "getDecades")
+
+case class ExtractIntervalYears(child: Expression)
+  extends ExtractIntervalPart(child, IntegerType, getYears, "getYears")
+
+case class ExtractIntervalQuarters(child: Expression)
+  extends ExtractIntervalPart(child, ByteType, getQuarters, "getQuarters")
+
+case class ExtractIntervalMonths(child: Expression)
+  extends ExtractIntervalPart(child, ByteType, getMonths, "getMonths")
+
+case class ExtractIntervalDays(child: Expression)
+  extends ExtractIntervalPart(child, LongType, getDays, "getDays")
+
+case class ExtractIntervalHours(child: Expression)
+  extends ExtractIntervalPart(child, ByteType, getHours, "getHours")
+
+case class ExtractIntervalMinutes(child: Expression)
+  extends ExtractIntervalPart(child, ByteType, getMinutes, "getMinutes")
+
+case class ExtractIntervalSeconds(child: Expression)
+  extends ExtractIntervalPart(child, DecimalType(8, 6), getSeconds, "getSeconds")
+
+case class ExtractIntervalMilliseconds(child: Expression)
+  extends ExtractIntervalPart(child, DecimalType(8, 3), getMilliseconds, "getMilliseconds")
+
+case class ExtractIntervalMicroseconds(child: Expression)
+  extends ExtractIntervalPart(child, LongType, getMicroseconds, "getMicroseconds")
+
+// Number of seconds in 10000 years is 315576000001 (30 days per one month)
+// which is 12 digits + 6 digits for the fractional part of seconds.
+case class ExtractIntervalEpoch(child: Expression)
+  extends ExtractIntervalPart(child, DecimalType(18, 6), getEpoch, "getEpoch")
+
+object ExtractIntervalPart {
+
+  def parseExtractField(
+      extractField: String,
+      source: Expression,
+      errorHandleFunc: => Nothing): Expression = extractField.toUpperCase(Locale.ROOT) match {
+    case "MILLENNIUM" | "MILLENNIA" | "MIL" | "MILS" => ExtractIntervalMillenniums(source)
+    case "CENTURY" | "CENTURIES" | "C" | "CENT" => ExtractIntervalCenturies(source)
+    case "DECADE" | "DECADES" | "DEC" | "DECS" => ExtractIntervalDecades(source)
+    case "YEAR" | "Y" | "YEARS" | "YR" | "YRS" => ExtractIntervalYears(source)
+    case "QUARTER" | "QTR" => ExtractIntervalQuarters(source)
+    case "MONTH" | "MON" | "MONS" | "MONTHS" => ExtractIntervalMonths(source)
+    case "DAY" | "D" | "DAYS" => ExtractIntervalDays(source)
+    case "HOUR" | "H" | "HOURS" | "HR" | "HRS" => ExtractIntervalHours(source)
+    case "MINUTE" | "M" | "MIN" | "MINS" | "MINUTES" => ExtractIntervalMinutes(source)
+    case "SECOND" | "S" | "SEC" | "SECONDS" | "SECS" => ExtractIntervalSeconds(source)
+    case "MILLISECONDS" | "MSEC" | "MSECS" | "MILLISECON" | "MSECONDS" | "MS" =>
+      ExtractIntervalMilliseconds(source)
+    case "MICROSECONDS" | "USEC" | "USECS" | "USECONDS" | "MICROSECON" | "US" =>
+      ExtractIntervalMicroseconds(source)
+    case "EPOCH" => ExtractIntervalEpoch(source)
+    case _ => errorHandleFunc
+  }
+}
+
 abstract class IntervalNumOperation(
     interval: Expression,
     num: Expression,

sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/IntervalExpressionsSuite.scala

@@ -17,10 +17,179 @@
 
 package org.apache.spark.sql.catalyst.expressions
 
+import scala.language.implicitConversions
+
 import org.apache.spark.SparkFunSuite
+import org.apache.spark.sql.types.Decimal
 import org.apache.spark.unsafe.types.CalendarInterval.fromString
 
 class IntervalExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
+  implicit def interval(s: String): Literal = {
+    Literal(fromString("interval " + s))
+  }
+
+  test("millenniums") {
+    checkEvaluation(ExtractIntervalMillenniums("0 years"), 0)
+    checkEvaluation(ExtractIntervalMillenniums("9999 years"), 9)
+    checkEvaluation(ExtractIntervalMillenniums("1000 years"), 1)
+    checkEvaluation(ExtractIntervalMillenniums("-2000 years"), -2)
+    // Microseconds part must not be taken into account
+    checkEvaluation(ExtractIntervalMillenniums("999 years 400 days"), 0)
+    // Millennium must be taken from years and months
+    checkEvaluation(ExtractIntervalMillenniums("999 years 12 months"), 1)
+    checkEvaluation(ExtractIntervalMillenniums("1000 years -1 months"), 0)
+  }
+
+  test("centuries") {
+    checkEvaluation(ExtractIntervalCenturies("0 years"), 0)
+    checkEvaluation(ExtractIntervalCenturies("9999 years"), 99)
+    checkEvaluation(ExtractIntervalCenturies("1000 years"), 10)
+    checkEvaluation(ExtractIntervalCenturies("-2000 years"), -20)
+    // Microseconds part must not be taken into account
+    checkEvaluation(ExtractIntervalCenturies("99 years 400 days"), 0)
+    // Century must be taken from years and months
+    checkEvaluation(ExtractIntervalCenturies("99 years 12 months"), 1)
+    checkEvaluation(ExtractIntervalCenturies("100 years -1 months"), 0)
+  }
+
+  test("decades") {
+    checkEvaluation(ExtractIntervalDecades("0 years"), 0)
+    checkEvaluation(ExtractIntervalDecades("9999 years"), 999)
+    checkEvaluation(ExtractIntervalDecades("1000 years"), 100)
+    checkEvaluation(ExtractIntervalDecades("-2000 years"), -200)
+    // Microseconds part must not be taken into account
+    checkEvaluation(ExtractIntervalDecades("9 years 400 days"), 0)
+    // Decade must be taken from years and months
+    checkEvaluation(ExtractIntervalDecades("9 years 12 months"), 1)
+    checkEvaluation(ExtractIntervalDecades("10 years -1 months"), 0)
+  }
+
+  test("years") {
+    checkEvaluation(ExtractIntervalYears("0 years"), 0)
+    checkEvaluation(ExtractIntervalYears("9999 years"), 9999)
+    checkEvaluation(ExtractIntervalYears("1000 years"), 1000)
+    checkEvaluation(ExtractIntervalYears("-2000 years"), -2000)
+    // Microseconds part must not be taken into account
+    checkEvaluation(ExtractIntervalYears("9 years 400 days"), 9)
+    // Year must be taken from years and months
+    checkEvaluation(ExtractIntervalYears("9 years 12 months"), 10)
+    checkEvaluation(ExtractIntervalYears("10 years -1 months"), 9)
+  }
+
+  test("quarters") {
+    checkEvaluation(ExtractIntervalQuarters("0 months"), 1.toByte)
+    checkEvaluation(ExtractIntervalQuarters("1 months"), 1.toByte)
+    checkEvaluation(ExtractIntervalQuarters("-1 months"), 1.toByte)
+    checkEvaluation(ExtractIntervalQuarters("2 months"), 1.toByte)
+    checkEvaluation(ExtractIntervalQuarters("-2 months"), 1.toByte)
+    checkEvaluation(ExtractIntervalQuarters("1 years -1 months"), 4.toByte)
+    checkEvaluation(ExtractIntervalQuarters("-1 years 1 months"), -2.toByte)
+    checkEvaluation(ExtractIntervalQuarters("2 years 3 months"), 2.toByte)
+    checkEvaluation(ExtractIntervalQuarters("-2 years -3 months"), 0.toByte)
+    checkEvaluation(ExtractIntervalQuarters("9999 years"), 1.toByte)
+  }
+
+  test("months") {
+    checkEvaluation(ExtractIntervalMonths("0 year"), 0.toByte)
+    for (m <- -24 to 24) {
+      checkEvaluation(ExtractIntervalMonths(s"$m months"), (m % 12).toByte)
+    }
+    checkEvaluation(ExtractIntervalMonths("1 year 10 months"), 10.toByte)
+    checkEvaluation(ExtractIntervalMonths("-2 year -10 months"), -10.toByte)
+    checkEvaluation(ExtractIntervalMonths("9999 years"), 0.toByte)
+  }
+
+  private val largeInterval: String = "9999 years 11 months " +
+    "31 days 11 hours 59 minutes 59 seconds 999 milliseconds 999 microseconds"
+
+  test("days") {
+    checkEvaluation(ExtractIntervalDays("0 days"), 0L)
+    checkEvaluation(ExtractIntervalDays("1 days 100 seconds"), 1L)
+    checkEvaluation(ExtractIntervalDays("-1 days -100 seconds"), -1L)
+    checkEvaluation(ExtractIntervalDays("-365 days"), -365L)
+    checkEvaluation(ExtractIntervalDays("365 days"), 365L)
+    // Years and months must not be taken into account
+    checkEvaluation(ExtractIntervalDays("100 year 10 months 5 days"), 5L)
+    checkEvaluation(ExtractIntervalDays(largeInterval), 31L)
+  }
+
+  test("hours") {
+    checkEvaluation(ExtractIntervalHours("0 hours"), 0.toByte)
+    checkEvaluation(ExtractIntervalHours("1 hour"), 1.toByte)
+    checkEvaluation(ExtractIntervalHours("-1 hour"), -1.toByte)
+    checkEvaluation(ExtractIntervalHours("23 hours"), 23.toByte)
+    checkEvaluation(ExtractIntervalHours("-23 hours"), -23.toByte)
+    // Years and months must not be taken into account
+    checkEvaluation(ExtractIntervalHours("100 year 10 months 10 hours"), 10.toByte)
+    checkEvaluation(ExtractIntervalHours(largeInterval), 11.toByte)
+  }
+
+  test("minutes") {
+    checkEvaluation(ExtractIntervalMinutes("0 minute"), 0.toByte)
+    checkEvaluation(ExtractIntervalMinutes("1 minute"), 1.toByte)
+    checkEvaluation(ExtractIntervalMinutes("-1 minute"), -1.toByte)
+    checkEvaluation(ExtractIntervalMinutes("59 minute"), 59.toByte)
+    checkEvaluation(ExtractIntervalMinutes("-59 minute"), -59.toByte)
+    // Years and months must not be taken into account
+    checkEvaluation(ExtractIntervalMinutes("100 year 10 months 10 minutes"), 10.toByte)
+    checkEvaluation(ExtractIntervalMinutes(largeInterval), 59.toByte)
+  }
+
+  test("seconds") {
+    checkEvaluation(ExtractIntervalSeconds("0 second"), Decimal(0, 8, 6))
+    checkEvaluation(ExtractIntervalSeconds("1 second"), Decimal(1.0, 8, 6))
+    checkEvaluation(ExtractIntervalSeconds("-1 second"), Decimal(-1.0, 8, 6))
+    checkEvaluation(ExtractIntervalSeconds("1 minute 59 second"), Decimal(59.0, 8, 6))
+    checkEvaluation(ExtractIntervalSeconds("-59 minutes -59 seconds"), Decimal(-59.0, 8, 6))
+    // Years and months must not be taken into account
+    checkEvaluation(ExtractIntervalSeconds("100 year 10 months 10 seconds"), Decimal(10.0, 8, 6))
+    checkEvaluation(ExtractIntervalSeconds(largeInterval), Decimal(59.999999, 8, 6))
+    checkEvaluation(
+      ExtractIntervalSeconds("10 seconds 1 milliseconds 1 microseconds"),
+      Decimal(10001001, 8, 6))
+    checkEvaluation(ExtractIntervalSeconds("61 seconds 1 microseconds"), Decimal(1000001, 8, 6))
+  }
+
+  test("milliseconds") {
+    checkEvaluation(ExtractIntervalMilliseconds("0 milliseconds"), Decimal(0, 8, 3))
+    checkEvaluation(ExtractIntervalMilliseconds("1 milliseconds"), Decimal(1.0, 8, 3))
+    checkEvaluation(ExtractIntervalMilliseconds("-1 milliseconds"), Decimal(-1.0, 8, 3))
+    checkEvaluation(
+      ExtractIntervalMilliseconds("1 second 999 milliseconds"),
+      Decimal(1999.0, 8, 3))
+    checkEvaluation(
+      ExtractIntervalMilliseconds("999 milliseconds 1 microsecond"),
+      Decimal(999.001, 8, 3))
+    checkEvaluation(
+      ExtractIntervalMilliseconds("-1 second -999 milliseconds"),
+      Decimal(-1999.0, 8, 3))
+    // Years and months must not be taken into account
+    checkEvaluation(ExtractIntervalMilliseconds("100 year 1 millisecond"), Decimal(1.0, 8, 3))
+    checkEvaluation(ExtractIntervalMilliseconds(largeInterval), Decimal(59999.999, 8, 3))
+  }
+
+  test("microseconds") {
+    checkEvaluation(ExtractIntervalMicroseconds("0 microseconds"), 0L)
+    checkEvaluation(ExtractIntervalMicroseconds("1 microseconds"), 1L)
+    checkEvaluation(ExtractIntervalMicroseconds("-1 microseconds"), -1L)
+    checkEvaluation(ExtractIntervalMicroseconds("1 second 999 microseconds"), 1000999L)
+    checkEvaluation(ExtractIntervalMicroseconds("999 milliseconds 1 microseconds"), 999001L)
+    checkEvaluation(ExtractIntervalMicroseconds("-1 second -999 microseconds"), -1000999L)
+    // Years and months must not be taken into account
+    checkEvaluation(ExtractIntervalMicroseconds("11 year 1 microseconds"), 1L)
+    checkEvaluation(ExtractIntervalMicroseconds(largeInterval), 59999999L)
+  }
+
+  test("epoch") {
+    checkEvaluation(ExtractIntervalEpoch("0 months"), Decimal(0.0, 18, 6))
+    checkEvaluation(ExtractIntervalEpoch("10000 years"), Decimal(315576000000.0, 18, 6))
+    checkEvaluation(ExtractIntervalEpoch("1 year"), Decimal(31557600.0, 18, 6))
+    checkEvaluation(ExtractIntervalEpoch("-1 year"), Decimal(-31557600.0, 18, 6))
+    checkEvaluation(
+      ExtractIntervalEpoch("1 second 1 millisecond 1 microsecond"),
+      Decimal(1.001001, 18, 6))
+  }
+
   test("multiply") {
     def multiply(interval: String, num: Double): Expression = {
       MultiplyInterval(Literal(fromString(interval)), Literal(num))
@@ -46,6 +215,7 @@ class IntervalExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
     def divide(interval: String, num: Double): Expression = {
       DivideInterval(Literal(fromString(interval)), Literal(num))
     }
+
     checkEvaluation(divide("0 seconds", 10), fromString("0 seconds"))
     checkEvaluation(
       divide("12 months 3 milliseconds", 2),