Skip to content
Closed
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
37 commits
Select commit Hold shift + click to select a range
c78a347
WIP
hvanhovell Jul 25, 2023
0eb42c1
Fix conflicts
hvanhovell Jul 26, 2023
f648bef
Finish move of classes
hvanhovell Jul 26, 2023
be5b320
Fix legacy policy
hvanhovell Jul 26, 2023
c9fc20c
Remove ToJsonUtil
hvanhovell Jul 26, 2023
0b1b803
Add UDTUtilsImpl
hvanhovell Jul 26, 2023
783a1af
Fix stuff
hvanhovell Jul 26, 2023
4fa9124
Merge with master
hvanhovell Jul 26, 2023
4fc0e6d
Simplification & Fixes
hvanhovell Jul 26, 2023
65ea542
Disconnect Arrow encoders from Catalyst
hvanhovell Jul 26, 2023
b70413b
Disconnect Arrow encoders from Catalyst
hvanhovell Jul 26, 2023
67d1592
Fix proto functions
hvanhovell Jul 26, 2023
340304d
Fix proto functions
hvanhovell Jul 26, 2023
8ca1d2b
fix proto for real
hvanhovell Jul 26, 2023
4845456
Make tests compile!
hvanhovell Jul 26, 2023
42d8f0f
Capstone
hvanhovell Jul 26, 2023
69b8868
Fix compilation
hvanhovell Jul 26, 2023
1927189
Style
hvanhovell Jul 26, 2023
a4f3050
Fix formatting
hvanhovell Jul 27, 2023
ad78a8d
Merge branch 'SPARK-44538' into SPARK-41400-v1
hvanhovell Jul 27, 2023
9f10a8c
Merge remote-tracking branch 'apache/master' into SPARK-41400-v1
hvanhovell Jul 27, 2023
cc981b4
Integrate AnalyisException
hvanhovell Jul 27, 2023
bdd4346
Merge with master
hvanhovell Jul 27, 2023
2b8879d
Put back DateTimeUtils and enrich SparkDateTimeUtils.
hvanhovell Jul 27, 2023
d6a57a0
Merge remote-tracking branch 'apache/master' into SPARK-44538
hvanhovell Jul 27, 2023
e2b0dc4
Merge branch 'SPARK-44538' into SPARK-41400-v1
hvanhovell Jul 27, 2023
fcda03c
Put back string to/from bytebuffer
hvanhovell Jul 27, 2023
912f892
Undo change to RebaseDateTimeSuite
hvanhovell Jul 27, 2023
eeaa409
Undo change
hvanhovell Jul 27, 2023
30f3ce4
style...
hvanhovell Jul 28, 2023
6b53488
Merge branch 'SPARK-44538' into SPARK-41400-v1
hvanhovell Jul 28, 2023
470fed4
Merge remote-tracking branch 'apache/master' into SPARK-41400-v1
hvanhovell Jul 28, 2023
c81d847
Fix docs
hvanhovell Jul 28, 2023
064bc40
Merge remote-tracking branch 'apache/master' into SPARK-41400-v1
hvanhovell Jul 28, 2023
ace8fed
Weird Bug Fix
hvanhovell Jul 28, 2023
5c5aa60
Better fix
hvanhovell Jul 28, 2023
b7bc5c4
Fix MiMa
hvanhovell Jul 28, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Finish move of classes
  • Loading branch information
hvanhovell committed Jul 26, 2023
commit f648befe4eccff60fa49265cd43742b60ca55404
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ import org.apache.spark.sql.catalyst.ScalaReflection
import org.apache.spark.sql.catalyst.encoders.AgnosticEncoder
import org.apache.spark.sql.catalyst.encoders.AgnosticEncoders._
import org.apache.spark.sql.catalyst.expressions.GenericRowWithSchema
import org.apache.spark.sql.errors.{DataTypeErrors, QueryCompilationErrors}
import org.apache.spark.sql.errors.{ExecutionErrors, QueryCompilationErrors}
import org.apache.spark.sql.types.Decimal

/**
Expand Down Expand Up @@ -341,7 +341,7 @@ object ArrowDeserializers {
}

case (CalendarIntervalEncoder | _: UDTEncoder[_], _) =>
throw DataTypeErrors.unsupportedDataTypeError(encoder.dataType)
throw ExecutionErrors.unsupportedDataTypeError(encoder.dataType)

case _ =>
throw new RuntimeException(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ import org.apache.spark.sql.catalyst.DefinedByConstructorParams
import org.apache.spark.sql.catalyst.encoders.AgnosticEncoder
import org.apache.spark.sql.catalyst.encoders.AgnosticEncoders._
import org.apache.spark.sql.catalyst.util.{DateTimeUtils, IntervalUtils}
import org.apache.spark.sql.errors.DataTypeErrors
import org.apache.spark.sql.errors.ExecutionErrors
import org.apache.spark.sql.types.Decimal
import org.apache.spark.sql.util.ArrowUtils

Expand Down Expand Up @@ -439,7 +439,7 @@ object ArrowSerializer {
}

case (CalendarIntervalEncoder | _: UDTEncoder[_], _) =>
throw DataTypeErrors.unsupportedDataTypeError(encoder.dataType)
throw ExecutionErrors.unsupportedDataTypeError(encoder.dataType)

case _ =>
throw new RuntimeException(s"Unsupported Encoder($encoder)/Vector($v) combination.")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,9 +24,9 @@ import java.time.temporal.{ChronoField, TemporalAccessor, TemporalQueries}
import java.util
import java.util.{Collections, Date, Locale}

import org.apache.spark.SPARK_DOC_ROOT
import org.apache.spark.SparkBuildInfo.{spark_doc_root => SPARK_DOC_ROOT}
import org.apache.spark.sql.catalyst.util.DateTimeFormatterHelper._
import org.apache.spark.sql.errors.QueryExecutionErrors
import org.apache.spark.sql.errors.ExecutionErrors
import org.apache.spark.sql.internal.LegacyBehaviorPolicy._
import org.apache.spark.sql.internal.SqlApiConf

Expand All @@ -45,7 +45,7 @@ trait DateTimeFormatterHelper {
val actual = accessor.get(field)
val expected = candidate.get(field)
if (actual != expected) {
throw QueryExecutionErrors.fieldDiffersFromDerivedLocalDateError(
throw ExecutionErrors.fieldDiffersFromDerivedLocalDateError(
field, actual, expected, candidate)
}
}
Expand Down Expand Up @@ -140,7 +140,7 @@ trait DateTimeFormatterHelper {
} catch {
case _: Throwable => throw e
}
throw QueryExecutionErrors.failToParseDateTimeInNewParserError(s, e)
throw ExecutionErrors.failToParseDateTimeInNewParserError(s, e)
}

// When legacy time parser policy set to EXCEPTION, check whether we will get different results
Expand All @@ -156,7 +156,7 @@ trait DateTimeFormatterHelper {
} catch {
case _: Throwable => throw e
}
throw QueryExecutionErrors.failToParseDateTimeInNewParserError(resultCandidate, e)
throw ExecutionErrors.failToParseDateTimeInNewParserError(resultCandidate, e)
}

/**
Expand All @@ -179,22 +179,22 @@ trait DateTimeFormatterHelper {
} catch {
case _: Throwable => throw e
}
throw QueryExecutionErrors.failToRecognizePatternAfterUpgradeError(
throw ExecutionErrors.failToRecognizePatternAfterUpgradeError(
pattern, e, SPARK_DOC_ROOT)
}

protected def checkInvalidPattern(pattern: String): PartialFunction[Throwable, Nothing] = {
case e: IllegalArgumentException =>
throw QueryExecutionErrors.failToRecognizePatternError(pattern, e, SPARK_DOC_ROOT)
throw ExecutionErrors.failToRecognizePatternError(pattern, e, SPARK_DOC_ROOT)
}
}

private object DateTimeFormatterHelper {

private val cache: util.Map[(String, Locale, Boolean), DateTimeFormatter] =
Collections.synchronizedMap(new util.LinkedHashMap {
private type CacheKey = (String, Locale, Boolean)
private val cache: util.Map[CacheKey, DateTimeFormatter] =
Collections.synchronizedMap(new util.LinkedHashMap[CacheKey, DateTimeFormatter] {
override def removeEldestEntry(
eldest: util.Map.Entry[(String, Locale, Boolean), DateTimeFormatter]): Boolean = {
eldest: util.Map.Entry[CacheKey, DateTimeFormatter]): Boolean = {
size() > 128
}
})
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,8 +30,8 @@ import sun.util.calendar.ZoneInfo
import org.apache.spark.sql.catalyst.trees.SQLQueryContext
import org.apache.spark.sql.catalyst.util.DateTimeConstants._
import org.apache.spark.sql.catalyst.util.RebaseDateTime._
import org.apache.spark.sql.errors.QueryExecutionErrors
import org.apache.spark.sql.types.{DateType, Decimal, DoubleExactNumeric, DoubleType, StringType, TimestampNTZType, TimestampType}
import org.apache.spark.sql.errors.ExecutionErrors
import org.apache.spark.sql.types.{DateType, Decimal, DoubleExactNumeric, TimestampNTZType, TimestampType}
import org.apache.spark.unsafe.types.{CalendarInterval, UTF8String}

/**
Expand Down Expand Up @@ -471,15 +471,13 @@ object DateTimeUtils {
timeZoneId: ZoneId,
context: SQLQueryContext = null): Long = {
stringToTimestamp(s, timeZoneId).getOrElse {
throw QueryExecutionErrors.invalidInputInCastToDatetimeError(
s, StringType, TimestampType, context)
throw ExecutionErrors.invalidInputInCastToDatetimeError(s, TimestampType, context)
}
}

def doubleToTimestampAnsi(d: Double, context: SQLQueryContext): Long = {
if (d.isNaN || d.isInfinite) {
throw QueryExecutionErrors.invalidInputInCastToDatetimeError(
d, DoubleType, TimestampType, context)
throw ExecutionErrors.invalidInputInCastToDatetimeError(d, TimestampType, context)
} else {
DoubleExactNumeric.toLong(d * MICROS_PER_SECOND)
}
Expand Down Expand Up @@ -530,8 +528,7 @@ object DateTimeUtils {
s: UTF8String,
context: SQLQueryContext): Long = {
stringToTimestampWithoutTimeZone(s, true).getOrElse {
throw QueryExecutionErrors.invalidInputInCastToDatetimeError(
s, StringType, TimestampNTZType, context)
throw ExecutionErrors.invalidInputInCastToDatetimeError(s, TimestampNTZType, context)
}
}

Expand Down Expand Up @@ -651,8 +648,7 @@ object DateTimeUtils {
s: UTF8String,
context: SQLQueryContext = null): Int = {
stringToDate(s).getOrElse {
throw QueryExecutionErrors.invalidInputInCastToDatetimeError(
s, StringType, DateType, context)
throw ExecutionErrors.invalidInputInCastToDatetimeError(s, DateType, context)
}
}

Expand Down Expand Up @@ -846,7 +842,7 @@ object DateTimeUtils {
start: Int,
interval: CalendarInterval): Int = {
if (interval.microseconds != 0) {
throw QueryExecutionErrors.ansiIllegalArgumentError(
throw ExecutionErrors.ansiIllegalArgumentError(
"Cannot add hours, minutes or seconds, milliseconds, microseconds to a date")
}
val ld = daysToLocalDate(start).plusMonths(interval.months).plusDays(interval.days)
Expand Down Expand Up @@ -980,7 +976,7 @@ object DateTimeUtils {
case TRUNC_TO_YEAR => days - getDayInYear(days) + 1
case _ =>
// caller make sure that this should never be reached
throw QueryExecutionErrors.unreachableError(s": Invalid trunc level: $level")
throw ExecutionErrors.unreachableError(s": Invalid trunc level: $level")
}
}

Expand Down Expand Up @@ -1256,7 +1252,7 @@ object DateTimeUtils {
case _: scala.MatchError =>
throw new IllegalStateException(s"Got the unexpected unit '$unit'.")
case _: ArithmeticException | _: DateTimeException =>
throw QueryExecutionErrors.timestampAddOverflowError(micros, quantity, unit)
throw ExecutionErrors.timestampAddOverflowError(micros, quantity, unit)
case e: Throwable =>
throw new IllegalStateException(s"Failure of 'timestampAdd': ${e.getMessage}")
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
package org.apache.spark.sql.catalyst.util

import org.apache.spark.sql.catalyst.trees.SQLQueryContext
import org.apache.spark.sql.errors.QueryExecutionErrors
import org.apache.spark.sql.errors.ExecutionErrors

/**
* Helper functions for arithmetic operations with overflow.
Expand Down Expand Up @@ -83,7 +83,7 @@ object MathUtils {
f
} catch {
case e: ArithmeticException =>
throw QueryExecutionErrors.arithmeticOverflowError(e.getMessage, hint, context)
throw ExecutionErrors.arithmeticOverflowError(e.getMessage, hint, context)
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ import org.apache.spark.sql.catalyst.util.DateTimeConstants._
import org.apache.spark.sql.catalyst.util.DateTimeUtils._
import org.apache.spark.sql.catalyst.util.LegacyDateFormats.{LegacyDateFormat, LENIENT_SIMPLE_DATE_FORMAT}
import org.apache.spark.sql.catalyst.util.RebaseDateTime._
import org.apache.spark.sql.errors.QueryExecutionErrors
import org.apache.spark.sql.errors.ExecutionErrors
import org.apache.spark.sql.internal.LegacyBehaviorPolicy._
import org.apache.spark.sql.internal.SqlApiConf
import org.apache.spark.sql.types.{Decimal, TimestampNTZType}
Expand Down Expand Up @@ -212,7 +212,7 @@ class Iso8601TimestampFormatter(
parsed: TemporalAccessor,
allowTimeZone: Boolean): Long = {
if (!allowTimeZone && parsed.query(TemporalQueries.zone()) != null) {
throw QueryExecutionErrors.cannotParseStringAsDataTypeError(pattern, s, TimestampNTZType)
throw ExecutionErrors.cannotParseStringAsDataTypeError(pattern, s, TimestampNTZType)
}
val localDate = toLocalDate(parsed)
val localTime = toLocalTime(parsed)
Expand Down Expand Up @@ -292,7 +292,7 @@ class DefaultTimestampFormatter(
try {
val utf8Value = UTF8String.fromString(s)
DateTimeUtils.stringToTimestampWithoutTimeZone(utf8Value, allowTimeZone).getOrElse {
throw QueryExecutionErrors.cannotParseStringAsDataTypeError(
throw ExecutionErrors.cannotParseStringAsDataTypeError(
TimestampFormatter.defaultPattern(), s, TimestampNTZType)
}
} catch checkParsedDiff(s, legacyFormatter.parse)
Expand Down

This file was deleted.

Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
*/
package org.apache.spark.sql.errors

import org.apache.spark.{SparkArithmeticException, SparkDateTimeException, SparkException, SparkIllegalArgumentException, SparkNumberFormatException, SparkRuntimeException, SparkUnsupportedOperationException}
import org.apache.spark.{SparkArithmeticException, SparkException, SparkIllegalArgumentException, SparkNumberFormatException, SparkRuntimeException, SparkUnsupportedOperationException}
import org.apache.spark.sql.catalyst.trees.{Origin, SQLQueryContext}
import org.apache.spark.sql.catalyst.util.QuotingUtils
import org.apache.spark.sql.types.{DataType, Decimal, StringType}
Expand Down Expand Up @@ -295,28 +295,4 @@ private[sql] object DataTypeErrors extends DataTypeErrorsBase {
messageParameters = Map("operation" -> operation),
cause = null)
}

/*
def invalidInputInCastToDatetimeError(
value: Any,
from: DataType,
to: DataType,
context: SQLQueryContext): Throwable = {
new SparkDateTimeException(
errorClass = "CAST_INVALID_INPUT",
messageParameters = Map(
"expression" -> toSQLValue(value, from),
"sourceType" -> toSQLType(from),
"targetType" -> toSQLType(to),
"ansiConfig" -> toSQLConf(Sql.ANSI_ENABLED.key)),
context = getQueryContext(context),
summary = getSummary(context))
}
*/

def unsupportedDataTypeError(typeName: DataType): SparkUnsupportedOperationException = {
new SparkUnsupportedOperationException(
errorClass = "UNSUPPORTED_DATATYPE",
messageParameters = Map("typeName" -> toSQLType(typeName)))
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ import org.apache.spark.QueryContext
import org.apache.spark.sql.catalyst.trees.SQLQueryContext
import org.apache.spark.sql.catalyst.util.{AttributeNameParser, QuotingUtils, SparkStringUtils}
import org.apache.spark.sql.types.{AbstractDataType, DataType, TypeCollection}
import org.apache.spark.unsafe.types.UTF8String

private[sql] trait DataTypeErrorsBase {
def toSQLId(parts: String): String = {
Expand Down Expand Up @@ -54,14 +55,36 @@ private[sql] trait DataTypeErrorsBase {
case at => quoteByDefault(at.simpleString.toUpperCase(Locale.ROOT))
}

def dataTypeToSQLValue(value: String): String = {
def toSQLValue(value: String): String = {
if (value == null) {
"NULL"
} else {
"'" + value.replace("\\", "\\\\").replace("'", "\\'") + "'"
}
}

def toSQLValue(value: UTF8String): String = toSQLValue(value.toString)

def toSQLValue(value: Short): String = String.valueOf(value) + "S"

def toSQLValue(value: Int): String = String.valueOf(value)

def toSQLValue(value: Long): String = String.valueOf(value) + "L"

def toSQLValue(value: Float): String = {
if (value.isNaN) "NaN"
else if (value.isPosInfinity) "Infinity"
else if (value.isNegInfinity) "-Infinity"
else value.toString
}

def toSQLValue(value: Double): String = {
if (value.isNaN) "NaN"
else if (value.isPosInfinity) "Infinity"
else if (value.isNegInfinity) "-Infinity"
else value.toString
}

protected def quoteByDefault(elem: String): String = {
"\"" + elem + "\""
}
Expand Down
Loading