Skip to content
Closed
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
fix test, address rest of the comments.
  • Loading branch information
jovanpavl-db committed Sep 24, 2024
commit 16c1674adf437ab4bf8016afe02164ca3d5031b4
Original file line number Diff line number Diff line change
Expand Up @@ -507,11 +507,18 @@ private static CollationSpecUTF8 fromCollationId(int collationId) {
CASE_SENSITIVITY_OFFSET, CASE_SENSITIVITY_MASK);
// Extract space trimming from collation ID.
int spaceTrimmingOrdinal = getSpaceTrimming(collationId).ordinal();
assert(checkCollationId(collationId));
return new CollationSpecUTF8(
CaseSensitivity.values()[caseConversionOrdinal],
SpaceTrimming.values()[spaceTrimmingOrdinal]);
}

private static boolean checkCollationId(int collationId) {
collationId = SpecifierUtils.removeSpec(collationId, SPACE_TRIMMING_OFFSET, SPACE_TRIMMING_MASK);
collationId = SpecifierUtils.removeSpec(collationId, CASE_SENSITIVITY_OFFSET, CASE_SENSITIVITY_MASK);
return collationId == 0;
}

@Override
protected Collation buildCollation() {
if (caseSensitivity == CaseSensitivity.UNSPECIFIED) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,10 @@ object CollateExpressionBuilder extends ExpressionBuilder {
if (evalCollation == null) {
throw QueryCompilationErrors.unexpectedNullError("collation", collationExpr)
} else {
if (!SQLConf.get.trimCollationEnabled &&
evalCollation.toString.toUpperCase().contains("TRIM")) {
throw QueryCompilationErrors.trimCollationNotEnabledError()
}
Collate(e, evalCollation.toString)
}
case (_: StringType, false) => throw QueryCompilationErrors.nonFoldableArgumentError(
Expand Down Expand Up @@ -86,8 +90,6 @@ case class Collate(child: Expression, collationName: String)
override def sql: String = s"$prettyName(${child.sql}, $collationName)"

override def toString: String = s"$prettyName($child, $collationName)"

val usesTrimCollation : Boolean = CollationFactory.usesTrimCollation(collationId)
}

// scalastyle:off line.contains.tab
Expand All @@ -114,7 +116,4 @@ case class Collation(child: Expression)
Literal.create(collationName, SQLConf.get.defaultStringType)
}
override def inputTypes: Seq[AbstractDataType] = Seq(StringTypeAnyCollation)

val usesTrimCollation: Boolean = CollationFactory.usesTrimCollation(
child.dataType.asInstanceOf[StringType].collationId)
}
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ import org.apache.spark.SparkIllegalArgumentException
import org.apache.spark.sql.{AnalysisException, SaveMode, SparkSession}
import org.apache.spark.sql.catalyst.analysis._
import org.apache.spark.sql.catalyst.catalog._
import org.apache.spark.sql.catalyst.expressions.{Collate, Collation, Expression, InputFileBlockLength, InputFileBlockStart, InputFileName, RowOrdering}
import org.apache.spark.sql.catalyst.expressions.{Expression, InputFileBlockLength, InputFileBlockStart, InputFileName, RowOrdering}
import org.apache.spark.sql.catalyst.plans.logical._
import org.apache.spark.sql.catalyst.rules.Rule
import org.apache.spark.sql.catalyst.types.DataTypeUtils.toAttributes
Expand All @@ -37,7 +37,6 @@ import org.apache.spark.sql.execution.command.DDLUtils
import org.apache.spark.sql.execution.command.ViewHelper.generateViewProperties
import org.apache.spark.sql.execution.datasources.{CreateTable => CreateTableV1}
import org.apache.spark.sql.execution.datasources.v2.FileDataSourceV2
import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.sources.InsertableRelation
import org.apache.spark.sql.types.{StructField, StructType}
import org.apache.spark.sql.util.PartitioningUtils.normalizePartitionSpec
Expand Down Expand Up @@ -641,26 +640,6 @@ case class QualifyLocationWithWarehouse(catalog: SessionCatalog) extends Rule[Lo
}
}

object TrimCollationCheck extends (LogicalPlan => Unit) {
def apply(plan: LogicalPlan): Unit = {
plan.foreach {
operator: LogicalPlan =>
operator.expressions.foreach(_.foreach(
e =>
if (isCollationExpressionAndUsesTrimCollation(e) && !SQLConf.get.trimCollationEnabled) {
throw QueryCompilationErrors.trimCollationNotEnabledError()
}
)
)
}
}

private def isCollationExpressionAndUsesTrimCollation(expression: Expression): Boolean = {
(expression.isInstanceOf[Collation] && expression.asInstanceOf[Collation].usesTrimCollation) ||
(expression.isInstanceOf[Collate] && expression.asInstanceOf[Collate].usesTrimCollation)
}
}

/**
* This rule checks for references to views WITH SCHEMA [TYPE] EVOLUTION and synchronizes the
* catalog if evolution was detected.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -225,7 +225,6 @@ abstract class BaseSessionStateBuilder(
HiveOnlyCheck +:
TableCapabilityCheck +:
CommandCheck +:
TrimCollationCheck +:
ViewSyncSchemaToMetaStore +:
customCheckRules
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -891,7 +891,10 @@ class QueryCompilationErrorsSuite
).foreach { sqlText =>
checkError(
exception = intercept[AnalysisException](sql(sqlText)),
condition = "UNSUPPORTED_FEATURE.TRIM_COLLATION")
condition = "UNSUPPORTED_FEATURE.TRIM_COLLATION",
parameters = Map.empty,
context = ExpectedContext(
fragment = sqlText.substring(7), start = 7, stop = sqlText.length - 1))
}
}
}
Expand Down