Skip to content
Closed
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
65 commits
Select commit Hold shift + click to select a range
0860f9d
introduce table constraint
gengliangwang Feb 20, 2025
0b222ba
add AlterTableAddConstraintParseSuite and AlterTableDropConstraintPar…
gengliangwang Feb 20, 2025
40d27ae
update error conditions and add tests
gengliangwang Feb 21, 2025
f244c75
rename logical plans
gengliangwang Feb 21, 2025
b60e100
add methods in CatalogV2Util and tests
gengliangwang Feb 22, 2025
d2b7f0a
add DropConstraintSuite
gengliangwang Feb 24, 2025
8264d5a
refactor parse changes
gengliangwang Feb 24, 2025
5c57ce2
fix visitCreateTableClauses
gengliangwang Feb 25, 2025
c3a7c41
rename as AddCheckConstraint
gengliangwang Feb 25, 2025
90e1404
save for now
gengliangwang Feb 27, 2025
6381a24
refactor check constraint
gengliangwang Mar 4, 2025
5eaa069
rename AddConstraint
gengliangwang Mar 4, 2025
a63c1d9
introduce CheckConstraint expr
gengliangwang Mar 4, 2025
0063155
introduce Constraints expression
gengliangwang Mar 4, 2025
af2fa61
add CreateTableConstraintParseSuite
gengliangwang Mar 5, 2025
5d86c93
add new create table api
gengliangwang Mar 5, 2025
a8972ec
add CreateTableConstraintSuite
gengliangwang Mar 5, 2025
ea23855
fix CreateTableConstraintSuite
gengliangwang Mar 5, 2025
3944f60
resolve constraints with a fake project
gengliangwang Mar 5, 2025
6b1a4af
resolve constraint with a default analyzer
gengliangwang Mar 6, 2025
46aa448
improve error message
gengliangwang Mar 6, 2025
18f7c65
move constraint class
gengliangwang Mar 20, 2025
3acca28
remove Constraint.java
gengliangwang Mar 20, 2025
a0ab768
[SPARK-51441][SQL] Add DSv2 APIs for constraints
aokolnychyi Mar 11, 2025
d029a99
Flatten the structure, add builders
aokolnychyi Mar 20, 2025
12fe567
fix compiling
gengliangwang Mar 20, 2025
6c30863
refactor parser and fix tests
gengliangwang Mar 22, 2025
cdf55b5
new syntax; compiling version
gengliangwang Mar 24, 2025
c80e898
getOriginalText
gengliangwang Mar 24, 2025
41dcc23
add more tests for characteristic
gengliangwang Mar 25, 2025
8bc6c32
add tests in CreateTableConstraintParseSuite
gengliangwang Mar 25, 2025
24e3bf7
add tests for ConstraintCharacteristics in CheckConstraintSuite
gengliangwang Mar 25, 2025
252332e
save for now
gengliangwang Mar 25, 2025
462de00
fix create table constraint syntax
gengliangwang Mar 25, 2025
60f2d0b
refactor syntax
gengliangwang Mar 26, 2025
ff3133a
revise syntax; fix test failures
gengliangwang Mar 26, 2025
7a33e57
refactor test code
gengliangwang Mar 26, 2025
1923e1b
remove Expression Constraints
gengliangwang Mar 27, 2025
3c6cac5
rename ConstraintExpression as TableConstraint
gengliangwang Mar 27, 2025
7aea90c
refactor syntax as per standard
gengliangwang Mar 27, 2025
78c3904
remove column check constraint test cases
gengliangwang Mar 27, 2025
20d7cf9
add pk&fk
gengliangwang Mar 27, 2025
af3fb91
parse column constraint
gengliangwang Mar 27, 2025
337c35f
support PK & Unique; add tests
gengliangwang Mar 27, 2025
c8edfd0
create table with FK
gengliangwang Mar 28, 2025
5317f28
refactor alter
gengliangwang Mar 28, 2025
9ada207
add test case for unique
gengliangwang Mar 28, 2025
3451452
add test case for fk
gengliangwang Mar 28, 2025
657fce1
remove legacy CreateTableConstraintSuite.scala
gengliangwang Mar 28, 2025
1b5cc2f
change default value of rely; add PrimaryKeyConstraintSuite
gengliangwang Mar 28, 2025
44bbd78
change check constraint to NORELY
gengliangwang Mar 28, 2025
87d2206
change the valid status of check
gengliangwang Mar 28, 2025
258d10d
disallow enforce in pk/fk/unique
gengliangwang Mar 31, 2025
05bff35
refactor tests
gengliangwang Mar 31, 2025
d1abaf3
add test for unique and fk
gengliangwang Mar 31, 2025
528b6ff
save for now
gengliangwang Apr 1, 2025
d14f5e5
support generated name
gengliangwang Apr 1, 2025
0b84a72
unnamed check constraint
gengliangwang Apr 1, 2025
d3f7df4
simplify test
gengliangwang Apr 1, 2025
7e8273e
support unnamed constraint in replace table;add test cases for replac…
gengliangwang Apr 1, 2025
5cc47a3
handle Nondeterministic check
gengliangwang Apr 1, 2025
675b573
add more tests in CheckConstraintParseSuite
gengliangwang Apr 1, 2025
e51cbd0
fix PrimaryKeyConstraintParseSuite
gengliangwang Apr 1, 2025
aead1b2
fix ForeignKeyConstraintParseSuite
gengliangwang Apr 1, 2025
dc24937
fix UniqueConstraintParseSuite
gengliangwang Apr 2, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
refactor check constraint
  • Loading branch information
gengliangwang committed Mar 4, 2025
commit 6381a2414b9a16599f141aba6ec70945034c5735
Original file line number Diff line number Diff line change
Expand Up @@ -25,17 +25,20 @@ public interface Constraint {
boolean rely(); // indicates whether the constraint is believed to be true
boolean enforced(); // indicates whether the constraint must be enforced

static Constraint check(String name, Predicate predicate) {
return new Check(name, predicate);
static Constraint check(String name, String sql, Predicate predicate) {
return new Check(name, sql, predicate);
}

final class Check implements Constraint {
private final String name;
private final String sql;
private final Predicate predicate;
private Check(String name, Predicate predicate) {
private Check(String name, String sql, Predicate predicate) {
this.name = name;
this.sql = sql;
this.predicate = predicate;
}

@Override public String name() {
return name;
}
Expand All @@ -46,7 +49,7 @@ private Check(String name, Predicate predicate) {

@Override
public String toDDL() {
return "CHECK (" + predicate.toString() + ")";
return "CHECK (" + sql + ")";
}

@Override public boolean rely() {
Expand All @@ -56,5 +59,9 @@ public String toDDL() {
@Override public boolean enforced() {
return true;
}

public String sql() { return sql; }

public Predicate predicate() { return predicate; }
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -1191,7 +1191,7 @@ trait CheckAnalysis extends LookupCatalog with QueryErrorsBase with PlanToString
case _ =>
}

case addConstraint @ AddCheckConstraint(table: ResolvedTable, _, constraintExpr) =>
case addConstraint @ AddCheckConstraint(table: ResolvedTable, _, _, constraintExpr) =>
if (!constraintExpr.resolved) {
constraintExpr.failAnalysis(
errorClass = "INVALID_CHECK_CONSTRAINT.UNRESOLVED",
Expand All @@ -1205,13 +1205,6 @@ trait CheckAnalysis extends LookupCatalog with QueryErrorsBase with PlanToString
messageParameters = Map.empty
)
}

if (addConstraint.predicate.isEmpty) {
constraintExpr.failAnalysis(
errorClass = "INVALID_CHECK_CONSTRAINT.INVALID_V2_PREDICATE",
messageParameters = Map.empty
)
}
case _ =>
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4642,10 +4642,10 @@ class AstBuilder extends DataTypeAstBuilder
}
}

val constraints = ctx.constraintSpec().asScala.map(visitConstraintSpec).toSeq
// val constraints = ctx.constraintSpec().asScala.map(visitConstraintSpec).toSeq

(partTransforms, partCols, bucketSpec, cleanedProperties, cleanedOptions, newLocation, comment,
collation, serdeInfo, clusterBySpec, constraints)
collation, serdeInfo, clusterBySpec, Seq.empty)
}

protected def getSerdeInfo(
Expand Down Expand Up @@ -5241,14 +5241,6 @@ class AstBuilder extends DataTypeAstBuilder
AlterTableCollation(table, visitCollationSpec(ctx.collationSpec()))
}

override def visitConstraintSpec(ctx: ConstraintSpecContext): Expression = {
ctx.constraintExpression() match {
case c: CheckConstraintContext => expression(c.booleanExpression())
case other =>
throw QueryParsingErrors.constraintNotSupportedError(ctx, other.getText)
}
}

/**
* Parse a [[AddCheckConstraint]] command.
*
Expand All @@ -5261,8 +5253,16 @@ class AstBuilder extends DataTypeAstBuilder
withOrigin(ctx) {
val table = createUnresolvedTable(
ctx.identifierReference, "ALTER TABLE ... ADD CONSTRAINT")
val constraintExpression = visitConstraintSpec(ctx.constraintSpec())
AddCheckConstraint(table, ctx.constraintSpec().constraintName.getText, constraintExpression)
ctx.constraintSpec.constraintExpression() match {
case c: CheckConstraintContext =>
AddCheckConstraint(
table,
ctx.constraintSpec().constraintName.getText,
c.booleanExpression().getText,
expression(c.booleanExpression()))
case other =>
throw QueryParsingErrors.constraintNotSupportedError(ctx, other.getText)
}
}


Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -295,12 +295,13 @@ case class AlterTableCollation(
case class AddCheckConstraint(
table: LogicalPlan,
name: String,
constraintText: String,
constraintExpr: Expression) extends AlterTableCommand {

lazy val predicate = new V2ExpressionBuilder(constraintExpr, true).buildPredicate()

override def changes: Seq[TableChange] = {
val constraint = Constraint.check(name, predicate.get)
val constraint = Constraint.check(name, constraintText, predicate.orNull)
Seq(TableChange.addCheckConstraint(constraint, constraint.enforced()))
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1521,7 +1521,8 @@ case class UnresolvedTableSpec(
collation: Option[String],
serde: Option[SerdeInfo],
external: Boolean,
constraints: Seq[Expression]) extends UnaryExpression with Unevaluable with TableSpecBase {
constraints: Seq[Expression] = Seq.empty)
extends UnaryExpression with Unevaluable with TableSpecBase {

override def dataType: DataType =
throw new SparkUnsupportedOperationException("_LEGACY_ERROR_TEMP_3113")
Expand Down Expand Up @@ -1568,7 +1569,7 @@ case class TableSpec(
collation: Option[String],
serde: Option[SerdeInfo],
external: Boolean,
constraints: Seq[Constraint]) extends TableSpecBase {
constraints: Seq[Constraint] = Seq.empty) extends TableSpecBase {
def withNewLocation(newLocation: Option[String]): TableSpec = {
TableSpec(properties, provider, options, newLocation,
comment, collation, serde, external, constraints)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ class AlterTableAddConstraintParseSuite extends AnalysisTest with SharedSparkSes
Seq("a", "b", "c"),
"ALTER TABLE ... ADD CONSTRAINT"),
"c1",
"d > 0",
GreaterThan(UnresolvedAttribute("d"), Literal(0)))
comparePlans(parsed, expected)
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ package org.apache.spark.sql.execution.command.v2

import org.apache.spark.sql.{AnalysisException, QueryTest}
import org.apache.spark.sql.connector.catalog.Constraint.Check
import org.apache.spark.sql.connector.catalog.Table
import org.apache.spark.sql.execution.command.DDLCommandTestUtils

class CheckConstraintSuite extends QueryTest with CommandSuiteBase with DDLCommandTestUtils {
Expand Down Expand Up @@ -73,27 +74,26 @@ class CheckConstraintSuite extends QueryTest with CommandSuiteBase with DDLComma
}
}

test("Can't convert expression to V2 predicate") {
withTable("t") {
sql("create table t(i string) using parquet")
val query =
"""
|ALTER TABLE t ADD CONSTRAINT c1 CHECK (from_json(i, 'a INT').a > 1)
|""".stripMargin
val error = intercept[AnalysisException] {
sql(query)
}
checkError(
exception = error,
condition = "INVALID_CHECK_CONSTRAINT.INVALID_V2_PREDICATE",
sqlState = "42621",
parameters = Map.empty,
context = ExpectedContext(
fragment = "from_json(i, 'a INT').a > 1",
start = 40,
stop = 66
)
)
private def getCheckConstraint(table: Table): Check = {
assert(table.constraints.length == 1)
assert(table.constraints.head.isInstanceOf[Check])
table.constraints.head.asInstanceOf[Check]
val constraint = table.constraints.head.asInstanceOf[Check]
assert(constraint.rely())
assert(constraint.enforced())
constraint
}

test("Predicate should be null if it can't be converted to V2 predicate") {
withNamespaceAndTable("ns", "tbl", catalog) { t =>
sql(s"CREATE TABLE $t (id bigint, j string) $defaultUsing")
sql(s"ALTER TABLE $t ADD CONSTRAINT c1 CHECK (from_json(j, 'a INT').a > 1)")
val table = loadTable(catalog, "ns", "tbl")
val constraint = getCheckConstraint(table)
assert(constraint.name() == "c1")
assert(constraint.toDDL == "CHECK (from_json(j,'a INT').a>1)")
assert(constraint.sql() == "from_json(j,'a INT').a>1")
assert(constraint.predicate() == null)
}
}

Expand All @@ -104,13 +104,9 @@ class CheckConstraintSuite extends QueryTest with CommandSuiteBase with DDLComma

sql(s"ALTER TABLE $t ADD CONSTRAINT c1 CHECK (id > 0)")
val table = loadTable(catalog, "ns", "tbl")
assert(table.constraints.length == 1)
assert(table.constraints.head.isInstanceOf[Check])
val constraint = table.constraints.head.asInstanceOf[Check]
val constraint = getCheckConstraint(table)
assert(constraint.name() == "c1")
assert(constraint.rely())
assert(constraint.enforced())
assert(constraint.toDDL == "CHECK (id > 0)")
assert(constraint.toDDL == "CHECK (id>0)")
}
}

Expand All @@ -123,13 +119,13 @@ class CheckConstraintSuite extends QueryTest with CommandSuiteBase with DDLComma
// Constraint names are case-insensitive
Seq("abc", "ABC").foreach { name =>
val error = intercept[AnalysisException] {
sql(s"ALTER TABLE $t ADD CONSTRAINT $name CHECK (id > 0)")
sql(s"ALTER TABLE $t ADD CONSTRAINT $name CHECK (id>0)")
}
checkError(
exception = error,
condition = "CONSTRAINT_ALREADY_EXISTS",
sqlState = "42710",
parameters = Map("constraintName" -> "abc", "oldConstraint" -> "CHECK (id > 0)")
parameters = Map("constraintName" -> "abc", "oldConstraint" -> "CHECK (id>0)")
)
}
}
Expand Down