-
Notifications
You must be signed in to change notification settings - Fork 29k
[SPARK-12957][SQL] Initial support for constraint propagation in SparkSQL #10844
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from 1 commit
8d050df
fed48b8
76d2727
67e138d
04ff99a
7bde51d
494b28f
0c4c78b
7fb2f9c
f15ef96
53be837
8c6bb70
302444f
b52742a
2bd2735
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
- Loading branch information
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -30,16 +30,16 @@ abstract class QueryPlan[PlanType <: TreeNode[PlanType]] | |
| /** | ||
| * Extracts the output property from a given child. | ||
| */ | ||
| def extractConstraintsFromChild(child: QueryPlan[PlanType]): Seq[Expression] = { | ||
| def extractConstraintsFromChild(child: QueryPlan[PlanType]): Set[Expression] = { | ||
| child.constraints.filter(_.references.subsetOf(outputSet)) | ||
| } | ||
|
|
||
| /** | ||
| * An sequence of expressions that describes the data property of the output rows of this | ||
| * operator. For example, if the output of this operator is column `a`, an example `constraints` | ||
| * can be `Seq(a > 10, a < 20)`. | ||
| * can be `Set(a > 10, a < 20)`. | ||
| */ | ||
| def constraints: Seq[Expression] = Nil | ||
| def constraints: Set[Expression] = Set.empty | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This is probably going to be nontrivial to calculate for a large tree. We might consider having an internal method, |
||
|
|
||
| /** | ||
| * Returns the set of attributes that are output by this node. | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -89,9 +89,9 @@ case class Generate( | |
| case class Filter(condition: Expression, child: LogicalPlan) extends UnaryNode { | ||
| override def output: Seq[Attribute] = child.output | ||
|
|
||
| override def constraints: Seq[Expression] = { | ||
| override def constraints: Set[Expression] = { | ||
| val newConstraint = splitConjunctivePredicates(condition).filter( | ||
| _.references.subsetOf(outputSet)) | ||
| _.references.subsetOf(outputSet)).toSet | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. style nit: we typically avoid breaking in the middle of a function call and instead prefer to break in between calls (always pick the highest syntactic level) val newConstraint = splitConjunctivePredicates(condition)
.filter(_.references.subsetOf(outputSet))
.toSet |
||
| newConstraint.union(extractConstraintsFromChild(child)) | ||
| } | ||
| } | ||
|
|
@@ -103,9 +103,9 @@ abstract class SetOperation(left: LogicalPlan, right: LogicalPlan) extends Binar | |
| leftAttr.withNullability(leftAttr.nullable || rightAttr.nullable) | ||
| } | ||
|
|
||
| protected def leftConstraints: Seq[Expression] = extractConstraintsFromChild(left) | ||
| protected def leftConstraints: Set[Expression] = extractConstraintsFromChild(left) | ||
|
|
||
| protected def rightConstraints: Seq[Expression] = { | ||
| protected def rightConstraints: Set[Expression] = { | ||
| require(left.output.size == right.output.size) | ||
| val attributeRewrites = AttributeMap(left.output.zip(right.output)) | ||
| extractConstraintsFromChild(right).map(_ transform { | ||
|
|
@@ -135,7 +135,7 @@ case class Union(left: LogicalPlan, right: LogicalPlan) extends SetOperation(lef | |
| Statistics(sizeInBytes = sizeInBytes) | ||
| } | ||
|
|
||
| override def constraints: Seq[Expression] = { | ||
| override def constraints: Set[Expression] = { | ||
| leftConstraints.intersect(rightConstraints) | ||
| } | ||
| } | ||
|
|
@@ -147,7 +147,7 @@ case class Intersect(left: LogicalPlan, right: LogicalPlan) extends SetOperation | |
| leftAttr.withNullability(leftAttr.nullable && rightAttr.nullable) | ||
| } | ||
|
|
||
| override def constraints: Seq[Expression] = { | ||
| override def constraints: Set[Expression] = { | ||
| leftConstraints.union(rightConstraints) | ||
| } | ||
| } | ||
|
|
@@ -156,7 +156,7 @@ case class Except(left: LogicalPlan, right: LogicalPlan) extends SetOperation(le | |
| /** We don't use right.output because those rows get excluded from the set. */ | ||
| override def output: Seq[Attribute] = left.output | ||
|
|
||
| override def constraints: Seq[Expression] = leftConstraints | ||
| override def constraints: Set[Expression] = leftConstraints | ||
| } | ||
|
|
||
| case class Join( | ||
|
|
@@ -180,7 +180,7 @@ case class Join( | |
| } | ||
| } | ||
|
|
||
| override def constraints: Seq[Expression] = { | ||
| override def constraints: Set[Expression] = { | ||
| joinType match { | ||
| case LeftSemi => | ||
| extractConstraintsFromChild(left) | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
protected?Also I'm not sure I get the scala doc. Maybe
getReleventContraintsis a better name? It is taking the constraints and removing those that don't apply anymore because we removed columns right?