Skip to content
Closed
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
44 commits
Select commit Hold shift + click to select a range
1c5ab03
Only use metadata columns for resolution as last resort
karenfeng Feb 26, 2021
2fe733f
Resolve deduplicated common columns in NATURAL/USING JOIN with hidden…
karenfeng Feb 26, 2021
2c261bb
Merge branch 'master' of https://github.com/apache/spark into spark-3…
karenfeng Feb 26, 2021
80beda8
Fix behavior in Scala
karenfeng Feb 27, 2021
e1719d3
Fix nested expression
karenfeng Feb 27, 2021
6fa70ba
Merge branch 'master' of https://github.com/apache/spark into spark-3…
karenfeng Mar 2, 2021
0ba1916
Address comments
karenfeng Mar 4, 2021
2b7e730
Merge branch 'master' of https://github.com/apache/spark into spark-3…
karenfeng Mar 4, 2021
0c116a5
Push SQL output
karenfeng Mar 4, 2021
bf87f55
Re-do changes
karenfeng Mar 7, 2021
b5dc44f
Fixup tag
karenfeng Mar 8, 2021
6e32b3d
Merge branch 'master' of https://github.com/apache/spark into spark-3…
karenfeng Mar 8, 2021
7c3f5df
Add another failing test
karenfeng Mar 8, 2021
181751a
Merge add metadata and resolve missing references
karenfeng Mar 29, 2021
ad5e824
Resolve down to avoid prematurely projecting out
karenfeng Mar 29, 2021
e36e853
Remove printlns
karenfeng Mar 29, 2021
db44c53
Merge branch 'master' of https://github.com/apache/spark into spark-3…
karenfeng Mar 29, 2021
73b7c8a
Clean up
karenfeng Mar 29, 2021
1eb01e2
Style fixup
karenfeng Mar 29, 2021
9fd2490
Formatting fix
karenfeng Mar 29, 2021
f5cc3ae
Address comments
karenfeng Mar 30, 2021
fa7207e
Metadata output should be empty by default
karenfeng Mar 31, 2021
7af12ae
Clean up
karenfeng Mar 31, 2021
07f9ad5
Always resolve with metadata
karenfeng Mar 31, 2021
c474745
Merge branch 'master' of https://github.com/apache/spark into spark-3…
karenfeng Apr 6, 2021
0f267e7
Revert accidental metadata changes
karenfeng Apr 6, 2021
ed0270c
Revert DatasourceV2SQLSuite
karenfeng Apr 6, 2021
66ad572
Revert unneeded style changes
karenfeng Apr 6, 2021
fc3b16d
Revert accidental metadata output change
karenfeng Apr 6, 2021
f665030
Add childrens' hidden output to Project metadataOutput
karenfeng Apr 7, 2021
85b81b1
Retrigger tests
karenfeng Apr 7, 2021
eab7964
Add comments
karenfeng Apr 7, 2021
44ee9f8
Merge branch 'master' of https://github.com/apache/spark into spark-3…
karenfeng Apr 7, 2021
c84f396
Address comments and fix propagation through Projects
karenfeng Apr 7, 2021
0fe04a2
Retrigger tests
karenfeng Apr 7, 2021
c7c3df6
Retrigger tests
karenfeng Apr 7, 2021
b1bf28d
Retrigger tests
karenfeng Apr 8, 2021
47be66d
Wording
karenfeng Apr 9, 2021
8c5144e
Merge branch 'master' of https://github.com/apache/spark into spark-3…
karenfeng Apr 9, 2021
333a815
propagate hidden columns from nested NATURAL/USING JOINs
karenfeng Apr 12, 2021
49de5c5
Merge branch 'master' of https://github.com/apache/spark into spark-3…
karenfeng Apr 12, 2021
9e62d7d
Retrigger tests
karenfeng Apr 12, 2021
446d4bc
address comments
karenfeng Apr 13, 2021
8f70c2d
Merge branch 'master' of https://github.com/apache/spark into spark-3…
karenfeng Apr 13, 2021
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Revert accidental metadata changes
Signed-off-by: Karen Feng <[email protected]>
  • Loading branch information
karenfeng committed Apr 6, 2021
commit 0f267e797b53202ef25d360d50b2f09624c5dd73
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,4 @@ case class EventTimeWatermark(
a
}
}

override val metadataOutput: Seq[Attribute] = child.metadataOutput
}
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,6 @@ import org.apache.spark.util.random.RandomSampler
case class ReturnAnswer(child: LogicalPlan) extends UnaryNode {
override def maxRows: Option[Long] = child.maxRows
override def output: Seq[Attribute] = child.output
override def metadataOutput: Seq[Attribute] = child.metadataOutput
}

/**
Expand All @@ -53,7 +52,6 @@ case class ReturnAnswer(child: LogicalPlan) extends UnaryNode {
*/
case class Subquery(child: LogicalPlan, correlated: Boolean) extends OrderPreservingUnaryNode {
override def output: Seq[Attribute] = child.output
override def metadataOutput: Seq[Attribute] = child.metadataOutput
}

object Subquery {
Expand Down Expand Up @@ -118,8 +116,6 @@ case class Generate(
child: LogicalPlan)
extends UnaryNode {

val unrequiredSet: Set[Int] = unrequiredChildIndex.toSet

lazy val requiredChildOutput: Seq[Attribute] = {
val unrequiredSet = unrequiredChildIndex.toSet
child.output.zipWithIndex.filterNot(t => unrequiredSet.contains(t._2)).map(_._1)
Expand Down Expand Up @@ -147,13 +143,11 @@ case class Generate(
}

def output: Seq[Attribute] = requiredChildOutput ++ qualifiedGeneratorOutput
override def metadataOutput: Seq[Attribute] = child.metadataOutput
}

case class Filter(condition: Expression, child: LogicalPlan)
extends OrderPreservingUnaryNode with PredicateHelper {
override def output: Seq[Attribute] = child.output
override def metadataOutput: Seq[Attribute] = child.metadataOutput

override def maxRows: Option[Long] = child.maxRows

Expand Down Expand Up @@ -223,7 +217,6 @@ case class Except(
override def nodeName: String = getClass.getSimpleName + ( if ( isAll ) "All" else "" )
/** We don't use right.output because those rows get excluded from the set. */
override def output: Seq[Attribute] = left.output
override def metadataOutput: Seq[Attribute] = left.metadataOutput

override def metadataOutput: Seq[Attribute] = Nil

Expand Down Expand Up @@ -556,8 +549,6 @@ object View {
case class With(child: LogicalPlan, cteRelations: Seq[(String, SubqueryAlias)]) extends UnaryNode {
override def output: Seq[Attribute] = child.output

override def metadataOutput: Seq[Attribute] = child.metadataOutput

override def simpleString(maxFields: Int): String = {
val cteAliases = truncatedString(cteRelations.map(_._1), "[", ", ", "]", maxFields)
s"CTE $cteAliases"
Expand All @@ -570,7 +561,6 @@ case class WithWindowDefinition(
windowDefinitions: Map[String, WindowSpecDefinition],
child: LogicalPlan) extends UnaryNode {
override def output: Seq[Attribute] = child.output
override def metadataOutput: Seq[Attribute] = child.metadataOutput
}

/**
Expand All @@ -584,7 +574,6 @@ case class Sort(
global: Boolean,
child: LogicalPlan) extends UnaryNode {
override def output: Seq[Attribute] = child.output
override def metadataOutput: Seq[Attribute] = child.metadataOutput
override def maxRows: Option[Long] = child.maxRows
override def outputOrdering: Seq[SortOrder] = order
}
Expand Down Expand Up @@ -710,7 +699,6 @@ case class Window(
override def maxRows: Option[Long] = child.maxRows
override def output: Seq[Attribute] =
child.output ++ windowExpressions.map(_.toAttribute)
override def metadataOutput: Seq[Attribute] = child.metadataOutput

override def producedAttributes: AttributeSet = windowOutputSet

Expand Down Expand Up @@ -906,7 +894,6 @@ object Limit {
*/
case class GlobalLimit(limitExpr: Expression, child: LogicalPlan) extends OrderPreservingUnaryNode {
override def output: Seq[Attribute] = child.output
override def metadataOutput: Seq[Attribute] = child.metadataOutput
override def maxRows: Option[Long] = {
limitExpr match {
case IntegerLiteral(limit) => Some(limit)
Expand All @@ -923,7 +910,6 @@ case class GlobalLimit(limitExpr: Expression, child: LogicalPlan) extends OrderP
*/
case class LocalLimit(limitExpr: Expression, child: LogicalPlan) extends OrderPreservingUnaryNode {
override def output: Seq[Attribute] = child.output
override def metadataOutput: Seq[Attribute] = child.metadataOutput

override def maxRowsPerPartition: Option[Long] = {
limitExpr match {
Expand All @@ -945,7 +931,6 @@ case class LocalLimit(limitExpr: Expression, child: LogicalPlan) extends OrderPr
*/
case class Tail(limitExpr: Expression, child: LogicalPlan) extends OrderPreservingUnaryNode {
override def output: Seq[Attribute] = child.output
override def metadataOutput: Seq[Attribute] = child.metadataOutput
override def maxRows: Option[Long] = {
limitExpr match {
case IntegerLiteral(limit) => Some(limit)
Expand Down Expand Up @@ -1031,7 +1016,6 @@ case class Sample(

override def maxRows: Option[Long] = child.maxRows
override def output: Seq[Attribute] = child.output
override def metadataOutput: Seq[Attribute] = child.metadataOutput
}

/**
Expand All @@ -1040,7 +1024,6 @@ case class Sample(
case class Distinct(child: LogicalPlan) extends UnaryNode {
override def maxRows: Option[Long] = child.maxRows
override def output: Seq[Attribute] = child.output
override def metadataOutput: Seq[Attribute] = child.metadataOutput
}

/**
Expand Down Expand Up @@ -1145,7 +1128,6 @@ case class Deduplicate(
child: LogicalPlan) extends UnaryNode {
override def maxRows: Option[Long] = child.maxRows
override def output: Seq[Attribute] = child.output
override def metadataOutput: Seq[Attribute] = child.metadataOutput
}

/**
Expand Down Expand Up @@ -1174,5 +1156,4 @@ case class CollectMetrics(
}

override def output: Seq[Attribute] = child.output
override def metadataOutput: Seq[Attribute] = child.metadataOutput
}
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,6 @@ case class UnresolvedHint(name: String, parameters: Seq[Any], child: LogicalPlan

override lazy val resolved: Boolean = false
override def output: Seq[Attribute] = child.output
override def metadataOutput: Seq[Attribute] = child.metadataOutput
}

/**
Expand All @@ -43,8 +42,6 @@ case class ResolvedHint(child: LogicalPlan, hints: HintInfo = HintInfo())

override def output: Seq[Attribute] = child.output

override def metadataOutput: Seq[Attribute] = child.metadataOutput

override def doCanonicalize(): LogicalPlan = child.canonicalized
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -238,8 +238,6 @@ case class TypedFilter(

override def output: Seq[Attribute] = child.output

override def metadataOutput: Seq[Attribute] = child.metadataOutput

def withObjectProducerChild(obj: LogicalPlan): Filter = {
assert(obj.output.length == 1)
Filter(typedCondition(obj.output.head), obj)
Expand Down Expand Up @@ -335,8 +333,6 @@ case class AppendColumns(

override def output: Seq[Attribute] = child.output ++ newColumns

override def metadataOutput: Seq[Attribute] = child.metadataOutput

def newColumns: Seq[Attribute] = serializer.map(_.toAttribute)
}

Expand All @@ -350,8 +346,6 @@ case class AppendColumnsWithObject(
child: LogicalPlan) extends ObjectConsumer {

override def output: Seq[Attribute] = (childSerializer ++ newColumnsSerializer).map(_.toAttribute)

override def metadataOutput: Seq[Attribute] = child.metadataOutput
}

/** Factory for constructing new `MapGroups` nodes. */
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -80,8 +80,6 @@ trait BaseEvalPython extends UnaryNode {

override def output: Seq[Attribute] = child.output ++ resultAttrs

override def metadataOutput: Seq[Attribute] = child.metadataOutput

override def producedAttributes: AttributeSet = AttributeSet(resultAttrs)
}

Expand Down