Skip to content
Closed
Changes from 1 commit
Commits
Show all changes
34 commits
Select commit Hold shift + click to select a range
b988651
[SPARK-16804][SQL] Correlated subqueries containing LIMIT return inco…
nsyca Jul 29, 2016
069ed8f
[SPARK-16804][SQL] Correlated subqueries containing LIMIT return inco…
nsyca Jul 29, 2016
edca333
New positive test cases
nsyca Jul 30, 2016
64184fd
Fix unit test case failure
nsyca Aug 1, 2016
29f82b0
blocking TABLESAMPLE
nsyca Aug 5, 2016
ac43ab4
Fixing code styling
nsyca Aug 5, 2016
631d396
Correcting Scala test style
nsyca Aug 7, 2016
7eb9b2d
One (last) attempt to correct the Scala style tests
nsyca Aug 8, 2016
1387cf5
Merge remote-tracking branch 'upstream/master'
nsyca Aug 12, 2016
6d9bade
Merge remote-tracking branch 'upstream/master'
nsyca Nov 4, 2016
9a1f80b
Merge remote-tracking branch 'upstream/master'
nsyca Nov 4, 2016
3fe9429
Merge remote-tracking branch 'upstream/master'
nsyca Nov 5, 2016
0757b81
Merge remote-tracking branch 'upstream/master'
nsyca Nov 11, 2016
35b77f0
Merge remote-tracking branch 'upstream/master'
nsyca Nov 12, 2016
c63b8c6
Merge remote-tracking branch 'upstream/master'
nsyca Nov 14, 2016
f3351d5
Merge remote-tracking branch 'upstream/master'
nsyca Nov 18, 2016
9fc5c33
Merge remote-tracking branch 'upstream/master'
nsyca Nov 18, 2016
402e1d9
Merge remote-tracking branch 'upstream/master'
nsyca Nov 22, 2016
b117281
Merge remote-tracking branch 'upstream/master'
nsyca Nov 23, 2016
3023399
Merge remote-tracking branch 'upstream/master'
nsyca Nov 24, 2016
4b692f0
Merge remote-tracking branch 'upstream/master'
nsyca Nov 25, 2016
0d64512
working code #1
nsyca Nov 28, 2016
c8aadb5
Merge remote-tracking branch 'upstream/master'
nsyca Nov 28, 2016
3f184ea
Merge branch 'master' into spark18455.0
nsyca Nov 28, 2016
23e357c
Make the code more concise
nsyca Nov 28, 2016
d60f0de
Fix stylecheck failure
nsyca Nov 28, 2016
2181647
Merge remote-tracking branch 'upstream/master'
nsyca Nov 28, 2016
599f54b
Merge branch 'master' into spark18455.0
nsyca Nov 28, 2016
ca9e1a8
Cosmetic code changes
nsyca Nov 28, 2016
3f4c62a
Address review comment #1
nsyca Nov 30, 2016
c8588de
Merge remote-tracking branch 'upstream/master'
nsyca Nov 30, 2016
05fd7a3
Merge branch 'master' into spark18455.0
nsyca Nov 30, 2016
1d32958
Remove the extra space
nsyca Nov 30, 2016
0c9d0b5
Move LeftSemi to be the same group as LeftOuter
nsyca Dec 1, 2016
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Address review comment #1
  • Loading branch information
nsyca committed Nov 30, 2016
commit 3f4c62afb62f4d922ec0ae037b331e0de8de9193
Original file line number Diff line number Diff line change
Expand Up @@ -1013,7 +1013,7 @@ class Analyzer(

// Make sure a plan's subtree does not contain outer references
def failOnOuterReferenceInSubTree(p: LogicalPlan): Unit = {
if (p.collect(predicateMap).nonEmpty) {
if (p.collectFirst(predicateMap).nonEmpty) {
failAnalysis(s"Accessing outer query column is not allowed in:\n$p")
}
}
Expand Down Expand Up @@ -1081,7 +1081,8 @@ class Analyzer(
// Whitelist operators allowed in a correlated subquery
// There are 4 categories:
// 1. Operators that are allowed anywhere in a correlated subquery, and,
// by definition of the operators, they cannot host outer references.
// by definition of the operators, they either do not contain
// any columns or cannot host outer references.
// 2. Operators that are allowed anywhere in a correlated subquery
// so long as they do not host outer references.
// 3. Operators that need special handlings. These operators are
Expand All @@ -1096,29 +1097,25 @@ class Analyzer(
// up to the operator producing the correlated values.

// Category 1:
// Leaf node can be anywhere in a correlated subquery.
case n: LeafNode =>
n
// Category 2:
// These operators can be anywhere in a correlated subquery.
// so long as they do not host outer references in the operators.
// SubqueryAlias can be anywhere in a correlated subquery.
case p: SubqueryAlias =>
failOnOuterReference(p)
// BroadcastHint, Distinct, LeafNode, Repartition, and SubqueryAlias
case p: BroadcastHint =>
p
case p: Distinct =>
failOnOuterReference(p)
p
case p: Sort =>
failOnOuterReference(p)
case p: LeafNode =>
p
case p: Repartition =>
failOnOuterReference(p)
p
case p: RedistributeData =>
case p: SubqueryAlias =>
p

// Category 2:
// These operators can be anywhere in a correlated subquery.
// so long as they do not host outer references in the operators.
case p: Sort =>
failOnOuterReference(p)
p
case p: BroadcastHint =>
case p: RedistributeData =>
failOnOuterReference(p)
p

Expand Down Expand Up @@ -1151,6 +1148,7 @@ class Analyzer(
// but can be anywhere in a correlated subquery.
case p @ Project(expressions, child) =>
failOnOuterReference(p)

val referencesToAdd = missingReferences(p)
if (referencesToAdd.nonEmpty) {
Project(expressions ++ referencesToAdd, child)
Expand All @@ -1159,7 +1157,7 @@ class Analyzer(
}

// Aggregate cannot host any correlated expressions
// It can be on a correlation path if the correlation has
// It can be on a correlation path if the correlation contains
// only equality correlated predicates.
// It cannot be on a correlation path if the correlation has
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nit: has -> contains?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Will change.

// non-equality correlated predicates.
Expand Down Expand Up @@ -1209,10 +1207,8 @@ class Analyzer(
// but must not host any outer references.
// Note:
// Generator with join=false is treated as Category 4.
case p @ Generate(generator, join, _, _, _, _) if (join) =>
if (containsOuter(generator)) {
failOnOuterReference(p)
}
case p @ Generate(generator, true, _, _, _, _) =>
failOnOuterReference(p)
p

// Category 4: Any other operators not in the above 3 categories
Expand Down