Skip to content
Closed
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
52 commits
Select commit Hold shift + click to select a range
2ca2c38
init commit
lianhuiwang Jun 3, 2016
edea710
fix unit test
lianhuiwang Jun 3, 2016
8426522
Merge branch 'apache-master' into metadata-only
lianhuiwang Jun 3, 2016
153293e
fix unit test
lianhuiwang Jun 3, 2016
7dfb743
update
lianhuiwang Jun 24, 2016
68e6d6d
Revert "fix unit test"
lianhuiwang Jun 24, 2016
595ef36
Revert "fix unit test"
lianhuiwang Jun 24, 2016
7d7ece0
Merge branch 'apache-master' into metadata-only
lianhuiwang Jun 24, 2016
2e55a9d
Merge branch 'apache-master' into metadata-only
lianhuiwang Jun 24, 2016
b2b6eba
update
lianhuiwang Jun 24, 2016
c5a291e
Merge branch 'apache-master' into metadata-only
lianhuiwang Jun 24, 2016
6404c1f
update opt for core
lianhuiwang Jun 24, 2016
1bb5812
refactor
lianhuiwang Jun 24, 2016
7e3729e
add ut
lianhuiwang Jun 24, 2016
fbf5d61
fix ut
lianhuiwang Jun 24, 2016
3411fd6
fix project
lianhuiwang Jun 26, 2016
aefab7f
address comments
lianhuiwang Jun 27, 2016
c5ccdea
fix cube/rollup
lianhuiwang Jun 27, 2016
ae6cf9f
fix style
lianhuiwang Jun 27, 2016
159331b
refactor
lianhuiwang Jun 27, 2016
3a1438b
refactor
lianhuiwang Jun 28, 2016
c0a7d59
update
lianhuiwang Jun 29, 2016
a4045ca
add comments
lianhuiwang Jun 29, 2016
0a023e7
fix minor
lianhuiwang Jun 29, 2016
a9b38ab
rename
lianhuiwang Jun 29, 2016
a5ea995
update
lianhuiwang Jun 29, 2016
1bed08d
fix monir
lianhuiwang Jun 29, 2016
a22e962
refactor
lianhuiwang Jul 1, 2016
41fef2c
update
lianhuiwang Jul 1, 2016
bd53678
Merge branch 'apache-master' into metadata-only
lianhuiwang Jul 1, 2016
88f7308
update
lianhuiwang Jul 1, 2016
2568193
add ut
lianhuiwang Jul 1, 2016
26a97f4
address comments
lianhuiwang Jul 4, 2016
4297f9f
update name
lianhuiwang Jul 6, 2016
1a65aa7
address comments
lianhuiwang Jul 6, 2016
d5e0df4
update
lianhuiwang Jul 6, 2016
9d6dd76
update2
lianhuiwang Jul 6, 2016
9cb01d8
update
lianhuiwang Jul 6, 2016
3e2687d
doc improve
cloud-fan Jul 6, 2016
2b4faf3
update
cloud-fan Jul 7, 2016
88fd3bf
Merge pull request #2 from cloud-fan/metadata-only
lianhuiwang Jul 7, 2016
a894bb7
delete cases
lianhuiwang Jul 7, 2016
9546b40
Merge branch 'metadata-only' of https://github.com/lianhuiwang/spark …
lianhuiwang Jul 7, 2016
85b695b
update ut
lianhuiwang Jul 7, 2016
bcfe8e5
Merge branch 'master' of https://github.com/apache/spark into metadat…
lianhuiwang Jul 7, 2016
67211be
Merge branch 'master' of https://github.com/apache/spark into metadat…
lianhuiwang Jul 7, 2016
501f93b
address commetns
lianhuiwang Jul 11, 2016
8ee2a8c
refactor
lianhuiwang Jul 11, 2016
d888c85
fix minor
lianhuiwang Jul 11, 2016
ff16509
update
lianhuiwang Jul 12, 2016
358ad13
remove duplicate code
lianhuiwang Jul 12, 2016
030776a
fix minor
lianhuiwang Jul 12, 2016
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
address comments
  • Loading branch information
lianhuiwang committed Jun 27, 2016
commit aefab7fa2b2c99f861faf0693fc1e13c2cf5311c
Original file line number Diff line number Diff line change
Expand Up @@ -42,9 +42,9 @@ case class MetadataOnlyOptimizer(
}
}.distinct
if (aggregateExpressions.isEmpty) {
// Cannot support for aggregate that has no aggregateFunction.
// example: select col1 from table group by col1.
false
// Support for aggregate that has no aggregateFunction when expressions are partition columns
// example: select partitionCol from table group by partitionCol.
true
} else {
aggregateExpressions.forall { agg =>
if (agg.isDistinct) {
Expand Down Expand Up @@ -119,13 +119,11 @@ case class MetadataOnlyOptimizer(
val projectSet = parent.references ++ AttributeSet(filterColumns)
if (projectSet.subsetOf(AttributeSet(partitionColumns))) {
val selectedPartitions = files.location.listFiles(filters)
val partitionValues = selectedPartitions.map(_.values)
val valuesRdd = sparkSession.sparkContext.parallelize(partitionValues, 1)
val valuesRdd = sparkSession.sparkContext.parallelize(selectedPartitions.map(_.values), 1)
val valuesPlan = LogicalRDD(partitionColumns, valuesRdd)(sparkSession)
if (projectList.nonEmpty) {
parent.withNewChildren(Project(projectList, valuesPlan) :: Nil)
} else {
parent.withNewChildren(valuesPlan :: Nil)
parent.transform {
case l @ LogicalRelation(files: HadoopFsRelation, _, _) =>
valuesPlan
}
} else {
parent
Expand All @@ -150,12 +148,9 @@ case class MetadataOnlyOptimizer(
}
val valuesRdd = sparkSession.sparkContext.parallelize(partitionValues, 1)
val valuesPlan = LogicalRDD(partitionColumns, valuesRdd)(sparkSession)
val filterPlan =
filters.reduceLeftOption(And).map(Filter(_, valuesPlan)).getOrElse(valuesPlan)
if (projectList.nonEmpty) {
parent.withNewChildren(Project(projectList, filterPlan) :: Nil)
} else {
parent.withNewChildren(filterPlan :: Nil)
parent.transform {
case relation: CatalogRelation =>
valuesPlan
}
} else {
parent
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1701,6 +1701,7 @@ class SQLQuerySuite extends QueryTest with SQLTestUtils with TestHiveSingleton {
|select key, value from data_15752
""".stripMargin)
}
checkAnswer(sql("select hr from srcpart_15752 where hr = 11 group by hr"), Row(11))
checkAnswer(sql("select max(hr) from srcpart_15752"), Row(12))
checkAnswer(sql("select max(hr) from srcpart_15752 where hr = 11"), Row(11))
checkAnswer(sql("select max(hr) from (select hr from srcpart_15752) t"), Row(12))
Expand All @@ -1720,6 +1721,7 @@ class SQLQuerySuite extends QueryTest with SQLTestUtils with TestHiveSingleton {
""".stripMargin)
}

checkAnswer(sql("select hr from srctext_15752 where hr = 11 group by hr"), Row(11))
checkAnswer(sql("select max(hr) from srctext_15752"), Row(12))
checkAnswer(sql("select max(hr) from srctext_15752 where hr = 11"), Row(11))
checkAnswer(sql("select max(hr) from (select hr from srctext_15752) t"), Row(12))
Expand Down