Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
fix
  • Loading branch information
Zhenhua Wang committed Dec 28, 2017
commit 9617c2d982ed799580957a1467d47f42e8124636
Original file line number Diff line number Diff line change
Expand Up @@ -143,7 +143,12 @@ case class AnalyzeColumnCommand(
val percentilesRow = new QueryExecution(sparkSession, Aggregate(Nil, namedExprs, relation))
.executedPlan.executeTake(1).head
attrsToGenHistogram.zipWithIndex.foreach { case (attr, i) =>
attributePercentiles += attr -> percentilesRow.getArray(i)
val percentiles = percentilesRow.getArray(i)
// When there is no non-null value, `percentiles` is null. In such case, there is no
// need to generate histogram.
if (percentiles != null) {
attributePercentiles += attr -> percentiles
}
}
}
AttributeMap(attributePercentiles.toSeq)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -85,13 +85,24 @@ class StatisticsCollectionSuite extends StatisticsCollectionTestBase with Shared
test("analyze empty table") {
val table = "emptyTable"
withTable(table) {
sql(s"CREATE TABLE $table (key STRING, value STRING) USING PARQUET")
val df = Seq.empty[Int].toDF("key")
df.write.format("json").saveAsTable(table)
sql(s"ANALYZE TABLE $table COMPUTE STATISTICS noscan")
val fetchedStats1 = checkTableStats(table, hasSizeInBytes = true, expectedRowCounts = None)
assert(fetchedStats1.get.sizeInBytes == 0)
sql(s"ANALYZE TABLE $table COMPUTE STATISTICS")
val fetchedStats2 = checkTableStats(table, hasSizeInBytes = true, expectedRowCounts = Some(0))
assert(fetchedStats2.get.sizeInBytes == 0)

val expectedColStat =
"key" -> ColumnStat(0, None, None, 0, IntegerType.defaultSize, IntegerType.defaultSize)

// There won't be histogram for empty column.
Seq("true", "false").foreach { histogramEnabled =>
withSQLConf(SQLConf.HISTOGRAM_ENABLED.key -> histogramEnabled) {
checkColStats(df, mutable.LinkedHashMap(expectedColStat))
}
}
}
}

Expand Down Expand Up @@ -178,7 +189,13 @@ class StatisticsCollectionSuite extends StatisticsCollectionTestBase with Shared
val expectedColStats = dataTypes.map { case (tpe, idx) =>
(s"col$idx", ColumnStat(0, None, None, 1, tpe.defaultSize.toLong, tpe.defaultSize.toLong))
}
checkColStats(df, mutable.LinkedHashMap(expectedColStats: _*))

// There won't be histograms for null columns.
Seq("true", "false").foreach { histogramEnabled =>
withSQLConf(SQLConf.HISTOGRAM_ENABLED.key -> histogramEnabled) {
checkColStats(df, mutable.LinkedHashMap(expectedColStats: _*))
}
}
}

test("number format in statistics") {
Expand Down