Skip to content

[SPARK-8848] [SQL] [WIP] Refactors Parquet write path to follow Parquet format spec #7679

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
liancheng wants to merge 21 commits into from
Closed

[SPARK-8848] [SQL] [WIP] Refactors Parquet write path to follow Parquet format spec #7679

Changes from 1 commit
Commits
Show all changes
21 commits
Select commit Hold shift + click to select a range
e9305bd
Refactors Parquet write support to follow Parquet format spec
liancheng Jul 26, 2015
b465661
Cleans up code only used by old RowWriteSupport
liancheng Jul 26, 2015
821e9ec
Fixes test failures
liancheng Jul 27, 2015
2a1e884
Fixes writing UDT
liancheng Jul 27, 2015
e9638f0
Optimizes writing structs
liancheng Jul 27, 2015
a2aeba5
Fixes writing empty arrays and maps
liancheng Jul 27, 2015
678ccd4
Migrates large decimal precision support
liancheng Jul 28, 2015
b9f93db
Fixes writing negative decimal values
liancheng Jul 28, 2015
0e0a957
Minor comment updates
liancheng Jul 28, 2015
2859132
Fixes array type conversion in legacy mode
liancheng Jul 29, 2015
62c4829
Minor refactoring
liancheng Jul 29, 2015
b37fe77
Fixes compilation error introduced while rebasing
liancheng Jul 31, 2015
0d61b3b
Writes arrays using ArrayData
liancheng Jul 31, 2015
f901a16
Writes maps using MapData
liancheng Aug 1, 2015
5127b8d
Retrieves data from SpecializedGetters
liancheng Aug 1, 2015
1f1d4af
Renames "Catalyst*" classes to "Parquet*"
liancheng Aug 2, 2015
0395e95
Renames root Parquet message name
liancheng Aug 2, 2015
23e523d
Makes implicit arguments in ParquetSchemaSuite explicit
liancheng Aug 2, 2015
66419b7
Renames followParquetFormatSpec to writeLegacyParquetFormat (its mean…
liancheng Aug 2, 2015
6bda94b
Renames analysisRequire to checkConversionRequirement
liancheng Aug 2, 2015
679888a
Simplifies ParquetSchemaConverter and updates outdated comments
liancheng Aug 3, 2015
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Renames analysisRequire to checkConversionRequirement
  • Loading branch information
@liancheng
liancheng committed Aug 2, 2015
commit 6bda94ba32ebb61d089c5392f1afdbf5613b8549
18 changes: 9 additions & 9 deletions sql/core/src/main/scala/org/apache/spark/sql/parquet/ParquetSchemaConverter.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -134,7 +134,7 @@ private[parquet] class ParquetSchemaConverter(
val precision = field.getDecimalMetadata.getPrecision
val scale = field.getDecimalMetadata.getScale

ParquetSchemaConverter.analysisRequire(
ParquetSchemaConverter.checkConversionRequirement(
maxPrecision == -1 || 1 <= precision && precision <= maxPrecision,
s"Invalid decimal precision: $typeName cannot store $precision digits (max $maxPrecision)")

Expand Down Expand Up @@ -168,7 +168,7 @@ private[parquet] class ParquetSchemaConverter(
}

case INT96 =>
ParquetSchemaConverter.analysisRequire(
ParquetSchemaConverter.checkConversionRequirement(
assumeInt96IsTimestamp,
"INT96 is not supported unless it's interpreted as timestamp. " +
s"Please try to set ${SQLConf.PARQUET_INT96_AS_TIMESTAMP.key} to true.")
Expand Down Expand Up @@ -210,11 +210,11 @@ private[parquet] class ParquetSchemaConverter(
//
// See: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#lists
case LIST =>
ParquetSchemaConverter.analysisRequire(
ParquetSchemaConverter.checkConversionRequirement(
field.getFieldCount == 1, s"Invalid list type $field")

val repeatedType = field.getType(0)
ParquetSchemaConverter.analysisRequire(
ParquetSchemaConverter.checkConversionRequirement(
repeatedType.isRepetition(REPEATED), s"Invalid list type $field")

if (isElementType(repeatedType, field.getName)) {
Expand All @@ -230,17 +230,17 @@ private[parquet] class ParquetSchemaConverter(
// See: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#backward-compatibility-rules-1
// scalastyle:on
case MAP | MAP_KEY_VALUE =>
ParquetSchemaConverter.analysisRequire(
ParquetSchemaConverter.checkConversionRequirement(
field.getFieldCount == 1 && !field.getType(0).isPrimitive,
s"Invalid map type: $field")

val keyValueType = field.getType(0).asGroupType()
ParquetSchemaConverter.analysisRequire(
ParquetSchemaConverter.checkConversionRequirement(
keyValueType.isRepetition(REPEATED) && keyValueType.getFieldCount == 2,
s"Invalid map type: $field")

val keyType = keyValueType.getType(0)
ParquetSchemaConverter.analysisRequire(
ParquetSchemaConverter.checkConversionRequirement(
keyType.isPrimitive,
s"Map key type is expected to be a primitive type, but found: $keyType")

Expand Down Expand Up @@ -546,7 +546,7 @@ private[parquet] object ParquetSchemaConverter {

def checkFieldName(name: String): Unit = {
// ,;{}()\n\t= and space are special characters in Parquet schema
analysisRequire(
checkConversionRequirement(
!name.matches(".*[ ,;{}()\n\t=].*"),
s"""Attribute name "$name" contains invalid character(s) among " ,;{}()\\n\\t=".
|Please use alias to rename it.
Expand All @@ -558,7 +558,7 @@ private[parquet] object ParquetSchemaConverter {
schema
}

def analysisRequire(f: => Boolean, message: String): Unit = {
def checkConversionRequirement(f: => Boolean, message: String): Unit = {
if (!f) {
throw new AnalysisException(message)
}
Expand Down