Skip to content

[SPARK-10113][SQL] Explicit error message for unsigned Parquet logical types #9646

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
HyukjinKwon wants to merge 2 commits into from
Closed

[SPARK-10113][SQL] Explicit error message for unsigned Parquet logical types #9646

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
49e1624
[SPARK-10113][SQL] Support for unsigned Parquet logical types
HyukjinKwon Nov 12, 2015
2752c14
[SPARK-10113][SQL] Add a variable for readability
HyukjinKwon Nov 12, 2015
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Next Next commit
[SPARK-10113][SQL] Support for unsigned Parquet logical types
  • Loading branch information
@HyukjinKwon
HyukjinKwon committed Nov 12, 2015
commit 49e162480e33a0379e1b04979666533e66e33f6e
7 changes: 7 additions & 0 deletions ...in/scala/org/apache/spark/sql/execution/datasources/parquet/CatalystSchemaConverter.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,9 @@ private[parquet] class CatalystSchemaConverter(
def typeString =
if (originalType == null) s"$typeName" else s"$typeName ($originalType)"

def typeNotSupported() =
throw new AnalysisException(s"Parquet type not supported: $typeString")

def typeNotImplemented() =
throw new AnalysisException(s"Parquet type not yet supported: $typeString")

Expand Down Expand Up @@ -142,6 +145,9 @@ private[parquet] class CatalystSchemaConverter(
case INT_32 | null => IntegerType
case DATE => DateType
case DECIMAL => makeDecimalType(MAX_PRECISION_FOR_INT32)
case UINT_8 => typeNotSupported()
case UINT_16 => typeNotSupported()
case UINT_32 => typeNotSupported()
case TIME_MILLIS => typeNotImplemented()
case _ => illegalType()
}
Expand All @@ -150,6 +156,7 @@ private[parquet] class CatalystSchemaConverter(
originalType match {
case INT_64 | null => LongType
case DECIMAL => makeDecimalType(MAX_PRECISION_FOR_INT64)
case UINT_64 => typeNotSupported()
case TIMESTAMP_MILLIS => typeNotImplemented()
case _ => illegalType()
}
Expand Down
23 changes: 23 additions & 0 deletions ...re/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -206,6 +206,29 @@ class ParquetIOSuite extends QueryTest with ParquetTest with SharedSQLContext {
}
}

test("SPARK-10113 Support for unsigned Parquet logical types") {
val parquetSchema = MessageTypeParser.parseMessageType(
"""message root {
| required int32 c(UINT_32);
|}
""".stripMargin)

withTempPath { location =>
val fileMetadata = new FileMetaData(parquetSchema, Map.empty[String, String].asJava ,"Spark")
val path = new Path(location.getCanonicalPath)
val footer = List(
new Footer(path, new ParquetMetadata(fileMetadata, Collections.emptyList()))
).asJava

ParquetFileWriter.writeMetadataFile(sparkContext.hadoopConfiguration, path, footer)

val errorMessage = intercept[Throwable] {
sqlContext.read.parquet(path.toString).printSchema()
}.toString
assert(errorMessage.contains("Parquet type not supported"))
}
}

test("compression codec") {
def compressionCodecFor(path: String, codecName: String): String = {
val codecs = for {
Expand Down