Skip to content
Closed
Changes from 1 commit
Commits
Show all changes
48 commits
Select commit Hold shift + click to select a range
aa688fe
Adding conversion of nested Parquet schemas
AndreSchumacher Mar 26, 2014
4d4892a
First commit nested Parquet read converters
AndreSchumacher Mar 27, 2014
6125c75
First working nested Parquet record input
AndreSchumacher Mar 27, 2014
745a42b
Completing testcase for nested data (Addressbook(
AndreSchumacher Apr 1, 2014
ddb40d2
Extending tests for nested Parquet data
AndreSchumacher Apr 1, 2014
1b1b3d6
Fixing one problem with nested arrays
AndreSchumacher Apr 2, 2014
5d80461
fixing one problem with nested structs and breaking up files
AndreSchumacher Apr 2, 2014
98219cf
added struct converter
AndreSchumacher Apr 2, 2014
ee70125
fixing one problem with arrayconverter
AndreSchumacher Apr 3, 2014
b7fcc35
Documenting conversions, bugfix, wrappers of Rows
AndreSchumacher Apr 4, 2014
6dbc9b7
Fixing some problems intruduced during rebase
AndreSchumacher Apr 6, 2014
f8f8911
For primitive rows fall back to more efficient converter, code reorg
AndreSchumacher Apr 6, 2014
4e25fcb
Adding resolution of complex ArrayTypes
AndreSchumacher Apr 8, 2014
a594aed
Scalastyle
AndreSchumacher Apr 8, 2014
b539fde
First commit for MapType
AndreSchumacher Apr 11, 2014
824500c
Adding attribute resolution for MapType
AndreSchumacher Apr 11, 2014
f777b4b
Scalastyle
AndreSchumacher Apr 11, 2014
d1911dc
Simplifying ArrayType conversion
AndreSchumacher Apr 12, 2014
1dc5ac9
First version of WriteSupport for nested types
AndreSchumacher Apr 12, 2014
e99cc51
Fixing nested WriteSupport and adding tests
AndreSchumacher Apr 13, 2014
adc1258
Optimizing imports
AndreSchumacher Apr 13, 2014
f466ff0
Added ParquetAvro tests and revised Array conversion
AndreSchumacher Apr 13, 2014
79d81d5
Replacing field names for array and map in WriteSupport
AndreSchumacher Apr 13, 2014
619c397
Completing Map testcase
AndreSchumacher Apr 14, 2014
c52ff2c
Adding native-array converter
AndreSchumacher Apr 19, 2014
431f00f
Fixing problems introduced during rebase
AndreSchumacher Apr 19, 2014
a6b4f05
Cleaning up ArrayConverter, moving classTag to NativeType, adding Nat…
AndreSchumacher Apr 20, 2014
0ae9376
Doc strings and simplifying ParquetConverter.scala
AndreSchumacher May 10, 2014
32229c7
Removing Row nested values and placing by generic types
AndreSchumacher May 11, 2014
cbb5793
Code review feedback
AndreSchumacher May 11, 2014
191bc0d
Changing to Seq for ArrayType, refactoring SQLParser for nested field…
AndreSchumacher May 24, 2014
2f5a805
Removing stripMargin from test schemas
AndreSchumacher May 24, 2014
de02538
Cleaning up ParquetTestData
AndreSchumacher May 24, 2014
31465d6
Scalastyle: fixing commented out bottom
AndreSchumacher May 24, 2014
3c6b25f
Trying to reduce no-op changes wrt master
AndreSchumacher Jun 1, 2014
3104886
Nested Rows should be Rows, not Seqs.
marmbrus Jun 3, 2014
f7aeba3
[SPARK-1982] Support for ByteType and ShortType.
marmbrus Jun 3, 2014
3e1456c
WIP: Directly serialize catalyst attributes.
marmbrus Jun 4, 2014
14c3fd8
Attempting to fix Spark-Parquet schema conversion
AndreSchumacher Jun 4, 2014
37e0a0a
Cleaning up
AndreSchumacher Jun 4, 2014
88e6bdb
Attempting to fix loss of schema
AndreSchumacher Jun 4, 2014
63d1b57
Cleaning up and Scalastyle
AndreSchumacher Jun 8, 2014
b8a8b9a
More fixes to short and byte conversion
AndreSchumacher Jun 8, 2014
403061f
Fixing some issues with tests and schema metadata
AndreSchumacher Jun 8, 2014
94eea3a
Scalastyle
AndreSchumacher Jun 8, 2014
7eceb67
Review feedback
AndreSchumacher Jun 19, 2014
95c1367
Changes to ParquetRelation and its metadata
AndreSchumacher Jun 19, 2014
30708c8
Taking out AvroParquet test for now to remove Avro dependency
AndreSchumacher Jun 20, 2014
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Fixing problems introduced during rebase
  • Loading branch information
AndreSchumacher committed Jun 19, 2014
commit 431f00ff3ecaa119ce90a2331ff5dd0c07ab897f
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@

package org.apache.spark.sql.parquet

import java.io.File

import org.scalatest.{BeforeAndAfterAll, FunSuiteLike}

import org.apache.avro.{SchemaBuilder, Schema}
Expand All @@ -31,18 +33,13 @@ import parquet.hadoop.util.ContextUtil
import parquet.schema.MessageTypeParser

import org.apache.spark.sql._
import org.apache.spark.sql.catalyst.util.getTempFilePath
import org.apache.spark.sql.test.TestSQLContext
import org.apache.spark.sql.TestData
import org.apache.spark.sql.SchemaRDD
import org.apache.spark.sql.catalyst.expressions._
import org.apache.spark.sql.catalyst.types.IntegerType
import org.apache.spark.util.Utils
import org.apache.spark.sql.test.TestSQLContext
import org.apache.spark.sql.TestData
import org.apache.spark.sql.SchemaRDD
import org.apache.spark.sql.catalyst.util.getTempFilePath
import org.apache.spark.sql.catalyst.expressions.Row
import org.apache.spark.sql.catalyst.types.{StringType, IntegerType, DataType}
import org.apache.spark.sql.catalyst.expressions.AttributeReference
import org.apache.spark.util.Utils

// Implicits
Expand Down Expand Up @@ -557,6 +554,7 @@ class ParquetQuerySuite extends QueryTest with FunSuiteLike with BeforeAndAfterA
// has no effect in this test case
implicit def anyToRow(value: Any): Row = value.asInstanceOf[Row]
val tmpdir = Utils.createTempDir()
Utils.deleteRecursively(tmpdir)
val result = TestSQLContext
.parquetFile(ParquetTestData.testNestedDir1.toString)
.toSchemaRDD
Expand All @@ -581,6 +579,7 @@ class ParquetQuerySuite extends QueryTest with FunSuiteLike with BeforeAndAfterA
.parquetFile(ParquetTestData.testNestedDir4.toString)
.toSchemaRDD
val tmpdir = Utils.createTempDir()
Utils.deleteRecursively(tmpdir)
data.saveAsParquetFile(tmpdir.toString)
TestSQLContext
.parquetFile(tmpdir.toString)
Expand Down Expand Up @@ -744,18 +743,4 @@ class ParquetQuerySuite extends QueryTest with FunSuiteLike with BeforeAndAfterA
assert(mapResult2(0) === 1.1f)
assert(mapResult2(2) === 1.3f)
}

/**
* Creates an empty SchemaRDD backed by a ParquetRelation.
*
* TODO: since this is so experimental it is better to have it here and not
* in SQLContext. Also note that when creating new AttributeReferences
* one needs to take care not to create duplicate Attribute ID's.
*/
private def createParquetFile(path: String, schema: (Tuple2[String, DataType])*): SchemaRDD = {
val attributes = schema.map(t => new AttributeReference(t._1, t._2)())
new SchemaRDD(
TestSQLContext,
ParquetRelation.createEmpty(path, attributes, sparkContext.hadoopConfiguration))
}
}