Skip to content
Closed
Changes from 1 commit
Commits
Show all changes
31 commits
Select commit Hold shift + click to select a range
677541b
[SPARK-21786][SQL] The 'spark.sql.parquet.compression.codec' configur…
fjh100456 Sep 13, 2017
4e70fff
[SPARK-21786][SQL] The 'spark.sql.parquet.compression.codec' configur…
fjh100456 Sep 14, 2017
3f022f9
[SPARK-21786][SQL] The 'spark.sql.parquet.compression.codec' configur…
fjh100456 Sep 15, 2017
6d77bf9
[SPARK-21786][SQL] The 'spark.sql.parquet.compression.codec' configur…
fjh100456 Sep 15, 2017
42aca3d
[SPARK-21786][SQL] The 'spark.sql.parquet.compression.codec' configur…
fjh100456 Sep 15, 2017
5cbe999
[SPARK-21786][SQL] The 'spark.sql.parquet.compression.codec' configur…
fjh100456 Sep 16, 2017
732266c
[SPARK-21786][SQL] The 'spark.sql.parquet.compression.codec' configur…
fjh100456 Sep 16, 2017
c7ff62c
[SPARK-21786][SQL] The 'spark.sql.parquet.compression.codec' configur…
fjh100456 Sep 16, 2017
384ee04
[SPARK-21786][SQL] The 'spark.sql.parquet.compression.codec' and 'spa…
fjh100456 Sep 20, 2017
8c92074
[SPARK-21786][SQL] The 'spark.sql.parquet.compression.codec' and 'spa…
fjh100456 Sep 20, 2017
dd5060a
Merge branch 'master' into master
fjh100456 Sep 20, 2017
d427df5
Update InsertSuite.scala
fjh100456 Sep 20, 2017
35cfa01
Update InsertSuite.scala
fjh100456 Sep 20, 2017
5387497
Fix test problems
fjh100456 Sep 20, 2017
676d6a7
[SPARK-21786][SQL] The 'spark.sql.parquet.compression.codec' and 'spa…
fjh100456 Sep 27, 2017
ae1da8f
Fix scala style issue
fjh100456 Sep 27, 2017
fd73145
[SPARK-21786][SQL] The 'spark.sql.parquet.compression.codec' and 'spa…
fjh100456 Sep 28, 2017
7615939
[SPARK-21786][SQL] The 'spark.sql.parquet.compression.codec' and 'spa…
fjh100456 Sep 28, 2017
90cbcb3
[SPARK-21786][SQL] The 'spark.sql.parquet.compression.codec' and 'spa…
fjh100456 Oct 10, 2017
dd6d635
[SPARK-21786][SQL] The 'spark.sql.parquet.compression.codec' and 'spa…
fjh100456 Oct 10, 2017
4fe8170
[SPARK-21786][SQL] The 'spark.sql.parquet.compression.codec' and 'spa…
fjh100456 Oct 12, 2017
aa31261
[SPARK-21786][SQL] The 'spark.sql.parquet.compression.codec' and 'spa…
fjh100456 Oct 16, 2017
dfb36d9
Merge branch 'master' into master
fjh100456 Oct 16, 2017
c4801f6
[SPARK-21786][SQL] The 'spark.sql.parquet.compression.codec' and 'spa…
fjh100456 Oct 16, 2017
105e129
[SPARK-21786][SQL] The 'spark.sql.parquet.compression.codec' and 'spa…
fjh100456 Oct 16, 2017
dc12038
Merge pull request #1 from apache/master
fjh100456 Dec 18, 2017
d779ee6
[SPARK-21786][SQL] The 'spark.sql.parquet.compression.codec' and 'spa…
fjh100456 Dec 19, 2017
0cb7b7a
[SPARK-21786][SQL] The 'spark.sql.parquet.compression.codec' and 'spa…
fjh100456 Dec 20, 2017
78e0403
Resume the changing, and change it in another pr later.
fjh100456 Dec 23, 2017
7804f60
Change to public
fjh100456 Dec 23, 2017
52cdd75
Fix the code with gatorsmile's suggestion.
fjh100456 Dec 23, 2017
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
[SPARK-21786][SQL] The 'spark.sql.parquet.compression.codec' and 'spa…
…rk.sql.orc.compression.codec' configuration doesn't take effect on hive table writing

Fix test issue
  • Loading branch information
fjh100456 committed Oct 12, 2017
commit 4fe8170a915319a461b409ae284840a83e83c784
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ import org.apache.spark.sql.catalyst.plans.logical.InsertIntoTable
import org.apache.spark.sql.execution.datasources.parquet.ParquetTest
import org.apache.spark.sql.hive.orc.OrcFileOperator
import org.apache.spark.sql.hive.test.TestHiveSingleton
import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.types._
import org.apache.spark.util.Utils

Expand Down Expand Up @@ -734,16 +735,16 @@ class InsertSuite extends QueryTest with TestHiveSingleton with BeforeAndAfter
}

private def getConvertMetastoreConfName(format: String): String = format match {
case "parquet" => "spark.sql.hive.convertMetastoreParquet"
case "orc" => "spark.sql.hive.convertMetastoreOrc"
case "parquet" => HiveUtils.CONVERT_METASTORE_PARQUET.key
case "orc" => HiveUtils.CONVERT_METASTORE_ORC.key
}

private def getSparkCompressionConfName(format: String): String = format match {
case "parquet" => "spark.sql.parquet.compression.codec"
case "orc" => "spark.sql.orc.compression.codec"
case "parquet" => SQLConf.PARQUET_COMPRESSION.key
case "orc" => SQLConf.ORC_COMPRESSION.key
}

private def getTableCompressPropName(format: String): String = {
private def getHiveCompressPropName(format: String): String = {
format.toLowerCase match {
case "parquet" => "parquet.compression"
case "orc" => "orc.compress"
Expand Down Expand Up @@ -776,7 +777,7 @@ class InsertSuite extends QueryTest with TestHiveSingleton with BeforeAndAfter
format: String,
compressionCodec: Option[String]) {
val tblProperties = compressionCodec match {
case Some(prop) => s"TBLPROPERTIES('${getTableCompressPropName(format)}'='$prop')"
case Some(prop) => s"TBLPROPERTIES('${getHiveCompressPropName(format)}'='$prop')"
case _ => ""
}
val partitionCreate = if (isPartitioned) "PARTITIONED BY (p int)" else ""
Expand All @@ -794,7 +795,7 @@ class InsertSuite extends QueryTest with TestHiveSingleton with BeforeAndAfter
s"""
|INSERT OVERWRITE TABLE $tableName
|$partitionInsert
|SELECT * from table_source
|SELECT * FROM table_source
""".stripMargin)
}

Expand Down