Skip to content
Closed
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
25 commits
Select commit Hold shift + click to select a range
f59a213
BinaryComparison shouldn't auto cast string to int/long
wangyum Aug 5, 2017
bc83848
follow hive
wangyum Sep 10, 2017
cedb239
Remove useless code
wangyum Sep 10, 2017
8d37c72
Merge remote-tracking branch 'origin/master' into SPARK-21646
wangyum Sep 10, 2017
522c4cd
Fix test error
wangyum Sep 11, 2017
3bec6a2
Fix SQLQueryTestSuite test error
wangyum Sep 11, 2017
844aec7
Add spark.sql.binary.comparison.compatible.with.hive conf.
wangyum Sep 18, 2017
7812018
spark.sql.binary.comparison.compatible.with.hive -> spark.sql.autoTyp…
wangyum Sep 19, 2017
27d5b13
spark.sql.autoTypeCastingCompatibility -> spark.sql.typeCoercion.mode
wangyum Sep 20, 2017
53d673f
default -> legacy
wangyum Oct 7, 2017
8da0cf0
Fix test error
wangyum Oct 7, 2017
2ada11a
Refactor TypeCoercionModeSuite
wangyum Oct 9, 2017
d34f294
Add IN test suite
wangyum Oct 12, 2017
b99fb60
Merge branch 'master' into SPARK-21646
wangyum Nov 10, 2017
0d9cf69
legacy -> default
wangyum Nov 14, 2017
22d0355
Merge remote-tracking branch 'upstream/master' into SPARK-21646
wangyum Nov 14, 2017
558ff90
Merge remote-tracking branch 'upstream/master' into SPARK-21646
wangyum Dec 5, 2017
663eb35
Update doc
wangyum Dec 6, 2017
7802483
Remove duplicate InConversion
wangyum Dec 6, 2017
dffe5d2
Merge remote-tracking branch 'upstream/master' into SPARK-21646
wangyum Jan 9, 2018
97a071d
Merge SPARK-22894 to Hive mode.
wangyum Jan 9, 2018
408e889
InConversion -> NativeInConversion; PromoteStrings -> NativePromoteSt…
wangyum Jan 9, 2018
e763330
Lost WindowFrameCoercion
wangyum Jan 9, 2018
81067b9
Merge remote-tracking branch 'upstream/master' into SPARK-21646
wangyum Mar 31, 2018
d0a2089
Since Spark 2.3 -> Since Spark 2.4
wangyum Jun 10, 2018
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
InConversion -> NativeInConversion; PromoteStrings -> NativePromoteSt…
…rings
  • Loading branch information
wangyum committed Jan 9, 2018
commit 408e889caa8d61b7267f0f391be4af5fde82a0c9
6 changes: 2 additions & 4 deletions docs/sql-programming-guide.md
Original file line number Diff line number Diff line change
Expand Up @@ -1497,10 +1497,8 @@ that these options will be deprecated in future release as more optimizations ar
<td><code>spark.sql.typeCoercion.mode</code></td>
<td><code>default</code></td>
<td>
The <code>default</code> type coercion mode was used in spark prior to 2.3.0, and so it
continues to be the default to avoid breaking behavior. However, it has logical
inconsistencies. The <code>hive</code> mode is preferred for most new applications, though
it may require additional manual casting.
Since Spark 2.3, the <code>hive</code> mode is introduced for Hive compatiblity.
Spark SQL has its native type cocersion mode, which is enabled by default.
</td>
</tr>
</table>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -67,8 +67,8 @@ object TypeCoercion {
HivePromoteStrings
} else {
commonTypeCoercionRules :+
InConversion :+
PromoteStrings
NativeInConversion :+
NativePromoteStrings
}
}

Expand Down Expand Up @@ -348,7 +348,7 @@ object TypeCoercion {
/**
* Promotes strings that appear in arithmetic expressions.
*/
object PromoteStrings extends TypeCoercionRule {
object NativePromoteStrings extends TypeCoercionRule {

override protected def coerceTypes(
plan: LogicalPlan): LogicalPlan = plan transformAllExpressions {
Expand Down Expand Up @@ -451,7 +451,7 @@ object TypeCoercion {
* operator type is found the original expression will be returned and an
* Analysis Exception will be raised at the type checking phase.
*/
object InConversion extends TypeCoercionRule {
object NativeInConversion extends TypeCoercionRule {

override protected def coerceTypes(
plan: LogicalPlan): LogicalPlan = plan transformAllExpressions {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1032,11 +1032,8 @@ object SQLConf {

val typeCoercionMode =
buildConf("spark.sql.typeCoercion.mode")
.doc("The 'default' typeCoercion mode was used in spark prior to 2.3, " +
"and so it continues to be the default to avoid breaking behavior. " +
"However, it has logical inconsistencies. " +
"The 'hive' mode is preferred for most new applications, " +
"though it may require additional manual casting.")
.doc("Since Spark 2.3, the 'hive' mode is introduced for Hive compatiblity. " +
"Spark SQL has its native type cocersion mode, which is enabled by default.")
.stringConf
.transform(_.toLowerCase(Locale.ROOT))
.checkValues(Set("default", "hive"))
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1207,7 +1207,7 @@ class TypeCoercionSuite extends AnalysisTest {
*/
test("make sure rules do not fire early") {
// InConversion
val inConversion = TypeCoercion.InConversion
val inConversion = TypeCoercion.NativeInConversion
ruleTest(inConversion,
In(UnresolvedAttribute("a"), Seq(Literal(1))),
In(UnresolvedAttribute("a"), Seq(Literal(1)))
Expand Down Expand Up @@ -1251,16 +1251,16 @@ class TypeCoercionSuite extends AnalysisTest {
}

test("binary comparison with string promotion") {
ruleTest(PromoteStrings,
ruleTest(NativePromoteStrings,
GreaterThan(Literal("123"), Literal(1)),
GreaterThan(Cast(Literal("123"), IntegerType), Literal(1)))
ruleTest(PromoteStrings,
ruleTest(NativePromoteStrings,
LessThan(Literal(true), Literal("123")),
LessThan(Literal(true), Cast(Literal("123"), BooleanType)))
ruleTest(PromoteStrings,
ruleTest(NativePromoteStrings,
EqualTo(Literal(Array(1, 2)), Literal("123")),
EqualTo(Literal(Array(1, 2)), Literal("123")))
ruleTest(PromoteStrings,
ruleTest(NativePromoteStrings,
GreaterThan(Literal("1.5"), Literal(BigDecimal("0.5"))),
GreaterThan(Cast(Literal("1.5"), DoubleType), Cast(Literal(BigDecimal("0.5")), DoubleType)))
}
Expand Down