Skip to content

[SPARK-48658][SQL] Encode/Decode functions report coding errors instead of mojibake for unmappable characters #47017

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
yaooqinn wants to merge 12 commits into from
Closed

[SPARK-48658][SQL] Encode/Decode functions report coding errors instead of mojibake for unmappable characters #47017

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
6426fdd
[SPARK-48658][SQL] Encode/Decode functions report coding error instea…
yaooqinn Jun 19, 2024
aee78a5
[SPARK-48658][SQL] Encode/Decode functions report coding error instea…
yaooqinn Jun 19, 2024
afb2d08
[SPARK-48658][SQL] Encode/Decode functions report coding error instea…
yaooqinn Jun 19, 2024
f6dd4fa
fix ExplainSuite
yaooqinn Jun 19, 2024
851135c
fix golden file tests
yaooqinn Jun 19, 2024
d3473a4
fix golden file tests
yaooqinn Jun 19, 2024
3e90976
Encode RuntimeReplaceable with StaticInvoke
yaooqinn Jun 20, 2024
b0cf6ee
Decode RuntimeReplaceable with StaticInvoke
yaooqinn Jun 20, 2024
64f3c39
fix tests
yaooqinn Jun 20, 2024
9d2583c
Merge branch 'master' into SPARK-48658
yaooqinn Jun 20, 2024
8f5a236
fix
yaooqinn Jun 20, 2024
d7a4199
address comments
yaooqinn Jun 24, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
[SPARK-48658][SQL] Encode/Decode functions report coding error instea… 
…d of mojibake
  • Loading branch information
@yaooqinn
yaooqinn committed Jun 19, 2024
commit afb2d08a794edd56465c602e86ff1bf5f8618ae8
12 changes: 6 additions & 6 deletions sql/core/src/test/resources/sql-tests/analyzer-results/ansi/string-functions.sql.out
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -912,14 +912,14 @@ SetCommand (spark.sql.legacy.codingErrorAction,Some(true))


-- !query
select decode('E58A9DE5909BE69BB4E5B0BDE4B880E69DAFE98592', 'US-ASCII')
select decode(X'E58A9DE5909BE69BB4E5B0BDE4B880E69DAFE98592', 'US-ASCII')
-- !query analysis
Project [decode(E58A9DE5909BE69BB4E5B0BDE4B880E69DAFE98592, US-ASCII) AS decode(E58A9DE5909BE69BB4E5B0BDE4B880E69DAFE98592, US-ASCII)#x]
Project [decode(0xE58A9DE5909BE69BB4E5B0BDE4B880E69DAFE98592, US-ASCII) AS decode(X'E58A9DE5909BE69BB4E5B0BDE4B880E69DAFE98592', US-ASCII)#x]
+- OneRowRelation


-- !query
select decode(scol, ecol) from values('E58A9DE5909BE69BB4E5B0BDE4B880E69DAFE98592', 'US-ASCII') as t(scol, ecol)
select decode(scol, ecol) from values(X'E58A9DE5909BE69BB4E5B0BDE4B880E69DAFE98592', 'US-ASCII') as t(scol, ecol)
-- !query analysis
Project [decode(scol#x, ecol#x) AS decode(scol, ecol)#x]
+- SubqueryAlias t
Expand All @@ -933,14 +933,14 @@ SetCommand (spark.sql.legacy.codingErrorAction,Some(false))


-- !query
select decode('E8A5BFE587BAE998B3E585B3E697A0E69585E4BABA', 'US-ASCII')
select decode(X'E8A5BFE587BAE998B3E585B3E697A0E69585E4BABA', 'US-ASCII')
-- !query analysis
Project [decode(E8A5BFE587BAE998B3E585B3E697A0E69585E4BABA, US-ASCII) AS decode(E8A5BFE587BAE998B3E585B3E697A0E69585E4BABA, US-ASCII)#x]
Project [decode(0xE8A5BFE587BAE998B3E585B3E697A0E69585E4BABA, US-ASCII) AS decode(X'E8A5BFE587BAE998B3E585B3E697A0E69585E4BABA', US-ASCII)#x]
+- OneRowRelation


-- !query
select decode(scol, ecol) from values('E8A5BFE587BAE998B3E585B3E697A0E69585E4BABA', 'US-ASCII') as t(scol, ecol)
select decode(scol, ecol) from values(X'E8A5BFE587BAE998B3E585B3E697A0E69585E4BABA', 'US-ASCII') as t(scol, ecol)
-- !query analysis
Project [decode(scol#x, ecol#x) AS decode(scol, ecol)#x]
+- SubqueryAlias t
Expand Down
12 changes: 6 additions & 6 deletions sql/core/src/test/resources/sql-tests/analyzer-results/string-functions.sql.out
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -912,14 +912,14 @@ SetCommand (spark.sql.legacy.codingErrorAction,Some(true))


-- !query
select decode('E58A9DE5909BE69BB4E5B0BDE4B880E69DAFE98592', 'US-ASCII')
select decode(X'E58A9DE5909BE69BB4E5B0BDE4B880E69DAFE98592', 'US-ASCII')
-- !query analysis
Project [decode(E58A9DE5909BE69BB4E5B0BDE4B880E69DAFE98592, US-ASCII) AS decode(E58A9DE5909BE69BB4E5B0BDE4B880E69DAFE98592, US-ASCII)#x]
Project [decode(0xE58A9DE5909BE69BB4E5B0BDE4B880E69DAFE98592, US-ASCII) AS decode(X'E58A9DE5909BE69BB4E5B0BDE4B880E69DAFE98592', US-ASCII)#x]
+- OneRowRelation


-- !query
select decode(scol, ecol) from values('E58A9DE5909BE69BB4E5B0BDE4B880E69DAFE98592', 'US-ASCII') as t(scol, ecol)
select decode(scol, ecol) from values(X'E58A9DE5909BE69BB4E5B0BDE4B880E69DAFE98592', 'US-ASCII') as t(scol, ecol)
-- !query analysis
Project [decode(scol#x, ecol#x) AS decode(scol, ecol)#x]
+- SubqueryAlias t
Expand All @@ -933,14 +933,14 @@ SetCommand (spark.sql.legacy.codingErrorAction,Some(false))


-- !query
select decode('E8A5BFE587BAE998B3E585B3E697A0E69585E4BABA', 'US-ASCII')
select decode(X'E8A5BFE587BAE998B3E585B3E697A0E69585E4BABA', 'US-ASCII')
-- !query analysis
Project [decode(E8A5BFE587BAE998B3E585B3E697A0E69585E4BABA, US-ASCII) AS decode(E8A5BFE587BAE998B3E585B3E697A0E69585E4BABA, US-ASCII)#x]
Project [decode(0xE8A5BFE587BAE998B3E585B3E697A0E69585E4BABA, US-ASCII) AS decode(X'E8A5BFE587BAE998B3E585B3E697A0E69585E4BABA', US-ASCII)#x]
+- OneRowRelation


-- !query
select decode(scol, ecol) from values('E8A5BFE587BAE998B3E585B3E697A0E69585E4BABA', 'US-ASCII') as t(scol, ecol)
select decode(scol, ecol) from values(X'E8A5BFE587BAE998B3E585B3E697A0E69585E4BABA', 'US-ASCII') as t(scol, ecol)
-- !query analysis
Project [decode(scol#x, ecol#x) AS decode(scol, ecol)#x]
+- SubqueryAlias t
Expand Down
8 changes: 4 additions & 4 deletions sql/core/src/test/resources/sql-tests/inputs/string-functions.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -154,11 +154,11 @@ set spark.sql.legacy.javaCharsets=false;
select decode(X'68656c6c6f', 'WINDOWS-1252');
select decode(scol, ecol) from values(X'68656c6c6f', 'WINDOWS-1252') as t(scol, ecol);
set spark.sql.legacy.codingErrorAction=true;
select decode('E58A9DE5909BE69BB4E5B0BDE4B880E69DAFE98592', 'US-ASCII');
select decode(scol, ecol) from values('E58A9DE5909BE69BB4E5B0BDE4B880E69DAFE98592', 'US-ASCII') as t(scol, ecol);
select decode(X'E58A9DE5909BE69BB4E5B0BDE4B880E69DAFE98592', 'US-ASCII');
select decode(scol, ecol) from values(X'E58A9DE5909BE69BB4E5B0BDE4B880E69DAFE98592', 'US-ASCII') as t(scol, ecol);
set spark.sql.legacy.codingErrorAction=false;
select decode('E8A5BFE587BAE998B3E585B3E697A0E69585E4BABA', 'US-ASCII');
select decode(scol, ecol) from values('E8A5BFE587BAE998B3E585B3E697A0E69585E4BABA', 'US-ASCII') as t(scol, ecol);
select decode(X'E8A5BFE587BAE998B3E585B3E697A0E69585E4BABA', 'US-ASCII');
select decode(scol, ecol) from values(X'E8A5BFE587BAE998B3E585B3E697A0E69585E4BABA', 'US-ASCII') as t(scol, ecol);

-- contains
SELECT CONTAINS(null, 'Spark');
Expand Down
38 changes: 27 additions & 11 deletions sql/core/src/test/resources/sql-tests/results/ansi/string-functions.sql.out
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1198,19 +1198,19 @@ spark.sql.legacy.codingErrorAction true


-- !query
select decode('E58A9DE5909BE69BB4E5B0BDE4B880E69DAFE98592', 'US-ASCII')
select decode(X'E58A9DE5909BE69BB4E5B0BDE4B880E69DAFE98592', 'US-ASCII')
-- !query schema
struct<decode(E58A9DE5909BE69BB4E5B0BDE4B880E69DAFE98592, US-ASCII):string>
struct<decode(X'E58A9DE5909BE69BB4E5B0BDE4B880E69DAFE98592', US-ASCII):string>
-- !query output
E58A9DE5909BE69BB4E5B0BDE4B880E69DAFE98592
���������������������


-- !query
select decode(scol, ecol) from values('E58A9DE5909BE69BB4E5B0BDE4B880E69DAFE98592', 'US-ASCII') as t(scol, ecol)
select decode(scol, ecol) from values(X'E58A9DE5909BE69BB4E5B0BDE4B880E69DAFE98592', 'US-ASCII') as t(scol, ecol)
-- !query schema
struct<decode(scol, ecol):string>
-- !query output
E58A9DE5909BE69BB4E5B0BDE4B880E69DAFE98592
���������������������


-- !query
Expand All @@ -1222,19 +1222,35 @@ spark.sql.legacy.codingErrorAction false


-- !query
select decode('E8A5BFE587BAE998B3E585B3E697A0E69585E4BABA', 'US-ASCII')
select decode(X'E8A5BFE587BAE998B3E585B3E697A0E69585E4BABA', 'US-ASCII')
-- !query schema
struct<decode(E8A5BFE587BAE998B3E585B3E697A0E69585E4BABA, US-ASCII):string>
struct<>
-- !query output
E8A5BFE587BAE998B3E585B3E697A0E69585E4BABA
org.apache.spark.SparkRuntimeException
{
"errorClass" : "MALFORMED_CHARACTER_CODING",
"sqlState" : "22000",
"messageParameters" : {
"charset" : "US-ASCII",
"function" : "`decode`"
}
}


-- !query
select decode(scol, ecol) from values('E8A5BFE587BAE998B3E585B3E697A0E69585E4BABA', 'US-ASCII') as t(scol, ecol)
select decode(scol, ecol) from values(X'E8A5BFE587BAE998B3E585B3E697A0E69585E4BABA', 'US-ASCII') as t(scol, ecol)
-- !query schema
struct<decode(scol, ecol):string>
struct<>
-- !query output
E8A5BFE587BAE998B3E585B3E697A0E69585E4BABA
org.apache.spark.SparkRuntimeException
{
"errorClass" : "MALFORMED_CHARACTER_CODING",
"sqlState" : "22000",
"messageParameters" : {
"charset" : "US-ASCII",
"function" : "`decode`"
}
}


-- !query
Expand Down
38 changes: 27 additions & 11 deletions sql/core/src/test/resources/sql-tests/results/string-functions.sql.out
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1130,19 +1130,19 @@ spark.sql.legacy.codingErrorAction true


-- !query
select decode('E58A9DE5909BE69BB4E5B0BDE4B880E69DAFE98592', 'US-ASCII')
select decode(X'E58A9DE5909BE69BB4E5B0BDE4B880E69DAFE98592', 'US-ASCII')
-- !query schema
struct<decode(E58A9DE5909BE69BB4E5B0BDE4B880E69DAFE98592, US-ASCII):string>
struct<decode(X'E58A9DE5909BE69BB4E5B0BDE4B880E69DAFE98592', US-ASCII):string>
-- !query output
E58A9DE5909BE69BB4E5B0BDE4B880E69DAFE98592
���������������������


-- !query
select decode(scol, ecol) from values('E58A9DE5909BE69BB4E5B0BDE4B880E69DAFE98592', 'US-ASCII') as t(scol, ecol)
select decode(scol, ecol) from values(X'E58A9DE5909BE69BB4E5B0BDE4B880E69DAFE98592', 'US-ASCII') as t(scol, ecol)
-- !query schema
struct<decode(scol, ecol):string>
-- !query output
E58A9DE5909BE69BB4E5B0BDE4B880E69DAFE98592
���������������������


-- !query
Expand All @@ -1154,19 +1154,35 @@ spark.sql.legacy.codingErrorAction false


-- !query
select decode('E8A5BFE587BAE998B3E585B3E697A0E69585E4BABA', 'US-ASCII')
select decode(X'E8A5BFE587BAE998B3E585B3E697A0E69585E4BABA', 'US-ASCII')
-- !query schema
struct<decode(E8A5BFE587BAE998B3E585B3E697A0E69585E4BABA, US-ASCII):string>
struct<>
-- !query output
E8A5BFE587BAE998B3E585B3E697A0E69585E4BABA
org.apache.spark.SparkRuntimeException
{
"errorClass" : "MALFORMED_CHARACTER_CODING",
"sqlState" : "22000",
"messageParameters" : {
"charset" : "US-ASCII",
"function" : "`decode`"
}
}


-- !query
select decode(scol, ecol) from values('E8A5BFE587BAE998B3E585B3E697A0E69585E4BABA', 'US-ASCII') as t(scol, ecol)
select decode(scol, ecol) from values(X'E8A5BFE587BAE998B3E585B3E697A0E69585E4BABA', 'US-ASCII') as t(scol, ecol)
-- !query schema
struct<decode(scol, ecol):string>
struct<>
-- !query output
E8A5BFE587BAE998B3E585B3E697A0E69585E4BABA
org.apache.spark.SparkRuntimeException
{
"errorClass" : "MALFORMED_CHARACTER_CODING",
"sqlState" : "22000",
"messageParameters" : {
"charset" : "US-ASCII",
"function" : "`decode`"
}
}


-- !query
Expand Down