Skip to content

Commit d6795c7

Browse files
lw-linsrowen
authored andcommitted
[SPARK-16515][SQL][FOLLOW-UP] Fix test script on OS X/Windows...
## Problem The current `sed` in `test_script.sh` is missing a `$`, leading to the failure of `script` test on OS X: ``` == Results == !== Correct Answer - 2 == == Spark Answer - 2 == ![x1_y1] [x1] ![x2_y2] [x2] ``` In addition, this `script` test would also fail on systems like Windows where we couldn't be able to invoke `bash` or `echo | sed`. ## What changes were proposed in this pull request? This patch - fixes `sed` in `test_script.sh` - adds command guards so that the `script` test would pass on systems like Windows ## How was this patch tested? - Jenkins - Manually verified tests pass on OS X Author: Liwei Lin <[email protected]> Closes apache#14280 from lw-lin/osx-sed.
1 parent e3c7039 commit d6795c7

File tree

2 files changed

+19
-9
lines changed

2 files changed

+19
-9
lines changed

sql/hive/src/test/resources/test_script.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,5 +19,5 @@
1919

2020
while read line
2121
do
22-
echo "$line" | sed 's/\t/_/'
22+
echo "$line" | sed $'s/\t/_/'
2323
done < /dev/stdin

sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala

Lines changed: 18 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,9 @@ package org.apache.spark.sql.hive.execution
1919

2020
import java.sql.{Date, Timestamp}
2121

22+
import scala.sys.process.Process
23+
import scala.util.Try
24+
2225
import org.apache.hadoop.fs.Path
2326

2427
import org.apache.spark.sql._
@@ -64,14 +67,17 @@ class SQLQuerySuite extends QueryTest with SQLTestUtils with TestHiveSingleton {
6467
import spark.implicits._
6568

6669
test("script") {
67-
val df = Seq(("x1", "y1", "z1"), ("x2", "y2", "z2")).toDF("c1", "c2", "c3")
68-
df.createOrReplaceTempView("script_table")
69-
val query1 = sql(
70-
"""
71-
|SELECT col1 FROM (from(SELECT c1, c2, c3 FROM script_table) tempt_table
72-
|REDUCE c1, c2, c3 USING 'bash src/test/resources/test_script.sh' AS
73-
|(col1 STRING, col2 STRING)) script_test_table""".stripMargin)
74-
checkAnswer(query1, Row("x1_y1") :: Row("x2_y2") :: Nil)
70+
if (testCommandAvailable("bash") && testCommandAvailable("echo | sed")) {
71+
val df = Seq(("x1", "y1", "z1"), ("x2", "y2", "z2")).toDF("c1", "c2", "c3")
72+
df.createOrReplaceTempView("script_table")
73+
val query1 = sql(
74+
"""
75+
|SELECT col1 FROM (from(SELECT c1, c2, c3 FROM script_table) tempt_table
76+
|REDUCE c1, c2, c3 USING 'bash src/test/resources/test_script.sh' AS
77+
|(col1 STRING, col2 STRING)) script_test_table""".stripMargin)
78+
checkAnswer(query1, Row("x1_y1") :: Row("x2_y2") :: Nil)
79+
}
80+
// else skip this test
7581
}
7682

7783
test("UDTF") {
@@ -1766,4 +1772,8 @@ class SQLQuerySuite extends QueryTest with SQLTestUtils with TestHiveSingleton {
17661772
}
17671773
}
17681774
}
1775+
1776+
def testCommandAvailable(command: String): Boolean = {
1777+
Try(Process(command) !!).isSuccess
1778+
}
17691779
}

0 commit comments

Comments
 (0)