Skip to content
Closed
Changes from 1 commit
Commits
Show all changes
57 commits
Select commit Hold shift + click to select a range
afaf7bd
[SPARK-33084][CORE][SQL]Add jar support ivy path
AngersZhuuuu Oct 7, 2020
51daf9a
Merge branch 'master' into support-add-jar-ivy
AngersZhuuuu Oct 7, 2020
3579de0
Update SparkContextSuite.scala
AngersZhuuuu Oct 7, 2020
d6e8caf
Merge branch 'support-add-jar-ivy' of https://github.com/AngersZhuuuu…
AngersZhuuuu Oct 7, 2020
169e1f8
Update Utils.scala
AngersZhuuuu Oct 8, 2020
0e589ec
Merge branch 'master' into support-add-jar-ivy
AngersZhuuuu Nov 21, 2020
b3e3211
resolve conflict
AngersZhuuuu Nov 23, 2020
9161340
Update SessionState.scala
AngersZhuuuu Nov 23, 2020
0e3c1ec
Update sql-ref-syntax-aux-resource-mgmt-add-jar.md
AngersZhuuuu Nov 24, 2020
300ca56
follow comment
AngersZhuuuu Nov 24, 2020
63e877b
https://github.com/apache/spark/pull/29966#discussion_r529242208
AngersZhuuuu Nov 24, 2020
733e62c
follow comment
AngersZhuuuu Nov 24, 2020
b60ba1e
Update sql-ref-syntax-aux-resource-mgmt-add-jar.md
AngersZhuuuu Nov 24, 2020
883b9d3
fix uri re-use
AngersZhuuuu Nov 28, 2020
208afc2
follow comment
AngersZhuuuu Nov 28, 2020
ba9ea29
add warn message whe multiple trasitive
AngersZhuuuu Nov 28, 2020
10b3737
move DependencyUtils
AngersZhuuuu Nov 28, 2020
7f878c2
add ut
AngersZhuuuu Nov 28, 2020
d2c1950
Update SessionState.scala
AngersZhuuuu Nov 28, 2020
2200076
Update SessionState.scala
AngersZhuuuu Nov 28, 2020
5a9cc30
Update DependencyUtils.scala
AngersZhuuuu Nov 29, 2020
875d8a7
Add end to end test
AngersZhuuuu Nov 29, 2020
e921245
Update SQLQuerySuite.scala
AngersZhuuuu Nov 29, 2020
614a865
follow comment
AngersZhuuuu Nov 30, 2020
8c5cb7c
Update SparkContext.scala
AngersZhuuuu Nov 30, 2020
f460974
fix local path with comma
AngersZhuuuu Nov 30, 2020
1f7dc01
Merge branch 'support-add-jar-ivy' of https://github.com/AngersZhuuuu…
AngersZhuuuu Dec 1, 2020
050c410
follow comment
AngersZhuuuu Dec 1, 2020
ff611a6
Update SessionState.scala
AngersZhuuuu Dec 1, 2020
03aca3b
split UT and use simply dependency ivy path
AngersZhuuuu Dec 1, 2020
653b919
Update DependencyUtils.scala
AngersZhuuuu Dec 1, 2020
6e48275
Update SparkContext.scala
AngersZhuuuu Dec 1, 2020
bdc5035
follow comment
AngersZhuuuu Dec 2, 2020
9c22882
follow comment
AngersZhuuuu Dec 2, 2020
9c88f8d
follow comment
AngersZhuuuu Dec 2, 2020
8220e5a
Update SparkContextSuite.scala
AngersZhuuuu Dec 2, 2020
49ac62c
follow comment
AngersZhuuuu Dec 2, 2020
b69a62e
Update DependencyUtils.scala
AngersZhuuuu Dec 2, 2020
273a5ac
Follow comment
AngersZhuuuu Dec 3, 2020
ebe1c9c
Update DependencyUtils.scala
AngersZhuuuu Dec 4, 2020
6034fb2
Update sql-ref-syntax-aux-resource-mgmt-add-jar.md
AngersZhuuuu Dec 5, 2020
e22e398
Update SparkContext.scala
AngersZhuuuu Dec 7, 2020
afea73f
Update SparkContext.scala
AngersZhuuuu Dec 7, 2020
13000f2
Merge branch 'master' into support-add-jar-ivy
AngersZhuuuu Dec 14, 2020
bce3d40
Update SparkContext.scala
AngersZhuuuu Dec 14, 2020
d53f302
Merge branch 'support-add-jar-ivy' of https://github.com/AngersZhuuuu…
AngersZhuuuu Dec 14, 2020
57c351d
Update HiveQuerySuite.scala
AngersZhuuuu Dec 15, 2020
8c53b83
follow comment
AngersZhuuuu Dec 22, 2020
4048c5b
https://github.com/apache/spark/pull/29966#discussion_r547040115
AngersZhuuuu Dec 22, 2020
aa53482
Merge branch 'master' into support-add-jar-ivy
AngersZhuuuu Dec 22, 2020
2ffb431
Update SQLQuerySuite.scala
AngersZhuuuu Dec 22, 2020
8c18cdf
Update SparkContext.scala
AngersZhuuuu Dec 23, 2020
6bd41cd
Update SparkSubmit.scala
AngersZhuuuu Dec 23, 2020
fbc236c
follwo comment
AngersZhuuuu Dec 23, 2020
90491d5
Update DependencyUtils.scala
AngersZhuuuu Dec 23, 2020
75ff3ce
Update SparkContextSuite.scala
AngersZhuuuu Dec 23, 2020
4c44dae
follow comment remove default value
AngersZhuuuu Dec 24, 2020
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Update SparkContextSuite.scala
  • Loading branch information
AngersZhuuuu committed Dec 2, 2020
commit 8220e5ae03aaf6d06edcb83a9e583379f30ec90f
23 changes: 13 additions & 10 deletions core/src/test/scala/org/apache/spark/SparkContextSuite.scala
Original file line number Diff line number Diff line change
Expand Up @@ -1017,10 +1017,10 @@ class SparkContextSuite extends SparkFunSuite with LocalSparkContext with Eventu
// test dependency jars exist
sc.addJar("ivy://org.apache.hive:hive-storage-api:2.7.0?transitive=true")
assert(logAppender.loggingEvents.count(_.getRenderedMessage.contains(
"The dependency jars of ivy URI with" +
"The dependency jars of ivy uri" +
" ivy://org.apache.hive:hive-storage-api:2.7.0?transitive=true")) == 1)
val existMsg = logAppender.loggingEvents.filter(_.getRenderedMessage.contains(
"The dependency jars of ivy URI with" +
"The dependency jars of ivy uri" +
" ivy://org.apache.hive:hive-storage-api:2.7.0?transitive=true"))
.head.getRenderedMessage
dependencyJars.foreach(jar => assert(existMsg.contains(jar)))
Expand Down Expand Up @@ -1053,8 +1053,8 @@ class SparkContextSuite extends SparkFunSuite with LocalSparkContext with Eventu
"invalidParam1=foo&invalidParam2=boo")
assert(sc.listJars().exists(_.contains("org.apache.hive_hive-storage-api-2.7.0.jar")))
assert(logAppender.loggingEvents.exists(_.getRenderedMessage.contains(
"Invalid parameters `invalidParam1,invalidParam2` found in URI query" +
" `invalidParam1=foo&invalidParam2=boo`")))
"Invalid parameters `invalidParam1,invalidParam2` found in ivy URI query" +
" `invalidParam1=foo&invalidParam2=boo`.")))
}
}

Expand Down Expand Up @@ -1105,24 +1105,27 @@ class SparkContextSuite extends SparkFunSuite with LocalSparkContext with Eventu
val e2 = intercept[IllegalArgumentException] {
sc.addJar("ivy://org.apache.hive:hive-contrib")
}.getMessage
assert(e2.contains("Invalid url: Expected 'org:module:version'," +
" found org.apache.hive:hive-contrib"))
assert(e2.contains("Invalid ivy uri authority in uri ivy://org.apache.hive:hive-contrib:" +
" Expected 'org:module:version', found org.apache.hive:hive-contrib."))

val e3 = intercept[IllegalArgumentException] {
sc.addJar("ivy://org.apache.hive:hive-contrib:2.3.7?foo=")
}.getMessage
assert(e3.contains("Invalid query string: foo="))
assert(e3.contains("Invalid query string in ivy uri" +
" ivy://org.apache.hive:hive-contrib:2.3.7?foo=:"))

val e4 = intercept[IllegalArgumentException] {
sc.addJar("ivy://org.apache.hive:hive-contrib:2.3.7?bar=&baz=foo")
}.getMessage
assert(e4.contains("Invalid query string: bar=&baz=foo"))
assert(e4.contains("Invalid query string in ivy uri" +
" ivy://org.apache.hive:hive-contrib:2.3.7?bar=&baz=foo: bar=&baz=foo"))

val e5 = intercept[IllegalArgumentException] {
sc.addJar("ivy://org.apache.hive:hive-contrib:2.3.7?exclude=org.pentaho")
}.getMessage
assert(e5.contains(
"Invalid exclude string: expected 'org:module,org:module,..', found org.pentaho"))
assert(e5.contains("Invalid exclude string in ivy uri" +
" ivy://org.apache.hive:hive-contrib:2.3.7?exclude=org.pentaho:" +
" expected 'org:module,org:module,..', found org.pentaho"))
}
}

Expand Down