Skip to content

[SPARK-44914][BUILD] Upgrade Apache Ivy to 2.5.2 #45075

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
dongjoon-hyun wants to merge 2 commits into from
Closed

[SPARK-44914][BUILD] Upgrade Apache Ivy to 2.5.2 #45075

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
74b32b2
[SPARK-44914][BUILD] Upgrade Apache Ivy to 2.5.2
dongjoon-hyun Feb 9, 2024
19e2d4c
respect ivy.home
dongjoon-hyun Feb 23, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
respect ivy.home
  • Loading branch information
@dongjoon-hyun
dongjoon-hyun committed Feb 23, 2024
commit 19e2d4ca867ff1df7678ceba9a667bb9870bbc3c
6 changes: 4 additions & 2 deletions common/utils/src/main/scala/org/apache/spark/util/MavenUtils.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -327,7 +327,8 @@ private[spark] object MavenUtils extends Logging {
if (ivySettings.getDefaultIvyUserDir == null && ivySettings.getDefaultCache == null) {
// To protect old Ivy-based systems like old Spark from Apache Ivy 2.5.2's incompatibility.
// `processIvyPathArg` can overwrite these later.
val alternateIvyDir = System.getProperty("user.home") + File.separator + ".ivy2.5.2"
val alternateIvyDir = System.getProperty("ivy.home",
System.getProperty("user.home") + File.separator + ".ivy2.5.2")
ivySettings.setDefaultIvyUserDir(new File(alternateIvyDir))
ivySettings.setDefaultCache(new File(alternateIvyDir, "cache"))
}
Expand All @@ -344,7 +345,8 @@ private[spark] object MavenUtils extends Logging {
private def processIvyPathArg(ivySettings: IvySettings, ivyPath: Option[String]): Unit = {
val alternateIvyDir = ivyPath.filterNot(_.trim.isEmpty).getOrElse {
// To protect old Ivy-based systems like old Spark from Apache Ivy 2.5.2's incompatibility.
System.getProperty("user.home") + File.separator + ".ivy2.5.2"
System.getProperty("ivy.home",
System.getProperty("user.home") + File.separator + ".ivy2.5.2")
}
ivySettings.setDefaultIvyUserDir(new File(alternateIvyDir))
ivySettings.setDefaultCache(new File(alternateIvyDir, "cache"))
Expand Down
11 changes: 4 additions & 7 deletions sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,6 @@ import org.apache.spark.sql.types._
import org.apache.spark.tags.ExtendedSQLTest
import org.apache.spark.unsafe.types.{CalendarInterval, UTF8String}
import org.apache.spark.util.ResetSystemProperties
import org.apache.spark.util.Utils

@ExtendedSQLTest
class SQLQuerySuite extends QueryTest with SharedSparkSession with AdaptiveSparkPlanHelper
Expand Down Expand Up @@ -3874,12 +3873,12 @@ class SQLQuerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark
test("SPARK-33084: Add jar support Ivy URI in SQL -- jar contains udf class") {
val sumFuncClass = "org.apache.spark.examples.sql.Spark33084"
val functionName = "test_udf"
val targetCacheJarDir = new File(
System.getProperty("user.home") + File.separator + ".ivy2.5.2",
"/local/org.apache.spark/SPARK-33084/1.0/jars/")
try {
withTempDir { dir =>
System.setProperty("ivy.home", dir.getAbsolutePath)
val sourceJar = new File(Thread.currentThread().getContextClassLoader
.getResource("SPARK-33084.jar").getFile)
val targetCacheJarDir = new File(dir.getAbsolutePath +
"/local/org.apache.spark/SPARK-33084/1.0/jars/")
targetCacheJarDir.mkdir()
// copy jar to local cache
FileUtils.copyFileToDirectory(sourceJar, targetCacheJarDir)
Expand All @@ -3906,8 +3905,6 @@ class SQLQuerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark
checkAnswer(sql("SELECT * FROM v1"), Seq(Row(2.0)))
}
}
} finally {
Utils.deleteRecursively(targetCacheJarDir)
}
}

Expand Down