From 27d0c210dcaa72613f1042aa2406d22fc1e5cf7a Mon Sep 17 00:00:00 2001 From: Bryan Cutler Date: Tue, 31 May 2016 17:24:37 -0700 Subject: [PATCH 1/4] Added fallback wildcard for the case where 'default' conf doesn't exist --- core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala index 78606e06fbd32..50e4216314a82 100644 --- a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala +++ b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala @@ -939,7 +939,7 @@ private[spark] object SparkSubmitUtils { artifacts.foreach { mvn => val ri = ModuleRevisionId.newInstance(mvn.groupId, mvn.artifactId, mvn.version) val dd = new DefaultDependencyDescriptor(ri, false, false) - dd.addDependencyConfiguration(ivyConfName, ivyConfName) + dd.addDependencyConfiguration(ivyConfName, ivyConfName + "(*)") // scalastyle:off println printStream.println(s"${dd.getDependencyId} added as a dependency") // scalastyle:on println From ece86f1e463707433ccdc34cb5ed10a506da80cb Mon Sep 17 00:00:00 2001 From: Bryan Cutler Date: Wed, 1 Jun 2016 14:57:54 -0700 Subject: [PATCH 2/4] needed to define an artifact pattern along with ivy pattern to resolve all transitive dependencies --- .../main/scala/org/apache/spark/deploy/SparkSubmit.scala | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala index 50e4216314a82..4c2b5e85af126 100644 --- a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala +++ b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala @@ -891,10 +891,12 @@ private[spark] object SparkSubmitUtils { val localIvy = new FileSystemResolver val localIvyRoot = new File(ivySettings.getDefaultIvyUserDir, "local") localIvy.setLocal(true) - localIvy.setRepository(new FileRepository(localIvyRoot)) - val ivyPattern = Seq("[organisation]", "[module]", "[revision]", "[type]s", - "[artifact](-[classifier]).[ext]").mkString(File.separator) + val ivyPattern = Seq("[organisation]", "[module]", "[revision]", "ivys", + "ivy.xml").mkString(File.separator) localIvy.addIvyPattern(localIvyRoot.getAbsolutePath + File.separator + ivyPattern) + val artifactPattern = Seq("[organisation]", "[module]", "[revision]", "[type]s", + "[artifact](-[classifier]).[ext]").mkString(File.separator) + localIvy.addArtifactPattern(localIvyRoot.getAbsolutePath + File.separator + artifactPattern) localIvy.setName("local-ivy-cache") cr.add(localIvy) From b87e0c95c140389148a2140962709fd37d61d427 Mon Sep 17 00:00:00 2001 From: Bryan Cutler Date: Wed, 1 Jun 2016 15:47:22 -0700 Subject: [PATCH 3/4] changed local-ivy-cache resolver to fallback to runtime conf if default not present --- core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala index 4c2b5e85af126..e5e8635d12955 100644 --- a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala +++ b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala @@ -941,7 +941,7 @@ private[spark] object SparkSubmitUtils { artifacts.foreach { mvn => val ri = ModuleRevisionId.newInstance(mvn.groupId, mvn.artifactId, mvn.version) val dd = new DefaultDependencyDescriptor(ri, false, false) - dd.addDependencyConfiguration(ivyConfName, ivyConfName + "(*)") + dd.addDependencyConfiguration(ivyConfName, ivyConfName + "(runtime)") // scalastyle:off println printStream.println(s"${dd.getDependencyId} added as a dependency") // scalastyle:on println From df1ac92b6ae73eaf05b4f4b3d9a4a9178d724f05 Mon Sep 17 00:00:00 2001 From: Bryan Cutler Date: Mon, 15 Aug 2016 17:23:48 -0700 Subject: [PATCH 4/4] added back in setting of FileResolver root as backup, combined path joining --- .../scala/org/apache/spark/deploy/SparkSubmit.scala | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala index e5e8635d12955..dd82eaa4db862 100644 --- a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala +++ b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala @@ -891,12 +891,13 @@ private[spark] object SparkSubmitUtils { val localIvy = new FileSystemResolver val localIvyRoot = new File(ivySettings.getDefaultIvyUserDir, "local") localIvy.setLocal(true) - val ivyPattern = Seq("[organisation]", "[module]", "[revision]", "ivys", - "ivy.xml").mkString(File.separator) - localIvy.addIvyPattern(localIvyRoot.getAbsolutePath + File.separator + ivyPattern) - val artifactPattern = Seq("[organisation]", "[module]", "[revision]", "[type]s", - "[artifact](-[classifier]).[ext]").mkString(File.separator) - localIvy.addArtifactPattern(localIvyRoot.getAbsolutePath + File.separator + artifactPattern) + localIvy.setRepository(new FileRepository(localIvyRoot)) + val ivyPattern = Seq(localIvyRoot.getAbsolutePath, "[organisation]", "[module]", "[revision]", + "ivys", "ivy.xml").mkString(File.separator) + localIvy.addIvyPattern(ivyPattern) + val artifactPattern = Seq(localIvyRoot.getAbsolutePath, "[organisation]", "[module]", + "[revision]", "[type]s", "[artifact](-[classifier]).[ext]").mkString(File.separator) + localIvy.addArtifactPattern(artifactPattern) localIvy.setName("local-ivy-cache") cr.add(localIvy)