Skip to content

Commit 45b4527

Browse files
brkyvzAndrew Or
brkyvz
authored and
Andrew Or
committed
[SPARK-8095][BACKPORT] Resolve dependencies of --packages in local ivy cache
Backported PR #6788 cc andrewor14 Author: Burak Yavuz <[email protected]> Closes #6923 from brkyvz/backport-local-ivy and squashes the following commits: eb17384 [Burak Yavuz] [SPARK-8095][BACKPORT] Resolve dependencies of --packages in local ivy cache
1 parent 0b8dce0 commit 45b4527

File tree

3 files changed

+138
-35
lines changed

3 files changed

+138
-35
lines changed

‎core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala‎

Lines changed: 16 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,8 @@ import org.apache.ivy.core.resolve.ResolveOptions
3535
import org.apache.ivy.core.retrieve.RetrieveOptions
3636
import org.apache.ivy.core.settings.IvySettings
3737
import org.apache.ivy.plugins.matcher.GlobPatternMatcher
38-
import org.apache.ivy.plugins.resolver.{ChainResolver, IBiblioResolver}
38+
import org.apache.ivy.plugins.repository.file.FileRepository
39+
import org.apache.ivy.plugins.resolver.{FileSystemResolver, ChainResolver, IBiblioResolver}
3940

4041
import org.apache.spark.SPARK_VERSION
4142
import org.apache.spark.deploy.rest._
@@ -677,8 +678,14 @@ private[spark] object SparkSubmitUtils {
677678
}
678679

679680
/** Path of the local Maven cache. */
680-
private[spark] def m2Path: File = new File(System.getProperty("user.home"),
681-
".m2" + File.separator + "repository" + File.separator)
681+
private[spark] def m2Path: File = {
682+
if (Utils.isTesting) {
683+
// test builds delete the maven cache, and this can cause flakiness
684+
new File("dummy", ".m2" + File.separator + "repository")
685+
} else {
686+
new File(System.getProperty("user.home"), ".m2" + File.separator + "repository")
687+
}
688+
}
682689

683690
/**
684691
* Extracts maven coordinates from a comma-delimited string
@@ -700,12 +707,13 @@ private[spark] object SparkSubmitUtils {
700707
localM2.setName("local-m2-cache")
701708
cr.add(localM2)
702709

703-
val localIvy = new IBiblioResolver
704-
localIvy.setRoot(new File(ivySettings.getDefaultIvyUserDir,
705-
"local" + File.separator).toURI.toString)
710+
val localIvy = new FileSystemResolver
711+
val localIvyRoot = new File(ivySettings.getDefaultIvyUserDir, "local")
712+
localIvy.setLocal(true)
713+
localIvy.setRepository(new FileRepository(localIvyRoot))
706714
val ivyPattern = Seq("[organisation]", "[module]", "[revision]", "[type]s",
707715
"[artifact](-[classifier]).[ext]").mkString(File.separator)
708-
localIvy.setPattern(ivyPattern)
716+
localIvy.addIvyPattern(localIvyRoot.getAbsolutePath + File.separator + ivyPattern)
709717
localIvy.setName("local-ivy-cache")
710718
cr.add(localIvy)
711719

@@ -769,7 +777,7 @@ private[spark] object SparkSubmitUtils {
769777
md.addDependency(dd)
770778
}
771779
}
772-
780+
773781
/** Add exclusion rules for dependencies already included in the spark-assembly */
774782
private[spark] def addExclusionRules(
775783
ivySettings: IvySettings,

‎core/src/test/scala/org/apache/spark/deploy/IvyTestUtils.scala‎

Lines changed: 108 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,8 @@ import com.google.common.io.{Files, ByteStreams}
2424

2525
import org.apache.commons.io.FileUtils
2626

27+
import org.apache.ivy.core.settings.IvySettings
28+
2729
import org.apache.spark.TestUtils.{createCompiledClass, JavaSourceFromString}
2830
import org.apache.spark.deploy.SparkSubmitUtils.MavenCoordinate
2931

@@ -44,13 +46,30 @@ private[deploy] object IvyTestUtils {
4446
if (!useIvyLayout) {
4547
Seq(groupDirs, artifactDirs, artifact.version).mkString(File.separator)
4648
} else {
47-
Seq(groupDirs, artifactDirs, artifact.version, ext + "s").mkString(File.separator)
49+
Seq(artifact.groupId, artifactDirs, artifact.version, ext + "s").mkString(File.separator)
4850
}
4951
new File(prefix, artifactPath)
5052
}
5153

52-
private def artifactName(artifact: MavenCoordinate, ext: String = ".jar"): String = {
53-
s"${artifact.artifactId}-${artifact.version}$ext"
54+
/** Returns the artifact naming based on standard ivy or maven format. */
55+
private def artifactName(
56+
artifact: MavenCoordinate,
57+
useIvyLayout: Boolean,
58+
ext: String = ".jar"): String = {
59+
if (!useIvyLayout) {
60+
s"${artifact.artifactId}-${artifact.version}$ext"
61+
} else {
62+
s"${artifact.artifactId}$ext"
63+
}
64+
}
65+
66+
/** Returns the directory for the given groupId based on standard ivy or maven format. */
67+
private def getBaseGroupDirectory(artifact: MavenCoordinate, useIvyLayout: Boolean): String = {
68+
if (!useIvyLayout) {
69+
artifact.groupId.replace(".", File.separator)
70+
} else {
71+
artifact.groupId
72+
}
5473
}
5574

5675
/** Write the contents to a file to the supplied directory. */
@@ -92,6 +111,23 @@ private[deploy] object IvyTestUtils {
92111
createCompiledClass(className, dir, sourceFile, Seq.empty)
93112
}
94113

114+
private def createDescriptor(
115+
tempPath: File,
116+
artifact: MavenCoordinate,
117+
dependencies: Option[Seq[MavenCoordinate]],
118+
useIvyLayout: Boolean): File = {
119+
if (useIvyLayout) {
120+
val ivyXmlPath = pathFromCoordinate(artifact, tempPath, "ivy", true)
121+
Files.createParentDirs(new File(ivyXmlPath, "dummy"))
122+
createIvyDescriptor(ivyXmlPath, artifact, dependencies)
123+
} else {
124+
val pomPath = pathFromCoordinate(artifact, tempPath, "pom", useIvyLayout)
125+
Files.createParentDirs(new File(pomPath, "dummy"))
126+
createPom(pomPath, artifact, dependencies)
127+
}
128+
}
129+
130+
95131
/** Helper method to write artifact information in the pom. */
96132
private def pomArtifactWriter(artifact: MavenCoordinate, tabCount: Int = 1): String = {
97133
var result = "\n" + " " * tabCount + s"<groupId>${artifact.groupId}</groupId>"
@@ -121,15 +157,55 @@ private[deploy] object IvyTestUtils {
121157
"\n <dependencies>\n" + inside + "\n </dependencies>"
122158
}.getOrElse("")
123159
content += "\n</project>"
124-
writeFile(dir, artifactName(artifact, ".pom"), content.trim)
160+
writeFile(dir, artifactName(artifact, false, ".pom"), content.trim)
161+
}
162+
163+
/** Helper method to write artifact information in the ivy.xml. */
164+
private def ivyArtifactWriter(artifact: MavenCoordinate): String = {
165+
s"""<dependency org="${artifact.groupId}" name="${artifact.artifactId}"
166+
| rev="${artifact.version}" force="true"
167+
| conf="compile->compile(*),master(*);runtime->runtime(*)"/>""".stripMargin
168+
}
169+
170+
/** Create a pom file for this artifact. */
171+
private def createIvyDescriptor(
172+
dir: File,
173+
artifact: MavenCoordinate,
174+
dependencies: Option[Seq[MavenCoordinate]]): File = {
175+
var content = s"""
176+
|<?xml version="1.0" encoding="UTF-8"?>
177+
|<ivy-module version="2.0" xmlns:m="http://ant.apache.org/ivy/maven">
178+
| <info organisation="${artifact.groupId}"
179+
| module="${artifact.artifactId}"
180+
| revision="${artifact.version}"
181+
| status="release" publication="20150405222456" />
182+
| <configurations>
183+
| <conf name="default" visibility="public" description="" extends="runtime,master"/>
184+
| <conf name="compile" visibility="public" description=""/>
185+
| <conf name="master" visibility="public" description=""/>
186+
| <conf name="runtime" visibility="public" description="" extends="compile"/>
187+
| <conf name="pom" visibility="public" description=""/>
188+
| </configurations>
189+
| <publications>
190+
| <artifact name="${artifactName(artifact, true, "")}" type="jar" ext="jar"
191+
| conf="master"/>
192+
| </publications>
193+
""".stripMargin.trim
194+
content += dependencies.map { deps =>
195+
val inside = deps.map(ivyArtifactWriter).mkString("\n")
196+
"\n <dependencies>\n" + inside + "\n </dependencies>"
197+
}.getOrElse("")
198+
content += "\n</ivy-module>"
199+
writeFile(dir, "ivy.xml", content.trim)
125200
}
126201

127202
/** Create the jar for the given maven coordinate, using the supplied files. */
128203
private def packJar(
129204
dir: File,
130205
artifact: MavenCoordinate,
131-
files: Seq[(String, File)]): File = {
132-
val jarFile = new File(dir, artifactName(artifact))
206+
files: Seq[(String, File)],
207+
useIvyLayout: Boolean): File = {
208+
val jarFile = new File(dir, artifactName(artifact, useIvyLayout))
133209
val jarFileStream = new FileOutputStream(jarFile)
134210
val jarStream = new JarOutputStream(jarFileStream, new java.util.jar.Manifest())
135211

@@ -187,12 +263,10 @@ private[deploy] object IvyTestUtils {
187263
} else {
188264
Seq(javaFile)
189265
}
190-
val jarFile = packJar(jarPath, artifact, allFiles)
266+
val jarFile = packJar(jarPath, artifact, allFiles, useIvyLayout)
191267
assert(jarFile.exists(), "Problem creating Jar file")
192-
val pomPath = pathFromCoordinate(artifact, tempPath, "pom", useIvyLayout)
193-
Files.createParentDirs(new File(pomPath, "dummy"))
194-
val pomFile = createPom(pomPath, artifact, dependencies)
195-
assert(pomFile.exists(), "Problem creating Pom file")
268+
val descriptor = createDescriptor(tempPath, artifact, dependencies, useIvyLayout)
269+
assert(descriptor.exists(), "Problem creating Pom file")
196270
} finally {
197271
FileUtils.deleteDirectory(root)
198272
}
@@ -237,25 +311,40 @@ private[deploy] object IvyTestUtils {
237311
dependencies: Option[String],
238312
rootDir: Option[File],
239313
useIvyLayout: Boolean = false,
240-
withPython: Boolean = false)(f: String => Unit): Unit = {
314+
withPython: Boolean = false,
315+
ivySettings: IvySettings = new IvySettings)(f: String => Unit): Unit = {
316+
val deps = dependencies.map(SparkSubmitUtils.extractMavenCoordinates)
317+
purgeLocalIvyCache(artifact, deps, ivySettings)
241318
val repo = createLocalRepositoryForTests(artifact, dependencies, rootDir, useIvyLayout,
242319
withPython)
243320
try {
244321
f(repo.toURI.toString)
245322
} finally {
246323
// Clean up
247324
if (repo.toString.contains(".m2") || repo.toString.contains(".ivy2")) {
248-
FileUtils.deleteDirectory(new File(repo,
249-
artifact.groupId.replace(".", File.separator) + File.separator + artifact.artifactId))
250-
dependencies.map(SparkSubmitUtils.extractMavenCoordinates).foreach { seq =>
251-
seq.foreach { dep =>
252-
FileUtils.deleteDirectory(new File(repo,
253-
dep.artifactId.replace(".", File.separator)))
254-
}
325+
val groupDir = getBaseGroupDirectory(artifact, useIvyLayout)
326+
FileUtils.deleteDirectory(new File(repo, groupDir + File.separator + artifact.artifactId))
327+
deps.foreach { _.foreach { dep =>
328+
FileUtils.deleteDirectory(new File(repo, getBaseGroupDirectory(dep, useIvyLayout)))
329+
}
255330
}
256331
} else {
257332
FileUtils.deleteDirectory(repo)
258333
}
334+
purgeLocalIvyCache(artifact, deps, ivySettings)
335+
}
336+
}
337+
338+
/** Deletes the test packages from the ivy cache */
339+
private def purgeLocalIvyCache(
340+
artifact: MavenCoordinate,
341+
dependencies: Option[Seq[MavenCoordinate]],
342+
ivySettings: IvySettings): Unit = {
343+
// delete the artifact from the cache as well if it already exists
344+
FileUtils.deleteDirectory(new File(ivySettings.getDefaultCache, artifact.groupId))
345+
dependencies.foreach { _.foreach { dep =>
346+
FileUtils.deleteDirectory(new File(ivySettings.getDefaultCache, dep.groupId))
347+
}
259348
}
260349
}
261350
}

‎core/src/test/scala/org/apache/spark/deploy/SparkSubmitUtilsSuite.scala‎

Lines changed: 14 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ import scala.collection.mutable.ArrayBuffer
2323

2424
import org.apache.ivy.core.module.descriptor.MDArtifact
2525
import org.apache.ivy.core.settings.IvySettings
26-
import org.apache.ivy.plugins.resolver.IBiblioResolver
26+
import org.apache.ivy.plugins.resolver.{AbstractResolver, FileSystemResolver, IBiblioResolver}
2727
import org.scalatest.BeforeAndAfterAll
2828

2929
import org.apache.spark.SparkFunSuite
@@ -68,18 +68,18 @@ class SparkSubmitUtilsSuite extends SparkFunSuite with BeforeAndAfterAll {
6868
// should have central and spark-packages by default
6969
assert(res1.getResolvers.size() === 4)
7070
assert(res1.getResolvers.get(0).asInstanceOf[IBiblioResolver].getName === "local-m2-cache")
71-
assert(res1.getResolvers.get(1).asInstanceOf[IBiblioResolver].getName === "local-ivy-cache")
71+
assert(res1.getResolvers.get(1).asInstanceOf[FileSystemResolver].getName === "local-ivy-cache")
7272
assert(res1.getResolvers.get(2).asInstanceOf[IBiblioResolver].getName === "central")
7373
assert(res1.getResolvers.get(3).asInstanceOf[IBiblioResolver].getName === "spark-packages")
7474

7575
val repos = "a/1,b/2,c/3"
7676
val resolver2 = SparkSubmitUtils.createRepoResolvers(Option(repos), settings)
7777
assert(resolver2.getResolvers.size() === 7)
7878
val expected = repos.split(",").map(r => s"$r/")
79-
resolver2.getResolvers.toArray.zipWithIndex.foreach { case (resolver: IBiblioResolver, i) =>
79+
resolver2.getResolvers.toArray.zipWithIndex.foreach { case (resolver: AbstractResolver, i) =>
8080
if (i > 3) {
8181
assert(resolver.getName === s"repo-${i - 3}")
82-
assert(resolver.getRoot === expected(i - 4))
82+
assert(resolver.asInstanceOf[IBiblioResolver].getRoot === expected(i - 4))
8383
}
8484
}
8585
}
@@ -112,26 +112,32 @@ class SparkSubmitUtilsSuite extends SparkFunSuite with BeforeAndAfterAll {
112112
}
113113

114114
test("search for artifact at local repositories") {
115-
val main = new MavenCoordinate("my.awesome.lib", "mylib", "0.1")
115+
val main = new MavenCoordinate("my.great.lib", "mylib", "0.1")
116+
val dep = "my.great.dep:mydep:0.5"
116117
// Local M2 repository
117-
IvyTestUtils.withRepository(main, None, Some(SparkSubmitUtils.m2Path)) { repo =>
118+
IvyTestUtils.withRepository(main, Some(dep), Some(SparkSubmitUtils.m2Path)) { repo =>
118119
val jarPath = SparkSubmitUtils.resolveMavenCoordinates(main.toString, None, None, true)
119120
assert(jarPath.indexOf("mylib") >= 0, "should find artifact")
121+
assert(jarPath.indexOf("mydep") >= 0, "should find dependency")
120122
}
121123
// Local Ivy Repository
122124
val settings = new IvySettings
123125
val ivyLocal = new File(settings.getDefaultIvyUserDir, "local" + File.separator)
124-
IvyTestUtils.withRepository(main, None, Some(ivyLocal), true) { repo =>
126+
IvyTestUtils.withRepository(main, Some(dep), Some(ivyLocal), useIvyLayout = true) { repo =>
125127
val jarPath = SparkSubmitUtils.resolveMavenCoordinates(main.toString, None, None, true)
126128
assert(jarPath.indexOf("mylib") >= 0, "should find artifact")
129+
assert(jarPath.indexOf("mydep") >= 0, "should find dependency")
127130
}
128131
// Local ivy repository with modified home
129132
val dummyIvyLocal = new File(tempIvyPath, "local" + File.separator)
130-
IvyTestUtils.withRepository(main, None, Some(dummyIvyLocal), true) { repo =>
133+
settings.setDefaultIvyUserDir(new File(tempIvyPath))
134+
IvyTestUtils.withRepository(main, Some(dep), Some(dummyIvyLocal), useIvyLayout = true,
135+
ivySettings = settings) { repo =>
131136
val jarPath = SparkSubmitUtils.resolveMavenCoordinates(main.toString, None,
132137
Some(tempIvyPath), true)
133138
assert(jarPath.indexOf("mylib") >= 0, "should find artifact")
134139
assert(jarPath.indexOf(tempIvyPath) >= 0, "should be in new ivy path")
140+
assert(jarPath.indexOf("mydep") >= 0, "should find dependency")
135141
}
136142
}
137143

0 commit comments

Comments
 (0)