Support running the test scope with --spark

Author: Gedochao

modules/cli/src/main/scala/scala/cli/commands/package0/Package.scala

@@ -807,38 +807,34 @@ object Package extends ScalaCommand[PackageOptions] with BuildCommandHelpers {
     * the whole dependency graph.
     */
   def providedFiles(
-    build: Build.Successful,
+    builds: Seq[Build.Successful],
     provided: Seq[dependency.AnyModule],
     logger: Logger
   ): Either[BuildException, Seq[os.Path]] = either {
 
     logger.debug(s"${provided.length} provided dependencies")
-    val res = build.artifacts.resolution.getOrElse {
+    val res = builds.map(_.artifacts.resolution.getOrElse {
       sys.error("Internal error: expected resolution to have been kept")
-    }
-    val modules = value {
+    })
+    val modules: Seq[coursier.Module] = value {
       provided
-        .map(_.toCs(build.scalaParams))
+        .map(_.toCs(builds.head.scalaParams)) // Scala params should be the same for all scopes
         .sequence
         .left.map(CompositeBuildException(_))
     }
     val modulesSet = modules.toSet
     val providedDeps = res
-      .dependencyArtifacts
-      .map(_._1)
+      .flatMap(_.dependencyArtifacts.map(_._1))
       .filter(dep => modulesSet.contains(dep.module))
-    val providedRes = res.subset(providedDeps)
-    val fileMap = build.artifacts.detailedRuntimeArtifacts
-      .map {
-        case (_, _, artifact, path) =>
-          artifact -> path
-      }
+    val providedRes = res.map(_.subset(providedDeps))
+    val fileMap = builds.flatMap(_.artifacts.detailedRuntimeArtifacts).distinct
+      .map { case (_, _, artifact, path) => artifact -> path }
       .toMap
-    val providedFiles = coursier.Artifacts.artifacts(providedRes, Set.empty, None, None, true)
+    val providedFiles = providedRes
+      .flatMap(r => coursier.Artifacts.artifacts(r, Set.empty, None, None, true))
+      .distinct
       .map(_._3)
-      .map { a =>
-        fileMap.getOrElse(a, sys.error(s"should not happen (missing: $a)"))
-      }
+      .map(a => fileMap.getOrElse(a, sys.error(s"should not happen (missing: $a)")))
     logger.debug {
       val it = Iterator(s"${providedFiles.size} provided JAR(s)") ++
         providedFiles.toVector.map(_.toString).sorted.iterator.map(f => s"  $f")
@@ -876,7 +872,7 @@ object Package extends ScalaCommand[PackageOptions] with BuildCommandHelpers {
     val jars =
       if (provided.isEmpty) allJars
       else {
-        val providedFilesSet = value(providedFiles(build, provided, logger)).toSet
+        val providedFilesSet = value(providedFiles(Seq(build), provided, logger)).toSet
         allJars.filterNot(providedFilesSet.contains)
       }
 

modules/cli/src/main/scala/scala/cli/commands/run/Run.scala

@@ -620,7 +620,7 @@ object Run extends ScalaCommand[RunOptions] with BuildCommandHelpers {
           case mode: RunMode.SparkSubmit =>
             value {
               RunSpark.run(
-                builds.head, // TODO: handle multiple builds
+                builds,
                 mainClass,
                 args,
                 mode.submitArgs,

modules/cli/src/main/scala/scala/cli/commands/util/RunSpark.scala

@@ -14,7 +14,7 @@ import scala.util.Properties
 object RunSpark {
 
   def run(
-    build: Build.Successful,
+    builds: Seq[Build.Successful],
     mainClass: String,
     args: Seq[String],
     submitArgs: Seq[String],
@@ -27,10 +27,10 @@ object RunSpark {
     // FIXME Get Spark.sparkModules via provided settings?
     val providedModules = Spark.sparkModules
     val providedFiles =
-      value(PackageCmd.providedFiles(build, providedModules, logger)).toSet
-    val depCp        = build.dependencyClassPath.filterNot(providedFiles)
-    val javaHomeInfo = build.options.javaHome().value
-    val javaOpts     = build.options.javaOptions.javaOpts.toSeq.map(_.value.value)
+      value(PackageCmd.providedFiles(builds, providedModules, logger)).toSet
+    val depCp        = builds.flatMap(_.dependencyClassPath).distinct.filterNot(providedFiles)
+    val javaHomeInfo = builds.head.options.javaHome().value
+    val javaOpts     = builds.head.options.javaOptions.javaOpts.toSeq.map(_.value.value)
     val ext          = if (Properties.isWin) ".cmd" else ""
     val submitCommand: String =
       EnvVar.Spark.sparkHome.valueOpt
@@ -44,28 +44,25 @@ object RunSpark {
       else Seq("--jars", depCp.mkString(","))
 
     scratchDirOpt.foreach(os.makeDir.all(_))
-    val library = Library.libraryJar(build)
+    val libraries = builds.map(Library.libraryJar(_))
 
     val finalCommand =
       Seq(submitCommand, "--class", mainClass) ++
         jarsArgs ++
         javaOpts.flatMap(opt => Seq("--driver-java-options", opt)) ++
         submitArgs ++
-        Seq(library.toString) ++
+        libraries.map(_.toString) ++
         args
     val envUpdates = javaHomeInfo.envUpdates(sys.env)
-    if (showCommand)
-      Left(Runner.envCommand(envUpdates) ++ finalCommand)
+    if showCommand then Left(Runner.envCommand(envUpdates) ++ finalCommand)
     else {
       val proc =
-        if (allowExecve)
+        if allowExecve then
           Runner.maybeExec("spark-submit", finalCommand, logger, extraEnv = envUpdates)
-        else
-          Runner.run(finalCommand, logger, extraEnv = envUpdates)
+        else Runner.run(finalCommand, logger, extraEnv = envUpdates)
       Right((
         proc,
-        if (scratchDirOpt.isEmpty) Some(() => os.remove(library, checkExists = true))
-        else None
+        if scratchDirOpt.isEmpty then Some(() => libraries.foreach(l => os.remove(l, checkExists = true))) else None
       ))
     }
   }
@@ -83,7 +80,7 @@ object RunSpark {
 
     // FIXME Get Spark.sparkModules via provided settings?
     val providedModules = Spark.sparkModules
-    val sparkClassPath  = value(PackageCmd.providedFiles(build, providedModules, logger))
+    val sparkClassPath  = value(PackageCmd.providedFiles(Seq(build), providedModules, logger)) // TODO: handle multiple builds
 
     scratchDirOpt.foreach(os.makeDir.all(_))
     val library = Library.libraryJar(build)