Storing JAR and POM files now also possible for the retrieve command

Author: Johannes Düsing

src/main/scala/de/upb/cs/swt/delphi/cli/DelphiCLI.scala

@@ -91,7 +91,18 @@ object DelphiCLI {
             arg[String]("id").action((x, c) => c.copy(id = x)).text("The ID of the project to retrieve"),
             opt[String]("csv").action((x, c) => c.copy(csv = x)).text("Path to the output .csv file (overwrites existing file)"),
             opt[Unit]('f', "file").action((_, c) => c.copy(opts = List("file"))).text("Use to load the ID from file, " +
-              "with the filepath given in place of the ID")
+              "with the filepath given in place of the ID"),
+            opt[String](name = "output")
+              .validate(x => if (Files.isDirectory(Paths.get(x))) success else failure(f"Output directory not found at $x"))
+              .action((x, c) => c.copy(output = x))
+              .text("Directory to write the result to"),
+            opt[String](name = "outputmode")
+              .validate(x => OutputMode.fromString(x) match {
+                case Some(_) => success
+                case None => failure("Only JarOnly, PomOnly and All are supported for output modes.")
+              })
+              .action((x, c) => c.copy(outputMode = OutputMode.fromString(x)))
+              .text("Defines what to store. Supported are JarOnly, PomOnly and All. Defaults to PomOnly. Requires output to be set.")
           )
 
         cmd("search").action((s, c) => c.copy(mode = "search"))

src/main/scala/de/upb/cs/swt/delphi/cli/FileOutput.scala

@@ -19,41 +19,60 @@ import java.io.{BufferedWriter, FileOutputStream, FileWriter}
 import java.nio.file.{Files, Paths}
 
 import com.softwaremill.sttp._
-import de.upb.cs.swt.delphi.cli.artifacts.{QueryStorageMetadata, SearchResult}
+import de.upb.cs.swt.delphi.cli.artifacts.{QueryStorageMetadata, Result, RetrieveResult, SearchResult}
 import org.joda.time.DateTime
 import org.joda.time.format.DateTimeFormat
 import spray.json._
-
 import de.upb.cs.swt.delphi.cli.artifacts.StorageMetadataJson.queryStorageMetadataFormat
 
 class FileOutput (serverVersion: String = "UNKNOWN")(implicit config:Config,  backend: SttpBackend[Id, Nothing]){
 
-  def writeSearchResults(results: List[SearchResult]) : Unit = {
+  def writeQueryResults(results: List[SearchResult]): Unit = {
     val metadata = buildQueryMetadata(results)
 
     val folderPath = Paths.get(config.output, DateTimeFormat.forPattern("YYYY-MM-dd_HH_mm_ss").print(metadata.timestamp))
     Files.createDirectory(folderPath)
 
-    val metadataPath = Paths.get(folderPath.toString, "query-metadata.json").toString
+    downloadResultFiles(results, metadata, folderPath.toString)
+  }
+
+  def writeRetrieveResults(results: Seq[RetrieveResult]): Unit = {
+    val metadata = buildRetrieveMetadata(results)
+    val first = results.head
+
+    val timestamp = DateTimeFormat.forPattern("YYYY-MM-dd_HH_mm_ss").print(metadata.timestamp)
+    val folderPath = Paths.get(config.output, s"${first.metadata.artifactId}-${first.metadata.version}-$timestamp")
+    Files.createDirectory(folderPath)
+
+    downloadResultFiles(results, metadata, folderPath.toString)
+  }
+
+
+  private def downloadResultFiles(results: Seq[Result], metadata: QueryStorageMetadata, folderPath: String) : Unit = {
+    // Write Metadata first
+    val metadataPath = Paths.get(folderPath, "query-metadata.json").toString
     val writer = new BufferedWriter(new FileWriter(metadataPath))
     writer.write(metadata.toJson.prettyPrint)
     writer.close()
 
     val outputMode = config.outputMode.getOrElse(OutputMode.PomOnly)
-    var progressCnt = 0f
 
     info()
-    info(f"Output Mode is ${outputMode.toString}, destination is ${folderPath.toString}")
+    outputMode match {
+      case OutputMode.PomOnly => info(f"All associated POM files will be stored in $folderPath")
+      case OutputMode.JarOnly => info(f"All associated JAR files will be stored in $folderPath")
+      case _ => info(f"All associated JAR and POM files will be stored in $folderPath")
+    }
+    var progressCnt = 0f
+
     info()
     print("Downloading files: 00 %")
 
     results
       .map(r => r.toMavenRelativeUrl() + s"/${r.metadata.artifactId}-${r.metadata.version}")
       .map(relUrl =>  "https://repo1.maven.org/maven2/" + relUrl).foreach( urlWithoutExtension => {
 
-      print("\b\b\b\b")
-      val progressValue = (100f * progressCnt ).toInt / results.size
-      print(s"${if (progressValue < 10) f"0$progressValue" else progressValue} %")
+      writeProgressValue((100f * progressCnt ).toInt / results.size)
       progressCnt += 1
 
       var artifactsToRetrieve = Seq[String]()
@@ -65,15 +84,13 @@ class FileOutput (serverVersion: String = "UNKNOWN")(implicit config:Config,  ba
       }
       artifactsToRetrieve.foreach( url => {
         sttp.get(uri"$url").response(asByteArray).send().body match {
-          case Right(value) =>
-            new FileOutputStream(Paths.get(folderPath.toString, url.splitAt(url.lastIndexOf('/'))._2).toString)
-              .write(value)
-          case Left(value) =>
-            error(f"Failed to download artifact from $url, got: $value")
+          case Right(value) => new FileOutputStream(Paths.get(folderPath, url.splitAt(url.lastIndexOf('/'))._2).toString)
+            .write(value)
+          case Left(value) => error(f"Failed to download artifact from $url, got: $value")
         }
       })
     })
-    print("\b\b\b\b100 %")
+    writeProgressValue(100)
     info()
     info()
     info(f"Successfully wrote results to $folderPath.")
@@ -82,7 +99,10 @@ class FileOutput (serverVersion: String = "UNKNOWN")(implicit config:Config,  ba
   private def info(value: String = ""):Unit = config.consoleOutput.outputInformation(value)
   private def error(value: String = ""):Unit = config.consoleOutput.outputError(value)
 
-
+  private def writeProgressValue(progressValue: Int): Unit = {
+    print("\b\b\b\b")
+    print(s"${if (progressValue < 10) f"0$progressValue" else progressValue} %")
+  }
   private def buildQueryMetadata(results: List[SearchResult]) =
     QueryStorageMetadata( query = config.query,
       results = results,
@@ -93,4 +113,15 @@ class FileOutput (serverVersion: String = "UNKNOWN")(implicit config:Config,  ba
       resultLimit = config.limit.getOrElse(50),
       outputMode = config.outputMode.getOrElse(OutputMode.PomOnly).toString
     )
+
+  private def buildRetrieveMetadata(results: Seq[RetrieveResult]) =
+    QueryStorageMetadata(query = f"Retrieve ${config.id}",
+      results = results,
+      serverVersion = serverVersion,
+      serverUrl = config.server,
+      clientVersion = BuildInfo.version,
+      timestamp = DateTime.now(),
+      resultLimit = 1,
+      outputMode = config.outputMode.getOrElse(OutputMode.PomOnly).toString
+    )
 }

src/main/scala/de/upb/cs/swt/delphi/cli/artifacts/SearchResult.scala

@@ -16,7 +16,7 @@
 
 package de.upb.cs.swt.delphi.cli.artifacts
 
-import spray.json.DefaultJsonProtocol
+import spray.json.{DefaultJsonProtocol, JsValue, RootJsonFormat}
 
 trait Result{
   val id: String
@@ -49,4 +49,13 @@ object SearchResultJson extends DefaultJsonProtocol {
   implicit val artifactFormat = jsonFormat5(ArtifactMetadata)
   implicit val searchResultFormat = jsonFormat3(SearchResult)
   implicit val retrieveResultFormat = jsonFormat3(RetrieveResult)
+
+  implicit object ResultJsonObject extends RootJsonFormat[Result] {
+    override def read(json: JsValue): Result = searchResultFormat.read(json)
+
+    override def write(obj: Result): JsValue = obj match {
+      case x: SearchResult => searchResultFormat.write(x)
+      case x: RetrieveResult => retrieveResultFormat.write(x)
+    }
+  }
 }

src/main/scala/de/upb/cs/swt/delphi/cli/artifacts/StorageMetadata.scala

@@ -18,7 +18,7 @@ package de.upb.cs.swt.delphi.cli.artifacts
 
 import org.joda.time.DateTime
 import spray.json.{DefaultJsonProtocol, JsonFormat}
-import de.upb.cs.swt.delphi.cli.artifacts.SearchResultJson.searchResultFormat
+import de.upb.cs.swt.delphi.cli.artifacts.SearchResultJson.ResultJsonObject
 import de.upb.cs.swt.delphi.cli.artifacts.SearchResultsJson.DateJsonFormat
 
 trait StorageMetadata {
@@ -30,7 +30,7 @@ trait StorageMetadata {
 }
 
 case class QueryStorageMetadata(query: String,
-                                results: Seq[SearchResult],
+                                results: Seq[Result],
                                 resultLimit: Int,
                                 clientVersion: String,
                                 serverVersion: String,

src/main/scala/de/upb/cs/swt/delphi/cli/commands/RetrieveCommand.scala

@@ -62,6 +62,13 @@ object RetrieveCommand extends Command {
           information.apply("Results written to file '" + config.csv + "'")
         }
       }
+      if(!config.output.equals("")){
+        val jsonArr = s.parseJson.asInstanceOf[JsArray].elements
+        val retrieveResults = jsonArr.map(r => r.convertTo[RetrieveResult])
+
+        new FileOutput(executeGet(Seq("version")).getOrElse("UNKNOWN"))
+          .writeRetrieveResults(retrieveResults)
+      }
     })
   }
 }

src/main/scala/de/upb/cs/swt/delphi/cli/commands/SearchCommand.scala

@@ -106,7 +106,7 @@ object SearchCommand extends Command with DefaultJsonProtocol{
 
       if (!config.output.equals("")){
         val output = new FileOutput(executeGet(Seq("version")).getOrElse("UNKNOWN"))
-        output.writeSearchResults(sr)
+        output.writeQueryResults(sr)
       }
 
       if (!config.csv.equals("")) {