Added CLI options to download JAR and POM files for search results

src/main/scala/de/upb/cs/swt/delphi/cli/Config.scala

@@ -16,6 +16,8 @@
 
 package de.upb.cs.swt.delphi.cli
 
+import de.upb.cs.swt.delphi.cli.OutputMode.OutputMode
+
 /**
   * Represents a configuration for the Delphi CLI
   *
@@ -27,6 +29,8 @@ case class Config(server: String = sys.env.getOrElse("DELPHI_SERVER", "https://d
                   verbose: Boolean = false,
                   raw: Boolean = false,
                   csv: String = "",
+                  output: String = "",
+                  outputMode: Option[OutputMode] = None,
                   silent: Boolean = false,
                   list : Boolean = false,
                   mode: String = "",
@@ -41,3 +45,15 @@ case class Config(server: String = sys.env.getOrElse("DELPHI_SERVER", "https://d
   lazy val csvOutput = new CsvOutput(this)
 
 }
+
+object OutputMode extends Enumeration {
+  type OutputMode = Value
+  val JarOnly, PomOnly, All = Value
+
+  def fromString(value:String): Option[OutputMode] = value.toLowerCase match {
+    case "jaronly" => Some(JarOnly)
+    case "pomonly" => Some(PomOnly)
+    case "all" => Some(All)
+    case _ => None
+  }
+}

src/main/scala/de/upb/cs/swt/delphi/cli/DelphiCLI.scala

@@ -16,6 +16,8 @@
 
 package de.upb.cs.swt.delphi.cli
 
+import java.nio.file.{Files, Paths}
+
 import com.softwaremill.sttp._
 import de.upb.cs.swt.delphi.cli.commands._
 
@@ -89,7 +91,18 @@ object DelphiCLI {
             arg[String]("id").action((x, c) => c.copy(id = x)).text("The ID of the project to retrieve"),
             opt[String]("csv").action((x, c) => c.copy(csv = x)).text("Path to the output .csv file (overwrites existing file)"),
             opt[Unit]('f', "file").action((_, c) => c.copy(opts = List("file"))).text("Use to load the ID from file, " +
-              "with the filepath given in place of the ID")
+              "with the filepath given in place of the ID"),
+            opt[String](name = "output")
+              .validate(x => if (Files.isDirectory(Paths.get(x))) success else failure(f"Output directory not found at $x"))
+              .action((x, c) => c.copy(output = x))
+              .text("Directory to write the result to"),
+            opt[String](name = "outputmode")
+              .validate(x => OutputMode.fromString(x) match {
+                case Some(_) => success
+                case None => failure("Only JarOnly, PomOnly and All are supported for output modes.")
+              })
+              .action((x, c) => c.copy(outputMode = OutputMode.fromString(x)))
+              .text("Defines what to store. Supported are JarOnly, PomOnly and All. Defaults to PomOnly. Requires output to be set.")
           )
 
         cmd("search").action((s, c) => c.copy(mode = "search"))
@@ -99,7 +112,18 @@ object DelphiCLI {
             opt[String]("csv").action((x, c) => c.copy(csv = x)).text("Path to the output .csv file (overwrites existing file)"),
             opt[Int]("limit").action((x, c) => c.copy(limit = Some(x))).text("The maximal number of results returned."),
             opt[Unit](name = "list").action((_, c) => c.copy(list = true)).text("Output results as list (raw option overrides this)"),
-            opt[Int]("timeout").action((x, c) => c.copy(timeout = Some(x))).text("Timeout in seconds.")
+            opt[Int]("timeout").action((x, c) => c.copy(timeout = Some(x))).text("Timeout in seconds."),
+            opt[String](name = "output")
+              .validate(x => if (Files.isDirectory(Paths.get(x))) success else failure(f"Output directory not found at $x"))
+              .action((x, c) => c.copy(output = x))
+              .text("Directory to write the search results to"),
+            opt[String](name = "outputmode")
+              .validate(x => OutputMode.fromString(x) match {
+                case Some(_) => success
+                case None => failure("Only JarOnly, PomOnly and All are supported for output modes.")
+              })
+              .action((x, c) => c.copy(outputMode = OutputMode.fromString(x)))
+              .text("Defines what to store. Supported are JarOnly, PomOnly and All. Defaults to PomOnly. Requires output to be set.")
           )
       }
     }

src/main/scala/de/upb/cs/swt/delphi/cli/FileOutput.scala

@@ -0,0 +1,127 @@
+// Copyright (C) 2018 The Delphi Team.
+// See the LICENCE file distributed with this work for additional
+// information regarding copyright ownership.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package de.upb.cs.swt.delphi.cli
+
+import java.io.{BufferedWriter, FileOutputStream, FileWriter}
+import java.nio.file.{Files, Paths}
+
+import com.softwaremill.sttp._
+import de.upb.cs.swt.delphi.cli.artifacts.{QueryStorageMetadata, Result, RetrieveResult, SearchResult}
+import org.joda.time.DateTime
+import org.joda.time.format.DateTimeFormat
+import spray.json._
+import de.upb.cs.swt.delphi.cli.artifacts.StorageMetadataJson.queryStorageMetadataFormat
+
+class FileOutput (serverVersion: String = "UNKNOWN")(implicit config:Config,  backend: SttpBackend[Id, Nothing]){
+
+  def writeQueryResults(results: List[SearchResult]): Unit = {
+    val metadata = buildQueryMetadata(results)
+
+    val folderPath = Paths.get(config.output, DateTimeFormat.forPattern("YYYY-MM-dd_HH_mm_ss").print(metadata.timestamp))
+    Files.createDirectory(folderPath)
+
+    downloadResultFiles(results, metadata, folderPath.toString)
+  }
+
+  def writeRetrieveResults(results: Seq[RetrieveResult]): Unit = {
+    val metadata = buildRetrieveMetadata(results)
+    val first = results.head
+
+    val timestamp = DateTimeFormat.forPattern("YYYY-MM-dd_HH_mm_ss").print(metadata.timestamp)
+    val folderPath = Paths.get(config.output, s"${first.metadata.artifactId}-${first.metadata.version}-$timestamp")
+    Files.createDirectory(folderPath)
+
+    downloadResultFiles(results, metadata, folderPath.toString)
+  }
+
+
+  private def downloadResultFiles(results: Seq[Result], metadata: QueryStorageMetadata, folderPath: String) : Unit = {
+    // Write Metadata first
+    val metadataPath = Paths.get(folderPath, "query-metadata.json").toString
+    val writer = new BufferedWriter(new FileWriter(metadataPath))
+    writer.write(metadata.toJson.prettyPrint)
+    writer.close()
+
+    val outputMode = config.outputMode.getOrElse(OutputMode.PomOnly)
+
+    info()
+    outputMode match {
+      case OutputMode.PomOnly => info(f"All associated POM files will be stored in $folderPath")
+      case OutputMode.JarOnly => info(f"All associated JAR files will be stored in $folderPath")
+      case _ => info(f"All associated JAR and POM files will be stored in $folderPath")
+    }
+    var progressCnt = 0f
+
+    info()
+    print("Downloading files: 00 %")
+
+    results
+      .map(r => r.toMavenRelativeUrl() + s"/${r.metadata.artifactId}-${r.metadata.version}")
+      .map(relUrl =>  "https://repo1.maven.org/maven2/" + relUrl).foreach( urlWithoutExtension => {
+
+      writeProgressValue((100f * progressCnt ).toInt / results.size)
+      progressCnt += 1
+
+      var artifactsToRetrieve = Seq[String]()
+      if (outputMode == OutputMode.PomOnly || outputMode == OutputMode.All){
+        artifactsToRetrieve = artifactsToRetrieve ++ Seq(s"$urlWithoutExtension.pom")
+      }
+      if(outputMode == OutputMode.JarOnly || outputMode == OutputMode.All){
+        artifactsToRetrieve = artifactsToRetrieve ++ Seq(s"$urlWithoutExtension.jar")
+      }
+      artifactsToRetrieve.foreach( url => {
+        sttp.get(uri"$url").response(asByteArray).send().body match {
+          case Right(value) => new FileOutputStream(Paths.get(folderPath, url.splitAt(url.lastIndexOf('/'))._2).toString)
+            .write(value)
+          case Left(value) => error(f"Failed to download artifact from $url, got: $value")
+        }
+      })
+    })
+    writeProgressValue(100)
+    info()
+    info()
+    info(f"Successfully wrote results to $folderPath.")
+  }
+
+  private def info(value: String = ""):Unit = config.consoleOutput.outputInformation(value)
+  private def error(value: String = ""):Unit = config.consoleOutput.outputError(value)
+
+  private def writeProgressValue(progressValue: Int): Unit = {
+    print("\b\b\b\b")
+    print(s"${if (progressValue < 10) f"0$progressValue" else progressValue} %")
+  }
+  private def buildQueryMetadata(results: List[SearchResult]) =
+    QueryStorageMetadata( query = config.query,
+      results = results,
+      serverVersion = serverVersion,
+      serverUrl = config.server,
+      clientVersion = BuildInfo.version,
+      timestamp = DateTime.now(),
+      resultLimit = config.limit.getOrElse(50),
+      outputMode = config.outputMode.getOrElse(OutputMode.PomOnly).toString
+    )
+
+  private def buildRetrieveMetadata(results: Seq[RetrieveResult]) =
+    QueryStorageMetadata(query = f"Retrieve ${config.id}",
+      results = results,
+      serverVersion = serverVersion,
+      serverUrl = config.server,
+      clientVersion = BuildInfo.version,
+      timestamp = DateTime.now(),
+      resultLimit = 1,
+      outputMode = config.outputMode.getOrElse(OutputMode.PomOnly).toString
+    )
+}

src/main/scala/de/upb/cs/swt/delphi/cli/artifacts/SearchResult.scala

@@ -16,7 +16,7 @@
 
 package de.upb.cs.swt.delphi.cli.artifacts
 
-import spray.json.DefaultJsonProtocol
+import spray.json.{DefaultJsonProtocol, JsValue, RootJsonFormat}
 
 trait Result{
   val id: String
@@ -25,6 +25,9 @@ trait Result{
 
   def toMavenIdentifier() : String = s"${metadata.groupId}:${metadata.artifactId}:${metadata.version}"
 
+  def toMavenRelativeUrl(): String =
+    s"${metadata.groupId.replace(".", "/")}/${metadata.artifactId}/${metadata.version}"
+
   def fieldNames() : List[String] = metricResults.keys.toList.sorted
 }
 
@@ -46,4 +49,13 @@ object SearchResultJson extends DefaultJsonProtocol {
   implicit val artifactFormat = jsonFormat5(ArtifactMetadata)
   implicit val searchResultFormat = jsonFormat3(SearchResult)
   implicit val retrieveResultFormat = jsonFormat3(RetrieveResult)
+
+  implicit object ResultJsonObject extends RootJsonFormat[Result] {
+    override def read(json: JsValue): Result = searchResultFormat.read(json)
+
+    override def write(obj: Result): JsValue = obj match {
+      case x: SearchResult => searchResultFormat.write(x)
+      case x: RetrieveResult => retrieveResultFormat.write(x)
+    }
+  }
 }

src/main/scala/de/upb/cs/swt/delphi/cli/artifacts/StorageMetadata.scala

@@ -0,0 +1,43 @@
+// Copyright (C) 2018 The Delphi Team.
+// See the LICENCE file distributed with this work for additional
+// information regarding copyright ownership.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package de.upb.cs.swt.delphi.cli.artifacts
+
+import org.joda.time.DateTime
+import spray.json.{DefaultJsonProtocol, JsonFormat}
+import de.upb.cs.swt.delphi.cli.artifacts.SearchResultJson.ResultJsonObject
+import de.upb.cs.swt.delphi.cli.artifacts.SearchResultsJson.DateJsonFormat
+
+trait StorageMetadata {
+  val clientVersion: String
+  val serverVersion: String
+  val serverUrl: String
+  val timestamp: DateTime
+  val outputMode: String
+}
+
+case class QueryStorageMetadata(query: String,
+                                results: Seq[Result],
+                                resultLimit: Int,
+                                clientVersion: String,
+                                serverVersion: String,
+                                serverUrl: String,
+                                outputMode: String,
+                                timestamp: DateTime) extends StorageMetadata
+
+object StorageMetadataJson extends DefaultJsonProtocol {
+  implicit val queryStorageMetadataFormat: JsonFormat[QueryStorageMetadata] = jsonFormat8(QueryStorageMetadata)
+}

src/main/scala/de/upb/cs/swt/delphi/cli/commands/RetrieveCommand.scala

@@ -62,6 +62,13 @@ object RetrieveCommand extends Command {
           information.apply("Results written to file '" + config.csv + "'")
         }
       }
+      if(!config.output.equals("")){
+        val jsonArr = s.parseJson.asInstanceOf[JsArray].elements
+        val retrieveResults = jsonArr.map(r => r.convertTo[RetrieveResult])
+
+        new FileOutput(executeGet(Seq("version")).getOrElse("UNKNOWN"))
+          .writeRetrieveResults(retrieveResults)
+      }
     })
   }
 }

src/main/scala/de/upb/cs/swt/delphi/cli/commands/SearchCommand.scala

@@ -20,8 +20,8 @@ import java.util.concurrent.TimeUnit
 
 import com.softwaremill.sttp._
 import com.softwaremill.sttp.sprayJson._
-import de.upb.cs.swt.delphi.cli.Config
-import de.upb.cs.swt.delphi.cli.artifacts.{SearchResult, SearchResults}
+import de.upb.cs.swt.delphi.cli.{Config, FileOutput}
+import de.upb.cs.swt.delphi.cli.artifacts.SearchResults
 import de.upb.cs.swt.delphi.cli.artifacts.SearchResultsJson._
 import spray.json._
 
@@ -75,7 +75,8 @@ object SearchCommand extends Command with DefaultJsonProtocol{
     (resStr, took)
   }
 
-  private def processResults(res: String, queryRuntime: FiniteDuration)(implicit config: Config) = {
+  private def processResults(res: String, queryRuntime: FiniteDuration)
+                            (implicit config: Config, backend: SttpBackend[Id, Nothing]) = {
 
     if (config.raw || res.equals("")) {
       reportResult.apply(res)
@@ -103,6 +104,11 @@ object SearchCommand extends Command with DefaultJsonProtocol{
 
       information.apply(f"Query roundtrip took ${queryRuntime.toUnit(TimeUnit.MILLISECONDS)}%.0fms.")
 
+      if (!config.output.equals("")){
+        val output = new FileOutput(executeGet(Seq("version")).getOrElse("UNKNOWN"))
+        output.writeQueryResults(sr)
+      }
+
       if (!config.csv.equals("")) {
         exportResult.apply(sr)
         information.apply("Results written to file '" + config.csv + "'")