From b2d7579282095f5f071976490caa2f82c197bc4d Mon Sep 17 00:00:00 2001 From: Johannes Duesing Date: Tue, 7 Aug 2018 16:52:28 +0200 Subject: [PATCH 01/43] Adapted configuration file to support Instance Registry --- .../instancemanagement/InstanceRegistry.scala | 25 +++++++++++++++++++ .../cs/swt/delphi/webapi/Configuration.scala | 20 +++++++++++++-- 2 files changed, 43 insertions(+), 2 deletions(-) create mode 100644 src/main/scala/de/upb/cs/swt/delphi/instancemanagement/InstanceRegistry.scala diff --git a/src/main/scala/de/upb/cs/swt/delphi/instancemanagement/InstanceRegistry.scala b/src/main/scala/de/upb/cs/swt/delphi/instancemanagement/InstanceRegistry.scala new file mode 100644 index 0000000..ed0b0df --- /dev/null +++ b/src/main/scala/de/upb/cs/swt/delphi/instancemanagement/InstanceRegistry.scala @@ -0,0 +1,25 @@ +package de.upb.cs.swt.delphi.instancemanagement + +import de.upb.cs.swt.delphi.webapi.Configuration + +import scala.util.{Failure, Success, Try} + +object InstanceRegistry { + def register(Name: String, configuration: Configuration) : Try[Configuration] = { + //TODO: Call generated API here + Success(configuration) + } + + def retrieveElasticSearchInstance(configuration: Configuration) : Try[String] = { + if(!configuration.usingInstanceRegistry) Failure + //TODO: Call generated API here + //TODO: Where to call sendMatchingResult ? + Success("") + } + + def sendMatchingResult(isElasticSearchReachable : Boolean, configuration: Configuration) : Try[Unit] = { + if(!configuration.usingInstanceRegistry) Failure + //TODO: Call generated API here + Success() + } +} diff --git a/src/main/scala/de/upb/cs/swt/delphi/webapi/Configuration.scala b/src/main/scala/de/upb/cs/swt/delphi/webapi/Configuration.scala index 53ed334..0fa3e5f 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/webapi/Configuration.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/webapi/Configuration.scala @@ -2,6 +2,9 @@ package de.upb.cs.swt.delphi.webapi import com.sksamuel.elastic4s.{ElasticsearchClientUri, IndexAndType} import com.sksamuel.elastic4s.http.ElasticDsl._ +import de.upb.cs.swt.delphi.instancemanagement.InstanceRegistry + +import scala.util.{Failure, Success} /** * @author Ben Hermann @@ -9,12 +12,25 @@ import com.sksamuel.elastic4s.http.ElasticDsl._ class Configuration( //Server and Elasticsearch configuration val bindHost: String = "0.0.0.0", val bindPort: Int = 8080, - val elasticsearchClientUri: ElasticsearchClientUri = ElasticsearchClientUri( - sys.env.getOrElse("DELPHI_ELASTIC_URI", "elasticsearch://localhost:9200")), val esProjectIndex: IndexAndType = "delphi" / "project", //Actor system configuration val elasticActorPoolSize: Int = 8 ) { + + lazy val elasticsearchClientUri: ElasticsearchClientUri = ElasticsearchClientUri(InstanceRegistry.retrieveElasticSearchInstance(this) match { + case Success(elasticIP) => elasticIP + case Failure(_) => sys.env.getOrElse("DELPHI_ELASTIC_URI","elasticsearch://localhost:9200") + }) + + val instanceRegistryUri : String = sys.env.getOrElse("DELPHI_IR_URI", "http://localhost:9300") + + lazy val usingInstanceRegistry = InstanceRegistry.register("MyWebApiInstance",this) match { + case Success(_) => true + case Failure(_) => { + println(s"Failed to connect to Instance Registry at ${instanceRegistryUri}. Using default configuration instead.") + false + } + } } From 59c21b60f07fc95a9be11a5c84bb67691d98d22b Mon Sep 17 00:00:00 2001 From: Ben Hermann Date: Sat, 11 Aug 2018 17:52:07 +0200 Subject: [PATCH 02/43] Added synk badge to README --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 4c912c8..5c5b545 100644 --- a/README.md +++ b/README.md @@ -5,10 +5,10 @@ The web API implementation for the Delphi platform. We are currently in pre-alpha state! There is no release and the code in this repository is purely experimental! -|branch | status | codacy | -| :---: | :---: | :---: | -| master | [![Build Status](https://travis-ci.org/delphi-hub/delphi-webapi.svg?branch=master)](https://travis-ci.org/delphi-hub/delphi-webapi) | [![Codacy Badge](https://api.codacy.com/project/badge/Grade/8ebe27850ffb4139af6280fd1cd6d540)](https://www.codacy.com/app/delphi-hub/delphi-webapi?utm_source=github.com&utm_medium=referral&utm_content=delphi-hub/delphi-webapi&utm_campaign=Badge_Grade)| -| develop | [![Build Status](https://travis-ci.org/delphi-hub/delphi-webapi.svg?branch=develop)](https://travis-ci.org/delphi-hub/delphi-webapi) | [![Codacy Badge](https://api.codacy.com/project/badge/Grade/8ebe27850ffb4139af6280fd1cd6d540?branch=develop)](https://www.codacy.com/app/delphi-hub/delphi-webapi?branch=develop&utm_source=github.com&utm_medium=referral&utm_content=delphi-hub/delphi-webapi&utm_campaign=Badge_Grade) | +|branch | status | codacy | snyk | +| :---: | :---: | :---: | :---: | +| master | [![Build Status](https://travis-ci.org/delphi-hub/delphi-webapi.svg?branch=master)](https://travis-ci.org/delphi-hub/delphi-webapi) | [![Codacy Badge](https://api.codacy.com/project/badge/Grade/8ebe27850ffb4139af6280fd1cd6d540)](https://www.codacy.com/app/delphi-hub/delphi-webapi?utm_source=github.com&utm_medium=referral&utm_content=delphi-hub/delphi-webapi&utm_campaign=Badge_Grade)| [![Known Vulnerabilities](https://snyk.io/test/github/delphi-hub/delphi-webapi/badge.svg?targetFile=build.sbt)](https://snyk.io/test/github/delphi-hub/delphi-webapi/?targetFile=build.sbt) | +| develop | [![Build Status](https://travis-ci.org/delphi-hub/delphi-webapi.svg?branch=develop)](https://travis-ci.org/delphi-hub/delphi-webapi) | [![Codacy Badge](https://api.codacy.com/project/badge/Grade/8ebe27850ffb4139af6280fd1cd6d540?branch=develop)](https://www.codacy.com/app/delphi-hub/delphi-webapi?branch=develop&utm_source=github.com&utm_medium=referral&utm_content=delphi-hub/delphi-webapi&utm_campaign=Badge_Grade) | [![Known Vulnerabilities](https://snyk.io/test/github/delphi-hub/delphi-webapi/develop/badge.svg?targetFile=build.sbt)](https://snyk.io/test/github/delphi-hub/delphi-webapi/develop/?targetFile=build.sbt) | ## What is the Delphi Web API? From 778090d1049c0e2d3f1f033be76eacf2ffb7e36c Mon Sep 17 00:00:00 2001 From: Hariharan Ramanathan Date: Tue, 14 Aug 2018 21:19:05 +0200 Subject: [PATCH 03/43] Extracting source object from es response --- .../cs/swt/delphi/webapi/ElasticActor.scala | 28 +++++++++++++------ 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/src/main/scala/de/upb/cs/swt/delphi/webapi/ElasticActor.scala b/src/main/scala/de/upb/cs/swt/delphi/webapi/ElasticActor.scala index c4d4dcd..f94e2a3 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/webapi/ElasticActor.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/webapi/ElasticActor.scala @@ -6,16 +6,19 @@ import com.sksamuel.elastic4s.http.{ElasticClient, RequestFailure, RequestSucces import com.sksamuel.elastic4s.http.ElasticDsl._ import de.upb.cs.swt.delphi.webapi.ElasticActorManager.{Enqueue, Retrieve} +import spray.json._ import scala.concurrent.ExecutionContext import scala.concurrent.duration._ -class ElasticActor(configuration: Configuration, index: IndexAndType) extends Actor with ActorLogging{ +class ElasticActor(configuration: Configuration, index: IndexAndType) extends Actor with ActorLogging { implicit val executionContext: ExecutionContext = context.system.dispatchers.lookup("elasticsearch-handling-dispatcher") val client = ElasticClient(configuration.elasticsearchClientUri) override def preStart(): Unit = log.info("Search actor started") + override def postStop(): Unit = log.info("Search actor shut down") + context.setReceiveTimeout(2 seconds) override def receive = { @@ -25,22 +28,31 @@ class ElasticActor(configuration: Configuration, index: IndexAndType) extends Ac private def getSource(id: String) = { log.info("Executing get on entry {}", id) - def queryResponse = client.execute{ + val searchByName = searchWithType(index) query must( + // matchQuery("name", s"http://repo1.maven.org/maven2/:$id") + matchQuery("name", id) + ) + log.info(s"Query {}",client.show(searchByName)) + def queryResponse = client.execute { log.info(s"Got retrieve request for $id.") - searchWithType(index) query must ( - matchQuery("name", s"http://repo1.maven.org/maven2/:$id") - ) + searchByName }.await val source = queryResponse match { - case results: RequestSuccess[_] => results.body.get + case results: RequestSuccess[_] => { + val resObj = results.body.get.parseJson.asJsObject + val hitsObj=resObj.fields.getOrElse("hits", JsObject.empty).asJsObject + val hitsArr=hitsObj.fields.getOrElse("hits",JsArray.empty).asInstanceOf[JsArray] + val source=hitsArr.elements.map(m=>m.asJsObject.fields.get("_source")) + source.head.getOrElse(JsObject.empty).toString() + } case failure: RequestFailure => Option.empty } sender().tell(source, context.self) } } -object ElasticActor{ - def props(configuration: Configuration, index: IndexAndType) : Props = Props(new ElasticActor(configuration, index)) +object ElasticActor { + def props(configuration: Configuration, index: IndexAndType): Props = Props(new ElasticActor(configuration, index)) .withMailbox("es-priority-mailbox") } From 957bbafb7f4eb7f75602ccecfd424f67b4b970a8 Mon Sep 17 00:00:00 2001 From: Santosh <24santoshr@gmail.com> Date: Wed, 5 Sep 2018 10:35:34 +0200 Subject: [PATCH 04/43] Changes for instance registry --- .../delphi/instancemanagement/Instance.scala | 45 +++++++ .../instancemanagement/InstanceRegistry.scala | 115 +++++++++++++++--- .../cs/swt/delphi/webapi/Configuration.scala | 28 +++-- .../de/upb/cs/swt/delphi/webapi/Server.scala | 8 +- 4 files changed, 168 insertions(+), 28 deletions(-) create mode 100644 src/main/scala/de/upb/cs/swt/delphi/instancemanagement/Instance.scala diff --git a/src/main/scala/de/upb/cs/swt/delphi/instancemanagement/Instance.scala b/src/main/scala/de/upb/cs/swt/delphi/instancemanagement/Instance.scala new file mode 100644 index 0000000..5a70bda --- /dev/null +++ b/src/main/scala/de/upb/cs/swt/delphi/instancemanagement/Instance.scala @@ -0,0 +1,45 @@ +package de.upb.cs.swt.delphi.instancemanagement + +import akka.http.scaladsl.marshallers.sprayjson.SprayJsonSupport +import spray.json.{DefaultJsonProtocol, JsString, JsValue, JsonFormat} + +trait JsonSupport extends SprayJsonSupport with DefaultJsonProtocol { + implicit val componentTypeFormat = new JsonFormat[InstanceEnums.ComponentType] { + def write(compType : InstanceEnums.ComponentType) = JsString(compType.toString()) + + def read(value: JsValue) = value match { + case JsString(s) => s match { + case "Crawler" => InstanceEnums.ComponentType.Crawler + case "WebApi" => InstanceEnums.ComponentType.WebApi + case "WebApp" => InstanceEnums.ComponentType.WebApp + case "DelphiManagement" => InstanceEnums.ComponentType.DelphiManagement + case "ElasticSearch" => InstanceEnums.ComponentType.ElasticSearch + case x => throw new RuntimeException(s"Unexpected string value $x for component type.") + } + case y => throw new RuntimeException(s"Unexpected type $y while deserializing component type.") + } + } + implicit val instanceFormat = jsonFormat5(Instance) +} + +final case class Instance ( + iD: Option[Long], + iP: Option[String], + portnumber: Option[Long], + name: Option[String], + /* Component Type */ + componentType: Option[InstanceEnums.ComponentType] + ) + +object InstanceEnums { + + type ComponentType = ComponentType.Value + object ComponentType extends Enumeration { + val Crawler = Value("Crawler") + val WebApi = Value("WebApi") + val WebApp = Value("WebApp") + val DelphiManagement = Value("DelphiManagement") + val ElasticSearch = Value("ElasticSearch") + } + +} diff --git a/src/main/scala/de/upb/cs/swt/delphi/instancemanagement/InstanceRegistry.scala b/src/main/scala/de/upb/cs/swt/delphi/instancemanagement/InstanceRegistry.scala index ed0b0df..4013c81 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/instancemanagement/InstanceRegistry.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/instancemanagement/InstanceRegistry.scala @@ -1,25 +1,112 @@ package de.upb.cs.swt.delphi.instancemanagement -import de.upb.cs.swt.delphi.webapi.Configuration +import java.net.InetAddress +import akka.http.scaladsl.Http +import akka.http.scaladsl.marshalling.Marshal +import akka.http.scaladsl.model._ +import akka.http.scaladsl.unmarshalling.Unmarshal +import de.upb.cs.swt.delphi.instancemanagement.InstanceEnums.ComponentType +import de.upb.cs.swt.delphi.webapi.{AppLogging, Configuration, Server} + +import scala.concurrent.{Await, Future} +import scala.concurrent.duration.Duration import scala.util.{Failure, Success, Try} -object InstanceRegistry { - def register(Name: String, configuration: Configuration) : Try[Configuration] = { - //TODO: Call generated API here - Success(configuration) +object InstanceRegistry extends JsonSupport with AppLogging +{ + + implicit val system = Server.system + implicit val ec = system.dispatcher + implicit val materializer = Server.materializer + + + def register(configuration: Configuration) : Boolean = { + + val instance = createInstance(None,configuration.bindPort, configuration.instanceName) + + Await.result(postInstance(instance, configuration.instanceRegistryUri + "/register") map {response => + if(response.status == StatusCodes.OK){ + Await.result(Unmarshal(response.entity).to[String] map { assignedID => + val id = assignedID.toLong + log.info(s"Successfully registered at Instance Registry, got ID $id.") + true + } recover { case ex => + log.warning(s"Failed to read assigned ID from Instance Registry, exception: $ex") + false + }, Duration.Inf) + } + else { + val statuscode = response.status + log.warning(s"Failed to register at Instance Registry, server returned $statuscode") + false + } + + } recover {case ex => + log.warning(s"Failed to register at Instance Registry, exception: $ex") + false + }, Duration.Inf) } - def retrieveElasticSearchInstance(configuration: Configuration) : Try[String] = { - if(!configuration.usingInstanceRegistry) Failure - //TODO: Call generated API here - //TODO: Where to call sendMatchingResult ? - Success("") + def retrieveElasticSearchInstance(configuration: Configuration) : Try[Instance] = { + if(!configuration.usingInstanceRegistry) Failure(new RuntimeException("Cannot get ElasticSearch instance from Instance Registry, no Instance Registry available.")) + else { + val request = HttpRequest(method = HttpMethods.GET, configuration.instanceRegistryUri + "/matchingInstance?ComponentType=ElasticSearch") + + Await.result(Http(system).singleRequest(request) map {response => + val status = response.status + if(status == StatusCodes.OK) { + + Await.result(Unmarshal(response.entity).to[Instance] map {instance => + val elasticIP = instance.iP + log.info(s"Instance Registry assigned ElasticSearch instance at ${elasticIP.getOrElse("None")}") + Success(instance) + } recover {case ex => + log.warning(s"Failed to read response from Instance Registry, exception: $ex") + Failure(ex) + }, Duration.Inf) + } + else{ + log.warning(s"Failed to read response from Instance Registry, server returned $status") + Failure(new RuntimeException(s"Failed to read response from Instance Registry, server returned $status")) + } + } recover { case ex => + log.warning(s"Failed to request ElasticSearch instance from Instance Registry, exception: $ex ") + Failure(ex) + }, Duration.Inf) + } } def sendMatchingResult(isElasticSearchReachable : Boolean, configuration: Configuration) : Try[Unit] = { - if(!configuration.usingInstanceRegistry) Failure - //TODO: Call generated API here - Success() + if(!configuration.usingInstanceRegistry) Failure(new RuntimeException("Cannot post matching result to Instance Registry, no Instance Registry available.")) + if(configuration.elasticsearchInstance.iD.isEmpty) Failure(new RuntimeException("Cannot post matching result to Instance Registry, assigned ElasticSearch instance has no ID.")) + + val IdToPost = configuration.elasticsearchInstance.iD.get + val request = HttpRequest(method = HttpMethods.POST, configuration.instanceRegistryUri + s"/matchingResult?Id=$IdToPost&MatchingSuccessful=$isElasticSearchReachable") + + Await.result(Http(system).singleRequest(request) map {response => + if(response.status == StatusCodes.OK){ + log.info("Successfully posted matching result to Instance Registry.") + Success() + } + else { + val statuscode = response.status + log.warning(s"Failed to post matching result to Instance Registry, server returned $statuscode") + Failure(new RuntimeException(s"Failed to post matching result to Instance Registry, server returned $statuscode")) + } + + } recover {case ex => + log.warning(s"Failed to post matching result to Instance Registry, exception: $ex") + Failure(new RuntimeException(s"Failed to post matching result tot Instance Registry, exception: $ex")) + }, Duration.Inf) } -} + + def postInstance(instance : Instance, uri: String) () : Future[HttpResponse] = + Marshal(instance).to[RequestEntity] flatMap { entity => + val request = HttpRequest(method = HttpMethods.POST, uri = uri, entity = entity) + Http(system).singleRequest(request) + } + + + private def createInstance(id: Option[Long], controlPort : Int, name : String) : Instance = Instance(id, Option(InetAddress.getLocalHost().getHostAddress()), Option(controlPort), Option(name), Option(ComponentType.WebApi)) +} \ No newline at end of file diff --git a/src/main/scala/de/upb/cs/swt/delphi/webapi/Configuration.scala b/src/main/scala/de/upb/cs/swt/delphi/webapi/Configuration.scala index 0fa3e5f..18c5b9e 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/webapi/Configuration.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/webapi/Configuration.scala @@ -2,7 +2,8 @@ package de.upb.cs.swt.delphi.webapi import com.sksamuel.elastic4s.{ElasticsearchClientUri, IndexAndType} import com.sksamuel.elastic4s.http.ElasticDsl._ -import de.upb.cs.swt.delphi.instancemanagement.InstanceRegistry +import de.upb.cs.swt.delphi.instancemanagement.InstanceEnums.ComponentType +import de.upb.cs.swt.delphi.instancemanagement.{Instance, InstanceRegistry} import scala.util.{Failure, Success} @@ -19,18 +20,21 @@ class Configuration( //Server and Elasticsearch configuration ) { - lazy val elasticsearchClientUri: ElasticsearchClientUri = ElasticsearchClientUri(InstanceRegistry.retrieveElasticSearchInstance(this) match { - case Success(elasticIP) => elasticIP - case Failure(_) => sys.env.getOrElse("DELPHI_ELASTIC_URI","elasticsearch://localhost:9200") + lazy val elasticsearchClientUri: ElasticsearchClientUri = ElasticsearchClientUri({ + if(elasticsearchInstance.portnumber.isEmpty){ + elasticsearchInstance.iP.get + }else{ + elasticsearchInstance.iP.get + ":" + elasticsearchInstance.portnumber.get + } }) - val instanceRegistryUri : String = sys.env.getOrElse("DELPHI_IR_URI", "http://localhost:9300") - - lazy val usingInstanceRegistry = InstanceRegistry.register("MyWebApiInstance",this) match { - case Success(_) => true - case Failure(_) => { - println(s"Failed to connect to Instance Registry at ${instanceRegistryUri}. Using default configuration instead.") - false - } + lazy val elasticsearchInstance : Instance = InstanceRegistry.retrieveElasticSearchInstance(this) match { + case Success(instance) => instance + case Failure(_) => Instance(None, Some(sys.env.getOrElse("DELPHI_ELASTIC_URI","elasticsearch://localhost:9200")), None, Some("Default ElasticSearch instance"), Some(ComponentType.ElasticSearch) ) } + + val instanceName = "MyWebApiInstance" + val instanceRegistryUri : String = sys.env.getOrElse("DELPHI_IR_URI", "http://localhost:8085") + lazy val usingInstanceRegistry = InstanceRegistry.register(this) + } diff --git a/src/main/scala/de/upb/cs/swt/delphi/webapi/Server.scala b/src/main/scala/de/upb/cs/swt/delphi/webapi/Server.scala index 02e72ee..a87d14f 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/webapi/Server.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/webapi/Server.scala @@ -5,6 +5,7 @@ import java.util.concurrent.TimeUnit import akka.actor.ActorSystem import akka.http.scaladsl.server.HttpApp import akka.pattern.ask +import akka.stream.ActorMaterializer import akka.util.Timeout import de.upb.cs.swt.delphi.featuredefinitions.FeatureListMapping import de.upb.cs.swt.delphi.webapi.ElasticActorManager.{Enqueue, Retrieve} @@ -17,12 +18,14 @@ import spray.json._ object Server extends HttpApp with JsonSupport with AppLogging { private val configuration = new Configuration() - private val system = ActorSystem("delphi-webapi") + implicit val system = ActorSystem("delphi-webapi") private val actorManager = system.actorOf(ElasticActorManager.props(configuration)) private val requestLimiter = system.actorOf(ElasticRequestLimiter.props(configuration, actorManager)) implicit val timeout = Timeout(5, TimeUnit.SECONDS) + implicit val materializer = ActorMaterializer() - override def routes = + + override def routes = path("version") { version } ~ path("features") { features } ~ pathPrefix("search" / Remaining) { query => search(query) } ~ @@ -80,6 +83,7 @@ object Server extends HttpApp with JsonSupport with AppLogging { def main(args: Array[String]): Unit = { val configuration = new Configuration() + Server.startServer(configuration.bindHost, configuration.bindPort) system.terminate() } From ca82795ffed9bcbef2fe3649bbe5f856663f2d6a Mon Sep 17 00:00:00 2001 From: Santosh <24santoshr@gmail.com> Date: Wed, 5 Sep 2018 16:12:18 +0200 Subject: [PATCH 05/43] Changes for Instance Registry --- .../instancemanagement/InstanceRegistry.scala | 90 +++++++++++++------ .../cs/swt/delphi/webapi/Configuration.scala | 9 +- .../de/upb/cs/swt/delphi/webapi/Server.scala | 28 +++++- 3 files changed, 98 insertions(+), 29 deletions(-) diff --git a/src/main/scala/de/upb/cs/swt/delphi/instancemanagement/InstanceRegistry.scala b/src/main/scala/de/upb/cs/swt/delphi/instancemanagement/InstanceRegistry.scala index 4013c81..a77c8be 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/instancemanagement/InstanceRegistry.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/instancemanagement/InstanceRegistry.scala @@ -9,6 +9,7 @@ import akka.http.scaladsl.unmarshalling.Unmarshal import de.upb.cs.swt.delphi.instancemanagement.InstanceEnums.ComponentType import de.upb.cs.swt.delphi.webapi.{AppLogging, Configuration, Server} + import scala.concurrent.{Await, Future} import scala.concurrent.duration.Duration import scala.util.{Failure, Success, Try} @@ -21,8 +22,7 @@ object InstanceRegistry extends JsonSupport with AppLogging implicit val materializer = Server.materializer - def register(configuration: Configuration) : Boolean = { - + def register(configuration: Configuration) : Try[Long] = { val instance = createInstance(None,configuration.bindPort, configuration.instanceName) Await.result(postInstance(instance, configuration.instanceRegistryUri + "/register") map {response => @@ -30,27 +30,28 @@ object InstanceRegistry extends JsonSupport with AppLogging Await.result(Unmarshal(response.entity).to[String] map { assignedID => val id = assignedID.toLong log.info(s"Successfully registered at Instance Registry, got ID $id.") - true + Success(id) } recover { case ex => log.warning(s"Failed to read assigned ID from Instance Registry, exception: $ex") - false + Failure(ex) }, Duration.Inf) } else { val statuscode = response.status log.warning(s"Failed to register at Instance Registry, server returned $statuscode") - false + Failure(new RuntimeException(s"Failed to register at Instance Registry, server returned $statuscode")) } } recover {case ex => log.warning(s"Failed to register at Instance Registry, exception: $ex") - false + Failure(ex) }, Duration.Inf) } def retrieveElasticSearchInstance(configuration: Configuration) : Try[Instance] = { - if(!configuration.usingInstanceRegistry) Failure(new RuntimeException("Cannot get ElasticSearch instance from Instance Registry, no Instance Registry available.")) - else { + if(!configuration.usingInstanceRegistry) { + Failure(new RuntimeException("Cannot get ElasticSearch instance from Instance Registry, no Instance Registry available.")) + } else { val request = HttpRequest(method = HttpMethods.GET, configuration.instanceRegistryUri + "/matchingInstance?ComponentType=ElasticSearch") Await.result(Http(system).singleRequest(request) map {response => @@ -78,27 +79,61 @@ object InstanceRegistry extends JsonSupport with AppLogging } def sendMatchingResult(isElasticSearchReachable : Boolean, configuration: Configuration) : Try[Unit] = { - if(!configuration.usingInstanceRegistry) Failure(new RuntimeException("Cannot post matching result to Instance Registry, no Instance Registry available.")) - if(configuration.elasticsearchInstance.iD.isEmpty) Failure(new RuntimeException("Cannot post matching result to Instance Registry, assigned ElasticSearch instance has no ID.")) + if(!configuration.usingInstanceRegistry) { + Failure(new RuntimeException("Cannot post matching result to Instance Registry, no Instance Registry available.")) + } else { + if(configuration.elasticsearchInstance.iD.isEmpty) { + Failure(new RuntimeException("Cannot post matching result to Instance Registry, assigned ElasticSearch instance has no ID.")) + } else { + val IdToPost = configuration.elasticsearchInstance.iD.get + val request = HttpRequest( + method = HttpMethods.POST, + configuration.instanceRegistryUri + s"/matchingResult?Id=$IdToPost&MatchingSuccessful=$isElasticSearchReachable") + + Await.result(Http(system).singleRequest(request) map {response => + if(response.status == StatusCodes.OK){ + log.info("Successfully posted matching result to Instance Registry.") + Success() + } + else { + val statuscode = response.status + log.warning(s"Failed to post matching result to Instance Registry, server returned $statuscode") + Failure(new RuntimeException(s"Failed to post matching result to Instance Registry, server returned $statuscode")) + } + + } recover {case ex => + log.warning(s"Failed to post matching result to Instance Registry, exception: $ex") + Failure(new RuntimeException(s"Failed to post matching result tot Instance Registry, exception: $ex")) + }, Duration.Inf) + } + } - val IdToPost = configuration.elasticsearchInstance.iD.get - val request = HttpRequest(method = HttpMethods.POST, configuration.instanceRegistryUri + s"/matchingResult?Id=$IdToPost&MatchingSuccessful=$isElasticSearchReachable") + } - Await.result(Http(system).singleRequest(request) map {response => - if(response.status == StatusCodes.OK){ - log.info("Successfully posted matching result to Instance Registry.") - Success() - } - else { - val statuscode = response.status - log.warning(s"Failed to post matching result to Instance Registry, server returned $statuscode") - Failure(new RuntimeException(s"Failed to post matching result to Instance Registry, server returned $statuscode")) - } + def deregister(configuration: Configuration) : Try[Unit] = { + if(!configuration.usingInstanceRegistry){ + Failure(new RuntimeException("Cannot deregister from Instance Registry, no Instance Registry available.")) + } else { + val id : Long = configuration.assignedID.get - } recover {case ex => - log.warning(s"Failed to post matching result to Instance Registry, exception: $ex") - Failure(new RuntimeException(s"Failed to post matching result tot Instance Registry, exception: $ex")) - }, Duration.Inf) + val request = HttpRequest(method = HttpMethods.POST, configuration.instanceRegistryUri + s"/deregister?Id=$id") + + Await.result(Http(system).singleRequest(request) map {response => + if(response.status == StatusCodes.OK){ + log.info("Successfully deregistered from Instance Registry.") + Success() + } + else { + val statuscode = response.status + log.warning(s"Failed to deregister from Instance Registry, server returned $statuscode") + Failure(new RuntimeException(s"Failed to deregister from Instance Registry, server returned $statuscode")) + } + + } recover {case ex => + log.warning(s"Failed to deregister to Instance Registry, exception: $ex") + Failure(ex) + }, Duration.Inf) + } } def postInstance(instance : Instance, uri: String) () : Future[HttpResponse] = @@ -108,5 +143,6 @@ object InstanceRegistry extends JsonSupport with AppLogging } - private def createInstance(id: Option[Long], controlPort : Int, name : String) : Instance = Instance(id, Option(InetAddress.getLocalHost().getHostAddress()), Option(controlPort), Option(name), Option(ComponentType.WebApi)) + private def createInstance(id: Option[Long], controlPort : Int, name : String) : Instance = + Instance(id, Option(InetAddress.getLocalHost.getHostAddress), Option(controlPort), Option(name), Option(ComponentType.Crawler)) } \ No newline at end of file diff --git a/src/main/scala/de/upb/cs/swt/delphi/webapi/Configuration.scala b/src/main/scala/de/upb/cs/swt/delphi/webapi/Configuration.scala index 18c5b9e..eb9c892 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/webapi/Configuration.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/webapi/Configuration.scala @@ -35,6 +35,13 @@ class Configuration( //Server and Elasticsearch configuration val instanceName = "MyWebApiInstance" val instanceRegistryUri : String = sys.env.getOrElse("DELPHI_IR_URI", "http://localhost:8085") - lazy val usingInstanceRegistry = InstanceRegistry.register(this) + lazy val usingInstanceRegistry : Boolean = assignedID match { + case Some(_) => true + case None => false + } + lazy val assignedID : Option[Long] = InstanceRegistry.register(this) match { + case Success(id) => Some(id) + case Failure(_) => None + } } diff --git a/src/main/scala/de/upb/cs/swt/delphi/webapi/Server.scala b/src/main/scala/de/upb/cs/swt/delphi/webapi/Server.scala index a87d14f..3b34da8 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/webapi/Server.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/webapi/Server.scala @@ -7,11 +7,18 @@ import akka.http.scaladsl.server.HttpApp import akka.pattern.ask import akka.stream.ActorMaterializer import akka.util.Timeout +import com.sksamuel.elastic4s.http.ElasticClient +import com.sksamuel.elastic4s.http.ElasticDsl._ import de.upb.cs.swt.delphi.featuredefinitions.FeatureListMapping +import de.upb.cs.swt.delphi.instancemanagement.InstanceRegistry import de.upb.cs.swt.delphi.webapi.ElasticActorManager.{Enqueue, Retrieve} import de.upb.cs.swt.delphi.webapi.ElasticRequestLimiter.Validate import spray.json._ +import scala.concurrent.duration.Duration +import scala.concurrent.{Await, ExecutionContext} +import scala.util.{Failure, Success} + /** * Web server configuration for Delphi web API. */ @@ -82,9 +89,28 @@ object Server extends HttpApp with JsonSupport with AppLogging { } def main(args: Array[String]): Unit = { - val configuration = new Configuration() + + implicit val ec : ExecutionContext = system.dispatcher + lazy val client = ElasticClient(configuration.elasticsearchClientUri) + + val f = (client.execute { + nodeInfo() + } map { i => { + if(configuration.usingInstanceRegistry) InstanceRegistry.sendMatchingResult(true, configuration) + Success(configuration) + } + } recover { case e => { + if(configuration.usingInstanceRegistry) InstanceRegistry.sendMatchingResult(false, configuration) + Failure(e) + } + }).andThen { + case _ => client.close() + } + + Await.ready(f, Duration.Inf) Server.startServer(configuration.bindHost, configuration.bindPort) + InstanceRegistry.deregister(configuration) system.terminate() } From ca17f1f41fc98e045b581addbda7e9426e726e18 Mon Sep 17 00:00:00 2001 From: Santosh <24santoshr@gmail.com> Date: Sat, 8 Sep 2018 13:35:05 +0200 Subject: [PATCH 06/43] Changes for Instance registry --- .../scala/de/upb/cs/swt/delphi/webapi/Configuration.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/scala/de/upb/cs/swt/delphi/webapi/Configuration.scala b/src/main/scala/de/upb/cs/swt/delphi/webapi/Configuration.scala index eb9c892..3a41049 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/webapi/Configuration.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/webapi/Configuration.scala @@ -22,9 +22,9 @@ class Configuration( //Server and Elasticsearch configuration lazy val elasticsearchClientUri: ElasticsearchClientUri = ElasticsearchClientUri({ if(elasticsearchInstance.portnumber.isEmpty){ - elasticsearchInstance.iP.get + elasticsearchInstance.iP.getOrElse("elasticsearch://localhost:9200") }else{ - elasticsearchInstance.iP.get + ":" + elasticsearchInstance.portnumber.get + elasticsearchInstance.iP.getOrElse("elasticsearch://localhost") + ":" + elasticsearchInstance.portnumber.getOrElse("9200") } }) From 5546b3150cd105ce642f51d1f22d15110d29c9b4 Mon Sep 17 00:00:00 2001 From: Santosh <24santoshr@gmail.com> Date: Mon, 10 Sep 2018 22:29:57 +0200 Subject: [PATCH 07/43] Changes for Instance registry --- .../delphi/instancemanagement/Instance.scala | 12 ++--- .../instancemanagement/InstanceRegistry.scala | 23 ++++----- .../cs/swt/delphi/webapi/Configuration.scala | 47 ++++++++++++++----- 3 files changed, 54 insertions(+), 28 deletions(-) diff --git a/src/main/scala/de/upb/cs/swt/delphi/instancemanagement/Instance.scala b/src/main/scala/de/upb/cs/swt/delphi/instancemanagement/Instance.scala index 5a70bda..258e268 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/instancemanagement/Instance.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/instancemanagement/Instance.scala @@ -5,7 +5,7 @@ import spray.json.{DefaultJsonProtocol, JsString, JsValue, JsonFormat} trait JsonSupport extends SprayJsonSupport with DefaultJsonProtocol { implicit val componentTypeFormat = new JsonFormat[InstanceEnums.ComponentType] { - def write(compType : InstanceEnums.ComponentType) = JsString(compType.toString()) + def write(compType : InstanceEnums.ComponentType) = JsString(compType.toString) def read(value: JsValue) = value match { case JsString(s) => s match { @@ -24,11 +24,11 @@ trait JsonSupport extends SprayJsonSupport with DefaultJsonProtocol { final case class Instance ( iD: Option[Long], - iP: Option[String], - portnumber: Option[Long], - name: Option[String], + host: String, + portnumber: Int, + name: String, /* Component Type */ - componentType: Option[InstanceEnums.ComponentType] + componentType: InstanceEnums.ComponentType ) object InstanceEnums { @@ -42,4 +42,4 @@ object InstanceEnums { val ElasticSearch = Value("ElasticSearch") } -} +} \ No newline at end of file diff --git a/src/main/scala/de/upb/cs/swt/delphi/instancemanagement/InstanceRegistry.scala b/src/main/scala/de/upb/cs/swt/delphi/instancemanagement/InstanceRegistry.scala index a77c8be..8993f62 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/instancemanagement/InstanceRegistry.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/instancemanagement/InstanceRegistry.scala @@ -2,24 +2,25 @@ package de.upb.cs.swt.delphi.instancemanagement import java.net.InetAddress +import akka.actor.ActorSystem import akka.http.scaladsl.Http import akka.http.scaladsl.marshalling.Marshal import akka.http.scaladsl.model._ import akka.http.scaladsl.unmarshalling.Unmarshal +import akka.stream.ActorMaterializer import de.upb.cs.swt.delphi.instancemanagement.InstanceEnums.ComponentType import de.upb.cs.swt.delphi.webapi.{AppLogging, Configuration, Server} - -import scala.concurrent.{Await, Future} +import scala.concurrent.{Await, ExecutionContext, Future} import scala.concurrent.duration.Duration import scala.util.{Failure, Success, Try} object InstanceRegistry extends JsonSupport with AppLogging { - implicit val system = Server.system - implicit val ec = system.dispatcher - implicit val materializer = Server.materializer + implicit val system : ActorSystem = Server.system + implicit val ec : ExecutionContext = system.dispatcher + implicit val materializer : ActorMaterializer = Server.materializer def register(configuration: Configuration) : Try[Long] = { @@ -59,8 +60,8 @@ object InstanceRegistry extends JsonSupport with AppLogging if(status == StatusCodes.OK) { Await.result(Unmarshal(response.entity).to[Instance] map {instance => - val elasticIP = instance.iP - log.info(s"Instance Registry assigned ElasticSearch instance at ${elasticIP.getOrElse("None")}") + val elasticIP = instance.host + log.info(s"Instance Registry assigned ElasticSearch instance at $elasticIP") Success(instance) } recover {case ex => log.warning(s"Failed to read response from Instance Registry, exception: $ex") @@ -85,10 +86,10 @@ object InstanceRegistry extends JsonSupport with AppLogging if(configuration.elasticsearchInstance.iD.isEmpty) { Failure(new RuntimeException("Cannot post matching result to Instance Registry, assigned ElasticSearch instance has no ID.")) } else { - val IdToPost = configuration.elasticsearchInstance.iD.get + val idToPost = configuration.elasticsearchInstance.iD.getOrElse(-1L) val request = HttpRequest( method = HttpMethods.POST, - configuration.instanceRegistryUri + s"/matchingResult?Id=$IdToPost&MatchingSuccessful=$isElasticSearchReachable") + configuration.instanceRegistryUri + s"/matchingResult?Id=$idToPost&MatchingSuccessful=$isElasticSearchReachable") Await.result(Http(system).singleRequest(request) map {response => if(response.status == StatusCodes.OK){ @@ -114,7 +115,7 @@ object InstanceRegistry extends JsonSupport with AppLogging if(!configuration.usingInstanceRegistry){ Failure(new RuntimeException("Cannot deregister from Instance Registry, no Instance Registry available.")) } else { - val id : Long = configuration.assignedID.get + val id : Long = configuration.assignedID.getOrElse(-1L) val request = HttpRequest(method = HttpMethods.POST, configuration.instanceRegistryUri + s"/deregister?Id=$id") @@ -144,5 +145,5 @@ object InstanceRegistry extends JsonSupport with AppLogging private def createInstance(id: Option[Long], controlPort : Int, name : String) : Instance = - Instance(id, Option(InetAddress.getLocalHost.getHostAddress), Option(controlPort), Option(name), Option(ComponentType.Crawler)) + Instance(id, InetAddress.getLocalHost.getHostAddress, controlPort, name, ComponentType.WebApi) } \ No newline at end of file diff --git a/src/main/scala/de/upb/cs/swt/delphi/webapi/Configuration.scala b/src/main/scala/de/upb/cs/swt/delphi/webapi/Configuration.scala index 3a41049..3fcc957 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/webapi/Configuration.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/webapi/Configuration.scala @@ -5,7 +5,7 @@ import com.sksamuel.elastic4s.http.ElasticDsl._ import de.upb.cs.swt.delphi.instancemanagement.InstanceEnums.ComponentType import de.upb.cs.swt.delphi.instancemanagement.{Instance, InstanceRegistry} -import scala.util.{Failure, Success} +import scala.util.{Failure, Success, Try} /** * @author Ben Hermann @@ -20,21 +20,22 @@ class Configuration( //Server and Elasticsearch configuration ) { - lazy val elasticsearchClientUri: ElasticsearchClientUri = ElasticsearchClientUri({ - if(elasticsearchInstance.portnumber.isEmpty){ - elasticsearchInstance.iP.getOrElse("elasticsearch://localhost:9200") - }else{ - elasticsearchInstance.iP.getOrElse("elasticsearch://localhost") + ":" + elasticsearchInstance.portnumber.getOrElse("9200") - } - }) + lazy val elasticsearchClientUri: ElasticsearchClientUri = ElasticsearchClientUri( + elasticsearchInstance.host + ":" + elasticsearchInstance.portnumber) lazy val elasticsearchInstance : Instance = InstanceRegistry.retrieveElasticSearchInstance(this) match { case Success(instance) => instance - case Failure(_) => Instance(None, Some(sys.env.getOrElse("DELPHI_ELASTIC_URI","elasticsearch://localhost:9200")), None, Some("Default ElasticSearch instance"), Some(ComponentType.ElasticSearch) ) + case Failure(_) => Instance( + None, + fallbackElasticSearchHost, + fallbackElasticSearchPort, + "Default ElasticSearch instance", + ComponentType.ElasticSearch) } - + val defaultElasticSearchPort : Int = 9200 + val defaultElasticSearchHost : String = "elasticsearch://localhost" val instanceName = "MyWebApiInstance" - val instanceRegistryUri : String = sys.env.getOrElse("DELPHI_IR_URI", "http://localhost:8085") + val instanceRegistryUri : String = sys.env.getOrElse("DELPHI_IR_URI", "http://localhost:8087") lazy val usingInstanceRegistry : Boolean = assignedID match { case Some(_) => true case None => false @@ -43,5 +44,29 @@ class Configuration( //Server and Elasticsearch configuration case Success(id) => Some(id) case Failure(_) => None } + lazy val fallbackElasticSearchPort : Int = sys.env.get("DELPHI_ELASTIC_URI") match { + case Some(hostString) => if(hostString.count(c => c == ':') == 3){ + Try(hostString.split(":")(2).toInt) match { + case Success(port) => port + case Failure(_) => defaultElasticSearchPort + } + } else { + defaultElasticSearchPort + } + case None => defaultElasticSearchPort + } + + lazy val fallbackElasticSearchHost : String = sys.env.get("DELPHI_ELASTIC_URI") match { + case Some(hostString) => + if(hostString.count(c => c == ':') == 2){ + hostString.substring(0,hostString.lastIndexOf(":")) + } else { + defaultElasticSearchHost + } + case None => defaultElasticSearchHost + + } } + + From fab695a8c1c9738f32b45058c64944c3ee9205fc Mon Sep 17 00:00:00 2001 From: Santosh <24santoshr@gmail.com> Date: Wed, 12 Sep 2018 15:14:05 +0200 Subject: [PATCH 08/43] Instance Registry changes --- .../cs/swt/delphi/instancemanagement/Instance.scala | 4 ++-- .../delphi/instancemanagement/InstanceRegistry.scala | 11 +++++++---- .../de/upb/cs/swt/delphi/webapi/Configuration.scala | 4 ++-- .../scala/de/upb/cs/swt/delphi/webapi/Server.scala | 1 + 4 files changed, 12 insertions(+), 8 deletions(-) diff --git a/src/main/scala/de/upb/cs/swt/delphi/instancemanagement/Instance.scala b/src/main/scala/de/upb/cs/swt/delphi/instancemanagement/Instance.scala index 258e268..1e51a03 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/instancemanagement/Instance.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/instancemanagement/Instance.scala @@ -23,9 +23,9 @@ trait JsonSupport extends SprayJsonSupport with DefaultJsonProtocol { } final case class Instance ( - iD: Option[Long], + id: Option[Long], host: String, - portnumber: Int, + portNumber: Int, name: String, /* Component Type */ componentType: InstanceEnums.ComponentType diff --git a/src/main/scala/de/upb/cs/swt/delphi/instancemanagement/InstanceRegistry.scala b/src/main/scala/de/upb/cs/swt/delphi/instancemanagement/InstanceRegistry.scala index 8993f62..04f3d66 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/instancemanagement/InstanceRegistry.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/instancemanagement/InstanceRegistry.scala @@ -61,7 +61,7 @@ object InstanceRegistry extends JsonSupport with AppLogging Await.result(Unmarshal(response.entity).to[Instance] map {instance => val elasticIP = instance.host - log.info(s"Instance Registry assigned ElasticSearch instance at $elasticIP") + log.info(s"Instance Registry assigned ElasticSearch instance at $elasticIP ") Success(instance) } recover {case ex => log.warning(s"Failed to read response from Instance Registry, exception: $ex") @@ -80,20 +80,23 @@ object InstanceRegistry extends JsonSupport with AppLogging } def sendMatchingResult(isElasticSearchReachable : Boolean, configuration: Configuration) : Try[Unit] = { + val id=configuration.elasticsearchInstance.id.get + if(!configuration.usingInstanceRegistry) { Failure(new RuntimeException("Cannot post matching result to Instance Registry, no Instance Registry available.")) } else { - if(configuration.elasticsearchInstance.iD.isEmpty) { + if(configuration.elasticsearchInstance.id.isEmpty) { Failure(new RuntimeException("Cannot post matching result to Instance Registry, assigned ElasticSearch instance has no ID.")) } else { - val idToPost = configuration.elasticsearchInstance.iD.getOrElse(-1L) + val idToPost = configuration.elasticsearchInstance.id.getOrElse(-1L) val request = HttpRequest( method = HttpMethods.POST, configuration.instanceRegistryUri + s"/matchingResult?Id=$idToPost&MatchingSuccessful=$isElasticSearchReachable") Await.result(Http(system).singleRequest(request) map {response => + val status=response.status if(response.status == StatusCodes.OK){ - log.info("Successfully posted matching result to Instance Registry.") + log.info(s"Successfully posted matching result to Instance Registry.") Success() } else { diff --git a/src/main/scala/de/upb/cs/swt/delphi/webapi/Configuration.scala b/src/main/scala/de/upb/cs/swt/delphi/webapi/Configuration.scala index 3fcc957..741a312 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/webapi/Configuration.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/webapi/Configuration.scala @@ -1,7 +1,7 @@ package de.upb.cs.swt.delphi.webapi -import com.sksamuel.elastic4s.{ElasticsearchClientUri, IndexAndType} import com.sksamuel.elastic4s.http.ElasticDsl._ +import com.sksamuel.elastic4s.{ElasticsearchClientUri, IndexAndType} import de.upb.cs.swt.delphi.instancemanagement.InstanceEnums.ComponentType import de.upb.cs.swt.delphi.instancemanagement.{Instance, InstanceRegistry} @@ -21,7 +21,7 @@ class Configuration( //Server and Elasticsearch configuration lazy val elasticsearchClientUri: ElasticsearchClientUri = ElasticsearchClientUri( - elasticsearchInstance.host + ":" + elasticsearchInstance.portnumber) + elasticsearchInstance.host + ":" + elasticsearchInstance.portNumber) lazy val elasticsearchInstance : Instance = InstanceRegistry.retrieveElasticSearchInstance(this) match { case Success(instance) => instance diff --git a/src/main/scala/de/upb/cs/swt/delphi/webapi/Server.scala b/src/main/scala/de/upb/cs/swt/delphi/webapi/Server.scala index 3b34da8..f7f0e82 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/webapi/Server.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/webapi/Server.scala @@ -115,6 +115,7 @@ object Server extends HttpApp with JsonSupport with AppLogging { } + } From f56a54da3394e6bcb29ac9fc713d36d5827a56c9 Mon Sep 17 00:00:00 2001 From: Santosh <32432999+24santoshr@users.noreply.github.com> Date: Sun, 16 Sep 2018 11:02:10 +0200 Subject: [PATCH 09/43] Update InstanceRegistry.scala --- .../cs/swt/delphi/instancemanagement/InstanceRegistry.scala | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/main/scala/de/upb/cs/swt/delphi/instancemanagement/InstanceRegistry.scala b/src/main/scala/de/upb/cs/swt/delphi/instancemanagement/InstanceRegistry.scala index 04f3d66..fdbd6c2 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/instancemanagement/InstanceRegistry.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/instancemanagement/InstanceRegistry.scala @@ -80,7 +80,6 @@ object InstanceRegistry extends JsonSupport with AppLogging } def sendMatchingResult(isElasticSearchReachable : Boolean, configuration: Configuration) : Try[Unit] = { - val id=configuration.elasticsearchInstance.id.get if(!configuration.usingInstanceRegistry) { Failure(new RuntimeException("Cannot post matching result to Instance Registry, no Instance Registry available.")) @@ -149,4 +148,4 @@ object InstanceRegistry extends JsonSupport with AppLogging private def createInstance(id: Option[Long], controlPort : Int, name : String) : Instance = Instance(id, InetAddress.getLocalHost.getHostAddress, controlPort, name, ComponentType.WebApi) -} \ No newline at end of file +} From 6297abeeba27fbeac3952a05ec9b9713c1424126 Mon Sep 17 00:00:00 2001 From: Ben Hermann Date: Mon, 17 Sep 2018 15:17:24 +0200 Subject: [PATCH 10/43] Bumped akka-http version b/c of CVE-2018-16131 --- build.sbt | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/build.sbt b/build.sbt index c9ab5ec..9d449eb 100644 --- a/build.sbt +++ b/build.sbt @@ -4,10 +4,12 @@ version := "1.0.0-SNAPSHOT" scalaVersion := "2.12.4" +val akkaHttpVersion = "10.1.5" + libraryDependencies += "org.parboiled" %% "parboiled" % "2.1.4" -libraryDependencies += "com.typesafe.akka" %% "akka-http" % "10.0.11" +libraryDependencies += "com.typesafe.akka" %% "akka-http" % akkaHttpVersion libraryDependencies += "com.typesafe.akka" %% "akka-stream" % "2.5.12" -libraryDependencies += "com.typesafe.akka" %% "akka-http-spray-json" % "10.1.1" +libraryDependencies += "com.typesafe.akka" %% "akka-http-spray-json" % akkaHttpVersion libraryDependencies += "io.spray" %% "spray-json" % "1.3.3" libraryDependencies += "org.parboiled" %% "parboiled" % "2.1.4" libraryDependencies += "org.scalactic" %% "scalactic" % "3.0.4" From 657bda4592044a0e9e026d630437f6dc0a86844a Mon Sep 17 00:00:00 2001 From: Santosh <24santoshr@gmail.com> Date: Wed, 3 Oct 2018 13:55:51 +0200 Subject: [PATCH 11/43] Instance Registry changes for WebApi --- .../delphi/instancemanagement/Instance.scala | 3 +- .../instancemanagement/InstanceRegistry.scala | 7 ++-- .../cs/swt/delphi/webapi/Configuration.scala | 1 - .../de/upb/cs/swt/delphi/webapi/Server.scala | 27 +-------------- .../cs/swt/delphi/webapi/StartupCheck.scala | 33 +++++++++++++++++++ 5 files changed, 39 insertions(+), 32 deletions(-) create mode 100644 src/main/scala/de/upb/cs/swt/delphi/webapi/StartupCheck.scala diff --git a/src/main/scala/de/upb/cs/swt/delphi/instancemanagement/Instance.scala b/src/main/scala/de/upb/cs/swt/delphi/instancemanagement/Instance.scala index 1e51a03..5417d54 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/instancemanagement/Instance.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/instancemanagement/Instance.scala @@ -27,7 +27,6 @@ final case class Instance ( host: String, portNumber: Int, name: String, - /* Component Type */ componentType: InstanceEnums.ComponentType ) @@ -42,4 +41,4 @@ object InstanceEnums { val ElasticSearch = Value("ElasticSearch") } -} \ No newline at end of file +} diff --git a/src/main/scala/de/upb/cs/swt/delphi/instancemanagement/InstanceRegistry.scala b/src/main/scala/de/upb/cs/swt/delphi/instancemanagement/InstanceRegistry.scala index 04f3d66..0fa1e3e 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/instancemanagement/InstanceRegistry.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/instancemanagement/InstanceRegistry.scala @@ -61,12 +61,15 @@ object InstanceRegistry extends JsonSupport with AppLogging Await.result(Unmarshal(response.entity).to[Instance] map {instance => val elasticIP = instance.host - log.info(s"Instance Registry assigned ElasticSearch instance at $elasticIP ") + log.info(s"Instance Registry assigned ElasticSearch instance at $elasticIP $status") Success(instance) } recover {case ex => log.warning(s"Failed to read response from Instance Registry, exception: $ex") Failure(ex) }, Duration.Inf) + } else if ( status == StatusCodes.NotFound) { + log.warning(s"No matching instance of ElasticSearch is present at the instance registry.") + Failure(new RuntimeException(s"Instance Registry did not contain matching instance, server returned $status")) } else{ log.warning(s"Failed to read response from Instance Registry, server returned $status") @@ -80,8 +83,6 @@ object InstanceRegistry extends JsonSupport with AppLogging } def sendMatchingResult(isElasticSearchReachable : Boolean, configuration: Configuration) : Try[Unit] = { - val id=configuration.elasticsearchInstance.id.get - if(!configuration.usingInstanceRegistry) { Failure(new RuntimeException("Cannot post matching result to Instance Registry, no Instance Registry available.")) } else { diff --git a/src/main/scala/de/upb/cs/swt/delphi/webapi/Configuration.scala b/src/main/scala/de/upb/cs/swt/delphi/webapi/Configuration.scala index 741a312..6da1ab6 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/webapi/Configuration.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/webapi/Configuration.scala @@ -66,7 +66,6 @@ class Configuration( //Server and Elasticsearch configuration case None => defaultElasticSearchHost } - } diff --git a/src/main/scala/de/upb/cs/swt/delphi/webapi/Server.scala b/src/main/scala/de/upb/cs/swt/delphi/webapi/Server.scala index f7f0e82..bdb6afd 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/webapi/Server.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/webapi/Server.scala @@ -7,18 +7,12 @@ import akka.http.scaladsl.server.HttpApp import akka.pattern.ask import akka.stream.ActorMaterializer import akka.util.Timeout -import com.sksamuel.elastic4s.http.ElasticClient -import com.sksamuel.elastic4s.http.ElasticDsl._ import de.upb.cs.swt.delphi.featuredefinitions.FeatureListMapping import de.upb.cs.swt.delphi.instancemanagement.InstanceRegistry import de.upb.cs.swt.delphi.webapi.ElasticActorManager.{Enqueue, Retrieve} import de.upb.cs.swt.delphi.webapi.ElasticRequestLimiter.Validate import spray.json._ -import scala.concurrent.duration.Duration -import scala.concurrent.{Await, ExecutionContext} -import scala.util.{Failure, Success} - /** * Web server configuration for Delphi web API. */ @@ -89,26 +83,7 @@ object Server extends HttpApp with JsonSupport with AppLogging { } def main(args: Array[String]): Unit = { - - implicit val ec : ExecutionContext = system.dispatcher - lazy val client = ElasticClient(configuration.elasticsearchClientUri) - - val f = (client.execute { - nodeInfo() - } map { i => { - if(configuration.usingInstanceRegistry) InstanceRegistry.sendMatchingResult(true, configuration) - Success(configuration) - } - } recover { case e => { - if(configuration.usingInstanceRegistry) InstanceRegistry.sendMatchingResult(false, configuration) - Failure(e) - } - }).andThen { - case _ => client.close() - } - - Await.ready(f, Duration.Inf) - + StartupCheck.check(configuration) Server.startServer(configuration.bindHost, configuration.bindPort) InstanceRegistry.deregister(configuration) system.terminate() diff --git a/src/main/scala/de/upb/cs/swt/delphi/webapi/StartupCheck.scala b/src/main/scala/de/upb/cs/swt/delphi/webapi/StartupCheck.scala new file mode 100644 index 0000000..ba0812b --- /dev/null +++ b/src/main/scala/de/upb/cs/swt/delphi/webapi/StartupCheck.scala @@ -0,0 +1,33 @@ +package de.upb.cs.swt.delphi.webapi + +import akka.actor.ActorSystem +import com.sksamuel.elastic4s.http.ElasticClient +import com.sksamuel.elastic4s.http.ElasticDsl._ +import de.upb.cs.swt.delphi.instancemanagement.InstanceRegistry +import scala.concurrent.duration.Duration +import scala.concurrent.{Await, ExecutionContext} +import scala.util.{Failure, Success, Try} + +object StartupCheck extends AppLogging { + def check(configuration: Configuration)(implicit system: ActorSystem): Try[Configuration] = { + log.warning("Performing Instance Registry checks") + implicit val ec : ExecutionContext = system.dispatcher + lazy val client = ElasticClient(configuration.elasticsearchClientUri) + + val f = (client.execute { + nodeInfo() + } map { i => { + InstanceRegistry.sendMatchingResult(isElasticSearchReachable = true, configuration) + Success(configuration) + } + } recover { case e => + InstanceRegistry.sendMatchingResult(isElasticSearchReachable = false, configuration) + Failure(e) + + }).andThen { + case _ => client.close() + } + + Await.result(f, Duration.Inf) + } +} From a9f8b2e7b066a09c3b05a90824e7dc21bf33a956 Mon Sep 17 00:00:00 2001 From: Santosh <24santoshr@gmail.com> Date: Wed, 3 Oct 2018 14:19:52 +0200 Subject: [PATCH 12/43] Changes for Instance Registry --- .../instancemanagement/InstanceRegistry.scala | 151 ++++++++++++++++++ 1 file changed, 151 insertions(+) create mode 100644 src/main/scala/de/upb/cs/swt/delphi/instancemanagement/InstanceRegistry.scala diff --git a/src/main/scala/de/upb/cs/swt/delphi/instancemanagement/InstanceRegistry.scala b/src/main/scala/de/upb/cs/swt/delphi/instancemanagement/InstanceRegistry.scala new file mode 100644 index 0000000..4ce683a --- /dev/null +++ b/src/main/scala/de/upb/cs/swt/delphi/instancemanagement/InstanceRegistry.scala @@ -0,0 +1,151 @@ +package de.upb.cs.swt.delphi.instancemanagement + +import java.net.InetAddress + +import akka.actor.ActorSystem +import akka.http.scaladsl.Http +import akka.http.scaladsl.marshalling.Marshal +import akka.http.scaladsl.model._ +import akka.http.scaladsl.unmarshalling.Unmarshal +import akka.stream.ActorMaterializer +import de.upb.cs.swt.delphi.instancemanagement.InstanceEnums.ComponentType +import de.upb.cs.swt.delphi.webapi.{AppLogging, Configuration, Server} + +import scala.concurrent.{Await, ExecutionContext, Future} +import scala.concurrent.duration.Duration +import scala.util.{Failure, Success, Try} + +object InstanceRegistry extends JsonSupport with AppLogging +{ + + implicit val system : ActorSystem = Server.system + implicit val ec : ExecutionContext = system.dispatcher + implicit val materializer : ActorMaterializer = Server.materializer + + + def register(configuration: Configuration) : Try[Long] = { + val instance = createInstance(None,configuration.bindPort, configuration.instanceName) + + Await.result(postInstance(instance, configuration.instanceRegistryUri + "/register") map {response => + if(response.status == StatusCodes.OK){ + Await.result(Unmarshal(response.entity).to[String] map { assignedID => + val id = assignedID.toLong + log.info(s"Successfully registered at Instance Registry, got ID $id.") + Success(id) + } recover { case ex => + log.warning(s"Failed to read assigned ID from Instance Registry, exception: $ex") + Failure(ex) + }, Duration.Inf) + } + else { + val statuscode = response.status + log.warning(s"Failed to register at Instance Registry, server returned $statuscode") + Failure(new RuntimeException(s"Failed to register at Instance Registry, server returned $statuscode")) + } + + } recover {case ex => + log.warning(s"Failed to register at Instance Registry, exception: $ex") + Failure(ex) + }, Duration.Inf) + } + + def retrieveElasticSearchInstance(configuration: Configuration) : Try[Instance] = { + if(!configuration.usingInstanceRegistry) { + Failure(new RuntimeException("Cannot get ElasticSearch instance from Instance Registry, no Instance Registry available.")) + } else { + val request = HttpRequest(method = HttpMethods.GET, configuration.instanceRegistryUri + "/matchingInstance?ComponentType=ElasticSearch") + + Await.result(Http(system).singleRequest(request) map {response => + val status = response.status + if(status == StatusCodes.OK) { + + Await.result(Unmarshal(response.entity).to[Instance] map {instance => + val elasticIP = instance.host + log.info(s"Instance Registry assigned ElasticSearch instance at $elasticIP ") + Success(instance) + } recover {case ex => + log.warning(s"Failed to read response from Instance Registry, exception: $ex") + Failure(ex) + }, Duration.Inf) + } + else{ + log.warning(s"Failed to read response from Instance Registry, server returned $status") + Failure(new RuntimeException(s"Failed to read response from Instance Registry, server returned $status")) + } + } recover { case ex => + log.warning(s"Failed to request ElasticSearch instance from Instance Registry, exception: $ex ") + Failure(ex) + }, Duration.Inf) + } + } + + def sendMatchingResult(isElasticSearchReachable : Boolean, configuration: Configuration) : Try[Unit] = { + + if(!configuration.usingInstanceRegistry) { + Failure(new RuntimeException("Cannot post matching result to Instance Registry, no Instance Registry available.")) + } else { + if(configuration.elasticsearchInstance.id.isEmpty) { + Failure(new RuntimeException("Cannot post matching result to Instance Registry, assigned ElasticSearch instance has no ID.")) + } else { + val idToPost = configuration.elasticsearchInstance.id.getOrElse(-1L) + val request = HttpRequest( + method = HttpMethods.POST, + configuration.instanceRegistryUri + s"/matchingResult?Id=$idToPost&MatchingSuccessful=$isElasticSearchReachable") + + Await.result(Http(system).singleRequest(request) map {response => + val status=response.status + if(response.status == StatusCodes.OK){ + log.info(s"Successfully posted matching result to Instance Registry.") + Success() + } + else { + val statuscode = response.status + log.warning(s"Failed to post matching result to Instance Registry, server returned $statuscode") + Failure(new RuntimeException(s"Failed to post matching result to Instance Registry, server returned $statuscode")) + } + + } recover {case ex => + log.warning(s"Failed to post matching result to Instance Registry, exception: $ex") + Failure(new RuntimeException(s"Failed to post matching result tot Instance Registry, exception: $ex")) + }, Duration.Inf) + } + } + + } + + def deregister(configuration: Configuration) : Try[Unit] = { + if(!configuration.usingInstanceRegistry){ + Failure(new RuntimeException("Cannot deregister from Instance Registry, no Instance Registry available.")) + } else { + val id : Long = configuration.assignedID.getOrElse(-1L) + + val request = HttpRequest(method = HttpMethods.POST, configuration.instanceRegistryUri + s"/deregister?Id=$id") + + Await.result(Http(system).singleRequest(request) map {response => + if(response.status == StatusCodes.OK){ + log.info("Successfully deregistered from Instance Registry.") + Success() + } + else { + val statuscode = response.status + log.warning(s"Failed to deregister from Instance Registry, server returned $statuscode") + Failure(new RuntimeException(s"Failed to deregister from Instance Registry, server returned $statuscode")) + } + + } recover {case ex => + log.warning(s"Failed to deregister to Instance Registry, exception: $ex") + Failure(ex) + }, Duration.Inf) + } + } + + def postInstance(instance : Instance, uri: String) () : Future[HttpResponse] = + Marshal(instance).to[RequestEntity] flatMap { entity => + val request = HttpRequest(method = HttpMethods.POST, uri = uri, entity = entity) + Http(system).singleRequest(request) + } + + + private def createInstance(id: Option[Long], controlPort : Int, name : String) : Instance = + Instance(id, InetAddress.getLocalHost.getHostAddress, controlPort, name, ComponentType.WebApi) +} \ No newline at end of file From 388ddafb02c04c94b69eb506332af2ab0c381e03 Mon Sep 17 00:00:00 2001 From: Hariharan Ramanathan Date: Thu, 4 Oct 2018 20:52:30 +0200 Subject: [PATCH 13/43] Added elastic actor test --- build.sbt | 6 +- .../cs/swt/delphi/webapi/ElasticActor.scala | 2 - .../swt/delphi/webapi/ElasticActorTest.scala | 113 ++++++++++++++++++ 3 files changed, 117 insertions(+), 4 deletions(-) create mode 100644 src/test/scala/de/upb/cs/swt/delphi/webapi/ElasticActorTest.scala diff --git a/build.sbt b/build.sbt index c9ab5ec..9bd713c 100644 --- a/build.sbt +++ b/build.sbt @@ -6,12 +6,14 @@ scalaVersion := "2.12.4" libraryDependencies += "org.parboiled" %% "parboiled" % "2.1.4" libraryDependencies += "com.typesafe.akka" %% "akka-http" % "10.0.11" +libraryDependencies += "com.typesafe.akka" %% "akka-http-testkit" % "10.0.11" libraryDependencies += "com.typesafe.akka" %% "akka-stream" % "2.5.12" libraryDependencies += "com.typesafe.akka" %% "akka-http-spray-json" % "10.1.1" -libraryDependencies += "io.spray" %% "spray-json" % "1.3.3" +libraryDependencies += "io.spray" %% "spray-json" % "1.3.3" libraryDependencies += "org.parboiled" %% "parboiled" % "2.1.4" libraryDependencies += "org.scalactic" %% "scalactic" % "3.0.4" libraryDependencies += "org.scalatest" %% "scalatest" % "3.0.4" % "test" +libraryDependencies += "org.apache.logging.log4j" % "log4j-core" % "2.11.1" val elastic4sVersion = "6.3.0" libraryDependencies ++= Seq( @@ -33,7 +35,7 @@ lazy val webapi = (project in file(".")). enablePlugins(JavaAppPackaging). enablePlugins(DockerPlugin). enablePlugins(ScalastylePlugin). - settings ( + settings( dockerBaseImage := "openjdk:jre-alpine" ). enablePlugins(AshScriptPlugin). diff --git a/src/main/scala/de/upb/cs/swt/delphi/webapi/ElasticActor.scala b/src/main/scala/de/upb/cs/swt/delphi/webapi/ElasticActor.scala index f94e2a3..2542d81 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/webapi/ElasticActor.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/webapi/ElasticActor.scala @@ -1,5 +1,4 @@ package de.upb.cs.swt.delphi.webapi - import akka.actor.{Actor, ActorLogging, Props} import com.sksamuel.elastic4s.IndexAndType import com.sksamuel.elastic4s.http.{ElasticClient, RequestFailure, RequestSuccess} @@ -29,7 +28,6 @@ class ElasticActor(configuration: Configuration, index: IndexAndType) extends Ac private def getSource(id: String) = { log.info("Executing get on entry {}", id) val searchByName = searchWithType(index) query must( - // matchQuery("name", s"http://repo1.maven.org/maven2/:$id") matchQuery("name", id) ) log.info(s"Query {}",client.show(searchByName)) diff --git a/src/test/scala/de/upb/cs/swt/delphi/webapi/ElasticActorTest.scala b/src/test/scala/de/upb/cs/swt/delphi/webapi/ElasticActorTest.scala new file mode 100644 index 0000000..d85e320 --- /dev/null +++ b/src/test/scala/de/upb/cs/swt/delphi/webapi/ElasticActorTest.scala @@ -0,0 +1,113 @@ + +// Copyright (C) 2018 The Delphi Team. +// See the LICENCE file distributed with this work for additional +// information regarding copyright ownership. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at + +// http://www.apache.org/licenses/LICENSE-2.0 + +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package de.upb.cs.swt.delphi.webapi + +import akka.actor.ActorSystem +import akka.http.scaladsl.Http +import akka.http.scaladsl.model.{HttpRequest, HttpResponse} +import akka.http.scaladsl.unmarshalling.Unmarshal +import akka.stream.ActorMaterializer +import com.sksamuel.elastic4s.RefreshPolicy +import com.sksamuel.elastic4s.embedded.LocalNode +import com.sksamuel.elastic4s.http.ElasticDsl._ +import org.elasticsearch.common.settings.Settings +import org.scalatest.{BeforeAndAfterAll, FlatSpec, Matchers} + +import scala.concurrent.duration._ +import scala.concurrent.{Await, Future} +import scala.util.{Failure, Success} + +/** + * @author Hariharan. + */ +class ElasticActorTest extends FlatSpec with Matchers with BeforeAndAfterAll { + implicit val system = ActorSystem() + implicit val materializer = ActorMaterializer()(system) + implicit val executionContext = system.dispatcher + val node = LocalNode("elasticsearch","local-path") + val client = node.client(shutdownNodeOnClose = true) + + override def beforeAll(): Unit = { + client.execute { + createIndex("delphi").mappings( + mapping("project").fields( + keywordField("name"), + keywordField("source"), + keywordField("identifier.groupId"), + keywordField("identifier.artifactId"), + keywordField("identifier.version") + ) + ) + }.await + + client.execute { + indexInto("delphi" / "project").fields( + "name" -> "yom:yom:1.0-alpha-2" + ).refresh(RefreshPolicy.IMMEDIATE) + }.await + } + + override def afterAll(): Unit = { + super.afterAll() + client.close() + system.terminate(); + } + + + + "Version no.." should + "match version from build.sbt" in { + val res: Future[HttpResponse] = Http().singleRequest(HttpRequest(uri = "http://localhost:8080/version")) + res.onComplete { + case Success(ver) => { + assert(ver.status.isSuccess()); + val res2Str: Future[String] = Unmarshal(ver.entity).to[String] + res2Str.onComplete { + case Success(value) => { + assert(value.equals(BuildInfo.version)) + } + case Failure(e) => { + assertThrows(e); + } + } + } + case Failure(e) => { + assertThrows(e); + } + } + Await.result(res, 5.seconds) + } + + "Retrive endpoint" should + "get yom:yom:1.0-alpha-2 artifact" in { + val mavenId = "yom:yom:1.0-alpha-2" + val url = s"http://localhost:8080/retrieve/${mavenId}" + val res: Future[HttpResponse] = Http().singleRequest(HttpRequest(uri = url)) + res.onComplete { + case Success(data) => { + assert(data.status.isSuccess()) + val res2Str: Future[String] = Unmarshal(data.entity).to[String] + println(res2Str) + } + case Failure(exception) => { + assertThrows(exception) + } + } + } + +} From 55b14a808c034173e2a8d5ade67ce9179ef2dcc0 Mon Sep 17 00:00:00 2001 From: Hariharan Ramanathan Date: Thu, 4 Oct 2018 22:17:58 +0200 Subject: [PATCH 14/43] Updated elastic actor test with mock data --- build.sbt | 4 +- .../de/upb/cs/swt/delphi/webapi/Server.scala | 22 +++++---- .../swt/delphi/webapi/ElasticActorTest.scala | 46 +++++++++---------- 3 files changed, 36 insertions(+), 36 deletions(-) diff --git a/build.sbt b/build.sbt index 9bd713c..895db2a 100644 --- a/build.sbt +++ b/build.sbt @@ -19,15 +19,13 @@ val elastic4sVersion = "6.3.0" libraryDependencies ++= Seq( "com.sksamuel.elastic4s" %% "elastic4s-core" % elastic4sVersion, + // for the http client "com.sksamuel.elastic4s" %% "elastic4s-http" % elastic4sVersion, // if you want to use reactive streams "com.sksamuel.elastic4s" %% "elastic4s-http-streams" % elastic4sVersion, - // testing - "com.sksamuel.elastic4s" %% "elastic4s-testkit" % elastic4sVersion % "test", - "com.sksamuel.elastic4s" %% "elastic4s-embedded" % elastic4sVersion % "test" ) diff --git a/src/main/scala/de/upb/cs/swt/delphi/webapi/Server.scala b/src/main/scala/de/upb/cs/swt/delphi/webapi/Server.scala index 02e72ee..3315520 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/webapi/Server.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/webapi/Server.scala @@ -22,12 +22,16 @@ object Server extends HttpApp with JsonSupport with AppLogging { private val requestLimiter = system.actorOf(ElasticRequestLimiter.props(configuration, actorManager)) implicit val timeout = Timeout(5, TimeUnit.SECONDS) - override def routes = - path("version") { version } ~ - path("features") { features } ~ - pathPrefix("search" / Remaining) { query => search(query) } ~ - pathPrefix("retrieve" / Remaining) { identifier => retrieve(identifier) } ~ - pathPrefix("enqueue" / Remaining) { identifier => enqueue(identifier) } + override def routes = + path("version") { + version + } ~ + path("features") { + features + } ~ + pathPrefix("search" / Remaining) { query => search(query) } ~ + pathPrefix("retrieve" / Remaining) { identifier => retrieve(identifier) } ~ + pathPrefix("enqueue" / Remaining) { identifier => enqueue(identifier) } private def version = { @@ -48,11 +52,11 @@ object Server extends HttpApp with JsonSupport with AppLogging { def retrieve(identifier: String) = { get { - pass { //TODO: Require authentication here + pass { //TODO: Require authentication here complete( (actorManager ? Retrieve(identifier)).mapTo[String] ) - } ~ extractClientIP{ ip => + } ~ extractClientIP { ip => complete( (requestLimiter ? Validate(ip, Retrieve(identifier))).mapTo[String] ) @@ -62,7 +66,7 @@ object Server extends HttpApp with JsonSupport with AppLogging { def enqueue(identifier: String) = { get { - pass { //TODO: Require authorization here + pass { //TODO: Require authorization here complete( (actorManager ? Enqueue(identifier)).mapTo[String] ) diff --git a/src/test/scala/de/upb/cs/swt/delphi/webapi/ElasticActorTest.scala b/src/test/scala/de/upb/cs/swt/delphi/webapi/ElasticActorTest.scala index d85e320..e144534 100644 --- a/src/test/scala/de/upb/cs/swt/delphi/webapi/ElasticActorTest.scala +++ b/src/test/scala/de/upb/cs/swt/delphi/webapi/ElasticActorTest.scala @@ -23,9 +23,8 @@ import akka.http.scaladsl.model.{HttpRequest, HttpResponse} import akka.http.scaladsl.unmarshalling.Unmarshal import akka.stream.ActorMaterializer import com.sksamuel.elastic4s.RefreshPolicy -import com.sksamuel.elastic4s.embedded.LocalNode +import com.sksamuel.elastic4s.http.ElasticClient import com.sksamuel.elastic4s.http.ElasticDsl._ -import org.elasticsearch.common.settings.Settings import org.scalatest.{BeforeAndAfterAll, FlatSpec, Matchers} import scala.concurrent.duration._ @@ -39,35 +38,26 @@ class ElasticActorTest extends FlatSpec with Matchers with BeforeAndAfterAll { implicit val system = ActorSystem() implicit val materializer = ActorMaterializer()(system) implicit val executionContext = system.dispatcher - val node = LocalNode("elasticsearch","local-path") - val client = node.client(shutdownNodeOnClose = true) + val configuration = new Configuration() + val client = ElasticClient(configuration.elasticsearchClientUri) override def beforeAll(): Unit = { - client.execute { - createIndex("delphi").mappings( - mapping("project").fields( - keywordField("name"), - keywordField("source"), - keywordField("identifier.groupId"), - keywordField("identifier.artifactId"), - keywordField("identifier.version") - ) - ) - }.await - client.execute { indexInto("delphi" / "project").fields( - "name" -> "yom:yom:1.0-alpha-2" + "name" -> "test:elastic-actor-test:1.0" ).refresh(RefreshPolicy.IMMEDIATE) }.await } + override def afterAll(): Unit = { - super.afterAll() + client.execute { + deleteByQuery("delphi", "project", matchQuery("name", "test:elastic-actor-test:1.0")) + }.await client.close() - system.terminate(); - } + system.terminate() + } "Version no.." should @@ -90,24 +80,32 @@ class ElasticActorTest extends FlatSpec with Matchers with BeforeAndAfterAll { assertThrows(e); } } - Await.result(res, 5.seconds) + Await.result(res, 2.seconds) } "Retrive endpoint" should - "get yom:yom:1.0-alpha-2 artifact" in { - val mavenId = "yom:yom:1.0-alpha-2" + "get test:elastic-actor-test:1.0 artifact" in { + val mavenId = "test:elastic-actor-test:1.0" val url = s"http://localhost:8080/retrieve/${mavenId}" val res: Future[HttpResponse] = Http().singleRequest(HttpRequest(uri = url)) res.onComplete { case Success(data) => { assert(data.status.isSuccess()) val res2Str: Future[String] = Unmarshal(data.entity).to[String] - println(res2Str) + res2Str.onComplete { + case Success(value) => { + assert(value.contains(mavenId)) + } + case Failure(e) => { + assertThrows(e); + } + } } case Failure(exception) => { assertThrows(exception) } } + Await.result(res, 2.seconds) } } From 4743facda5b9f01e33426d4f0da2b2991b6863b9 Mon Sep 17 00:00:00 2001 From: Hariharan Ramanathan Date: Fri, 5 Oct 2018 02:47:34 +0200 Subject: [PATCH 15/43] Moved ElasticActorTest to integration test --- build.sbt | 7 ++++++- .../de/upb/cs/swt/delphi/webapi/ElasticActorTest.scala | 0 2 files changed, 6 insertions(+), 1 deletion(-) rename src/{test => it}/scala/de/upb/cs/swt/delphi/webapi/ElasticActorTest.scala (100%) diff --git a/build.sbt b/build.sbt index 836107a..fb8b2fb 100644 --- a/build.sbt +++ b/build.sbt @@ -14,7 +14,7 @@ libraryDependencies += "com.typesafe.akka" %% "akka-stream" % "2.5.12" libraryDependencies += "com.typesafe.akka" %% "akka-http-spray-json" % "10.1.1" libraryDependencies += "io.spray" %% "spray-json" % "1.3.3" libraryDependencies += "org.scalactic" %% "scalactic" % "3.0.4" -libraryDependencies += "org.scalatest" %% "scalatest" % "3.0.4" % "test" +libraryDependencies += "org.scalatest" %% "scalatest" % "3.0.4" % "it,test" libraryDependencies += "org.apache.logging.log4j" % "log4j-core" % "2.11.1" val elastic4sVersion = "6.3.0" @@ -32,6 +32,11 @@ libraryDependencies ++= Seq( lazy val webapi = (project in file(".")). + //https://www.scala-sbt.org/1.x/docs/Testing.html +configs(IntegrationTest). + settings( + Defaults.itSettings, + ). enablePlugins(JavaAppPackaging). enablePlugins(DockerPlugin). enablePlugins(ScalastylePlugin). diff --git a/src/test/scala/de/upb/cs/swt/delphi/webapi/ElasticActorTest.scala b/src/it/scala/de/upb/cs/swt/delphi/webapi/ElasticActorTest.scala similarity index 100% rename from src/test/scala/de/upb/cs/swt/delphi/webapi/ElasticActorTest.scala rename to src/it/scala/de/upb/cs/swt/delphi/webapi/ElasticActorTest.scala From f0fa33ef59dcf4669029e150cde46b77919bc96d Mon Sep 17 00:00:00 2001 From: Hariharan Ramanathan Date: Fri, 5 Oct 2018 21:04:03 +0200 Subject: [PATCH 16/43] Reverting to latest akka http version --- build.sbt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/build.sbt b/build.sbt index fb8b2fb..2fbc8c4 100644 --- a/build.sbt +++ b/build.sbt @@ -8,10 +8,10 @@ val akkaHttpVersion = "10.1.5" libraryDependencies += "org.parboiled" %% "parboiled" % "2.1.4" -libraryDependencies += "com.typesafe.akka" %% "akka-http" % "10.0.11" -libraryDependencies += "com.typesafe.akka" %% "akka-http-testkit" % "10.0.11" +libraryDependencies += "com.typesafe.akka" %% "akka-http" % akkaHttpVersion +libraryDependencies += "com.typesafe.akka" %% "akka-http-testkit" % akkaHttpVersion libraryDependencies += "com.typesafe.akka" %% "akka-stream" % "2.5.12" -libraryDependencies += "com.typesafe.akka" %% "akka-http-spray-json" % "10.1.1" +libraryDependencies += "com.typesafe.akka" %% "akka-http-spray-json" % akkaHttpVersion libraryDependencies += "io.spray" %% "spray-json" % "1.3.3" libraryDependencies += "org.scalactic" %% "scalactic" % "3.0.4" libraryDependencies += "org.scalatest" %% "scalatest" % "3.0.4" % "it,test" From d47316de0f13721fc783478f1ad1fdd76398fc16 Mon Sep 17 00:00:00 2001 From: ayybeeshafi Date: Tue, 16 Oct 2018 19:38:03 +0200 Subject: [PATCH 17/43] Implementation of functions with respect to the new API. Other funtions are working fine, unable to test reportstop() and resportstart() from inside the container by exporting enviournment variable -INSTANCE_ID- refs #23 --- delphi-webapi | 1 + .../delphi/instancemanagement/Instance.scala | 76 ++++-- .../instancemanagement/InstanceRegistry.scala | 223 ++++++++++++++++-- .../cs/swt/delphi/webapi/Configuration.scala | 26 +- 4 files changed, 280 insertions(+), 46 deletions(-) create mode 160000 delphi-webapi diff --git a/delphi-webapi b/delphi-webapi new file mode 160000 index 0000000..ed901bf --- /dev/null +++ b/delphi-webapi @@ -0,0 +1 @@ +Subproject commit ed901bf01811ac64126906ab10340da0f87fe0e8 diff --git a/src/main/scala/de/upb/cs/swt/delphi/instancemanagement/Instance.scala b/src/main/scala/de/upb/cs/swt/delphi/instancemanagement/Instance.scala index 1e51a03..de9c694 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/instancemanagement/Instance.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/instancemanagement/Instance.scala @@ -1,45 +1,85 @@ +// Copyright (C) 2018 The Delphi Team. +// See the LICENCE file distributed with this work for additional +// information regarding copyright ownership. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. package de.upb.cs.swt.delphi.instancemanagement - import akka.http.scaladsl.marshallers.sprayjson.SprayJsonSupport -import spray.json.{DefaultJsonProtocol, JsString, JsValue, JsonFormat} +import spray.json.{DefaultJsonProtocol, DeserializationException, JsString, JsValue, JsonFormat} trait JsonSupport extends SprayJsonSupport with DefaultJsonProtocol { - implicit val componentTypeFormat = new JsonFormat[InstanceEnums.ComponentType] { + + implicit val componentTypeFormat : JsonFormat[InstanceEnums.ComponentType] = new JsonFormat[InstanceEnums.ComponentType] { + def write(compType : InstanceEnums.ComponentType) = JsString(compType.toString) - def read(value: JsValue) = value match { + def read(value: JsValue) : InstanceEnums.ComponentType = value match { case JsString(s) => s match { case "Crawler" => InstanceEnums.ComponentType.Crawler case "WebApi" => InstanceEnums.ComponentType.WebApi case "WebApp" => InstanceEnums.ComponentType.WebApp case "DelphiManagement" => InstanceEnums.ComponentType.DelphiManagement case "ElasticSearch" => InstanceEnums.ComponentType.ElasticSearch - case x => throw new RuntimeException(s"Unexpected string value $x for component type.") + case x => throw DeserializationException(s"Unexpected string value $x for component type.") } - case y => throw new RuntimeException(s"Unexpected type $y while deserializing component type.") + case y => throw DeserializationException(s"Unexpected type $y while deserializing component type.") } } - implicit val instanceFormat = jsonFormat5(Instance) + + implicit val stateFormat : JsonFormat[InstanceEnums.State] = new JsonFormat[InstanceEnums.State] { + + def write(compType : InstanceEnums.State) = JsString(compType.toString) + + def read(value: JsValue) : InstanceEnums.State = value match { + case JsString(s) => s match { + case "Running" => InstanceEnums.InstanceState.Running + case "Stopped" => InstanceEnums.InstanceState.Stopped + case "Failed" => InstanceEnums.InstanceState.Failed + case "Paused" => InstanceEnums.InstanceState.Paused + case "NotReachable" => InstanceEnums.InstanceState.NotReachable + case x => throw DeserializationException(s"Unexpected string value $x for instance state.") + } + case y => throw DeserializationException(s"Unexpected type $y while deserializing instance state.") + } + } + + implicit val instanceFormat : JsonFormat[Instance] = jsonFormat7(Instance) } final case class Instance ( id: Option[Long], host: String, - portNumber: Int, + portNumber: Long, name: String, - /* Component Type */ - componentType: InstanceEnums.ComponentType + componentType: InstanceEnums.ComponentType, + dockerId: Option[String], + instanceState: InstanceEnums.State ) - object InstanceEnums { - type ComponentType = ComponentType.Value object ComponentType extends Enumeration { - val Crawler = Value("Crawler") - val WebApi = Value("WebApi") - val WebApp = Value("WebApp") - val DelphiManagement = Value("DelphiManagement") - val ElasticSearch = Value("ElasticSearch") + val Crawler : Value = Value("Crawler") + val ElasticSearch : Value = Value("ElasticSearch") + val WebApi : Value = Value("WebApi") + val WebApp : Value = Value("WebApp") + val DelphiManagement : Value = Value("DelphiManagement") + } + type State = InstanceState.Value + object InstanceState extends Enumeration { + val Running : Value = Value("Running") + val Paused : Value = Value("Paused") + val Stopped : Value = Value("Stopped") + val Failed : Value = Value("Failed") + val NotReachable : Value = Value("NotReachable") } - } \ No newline at end of file diff --git a/src/main/scala/de/upb/cs/swt/delphi/instancemanagement/InstanceRegistry.scala b/src/main/scala/de/upb/cs/swt/delphi/instancemanagement/InstanceRegistry.scala index fdbd6c2..f01d42d 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/instancemanagement/InstanceRegistry.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/instancemanagement/InstanceRegistry.scala @@ -1,19 +1,36 @@ +// Copyright (C) 2018 The Delphi Team. +// See the LICENCE file distributed with this work for additional +// information regarding copyright ownership. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package de.upb.cs.swt.delphi.instancemanagement import java.net.InetAddress import akka.actor.ActorSystem import akka.http.scaladsl.Http -import akka.http.scaladsl.marshalling.Marshal import akka.http.scaladsl.model._ import akka.http.scaladsl.unmarshalling.Unmarshal import akka.stream.ActorMaterializer -import de.upb.cs.swt.delphi.instancemanagement.InstanceEnums.ComponentType +import akka.util.ByteString +import de.upb.cs.swt.delphi.instancemanagement.InstanceEnums.{ComponentType, InstanceState} import de.upb.cs.swt.delphi.webapi.{AppLogging, Configuration, Server} import scala.concurrent.{Await, ExecutionContext, Future} -import scala.concurrent.duration.Duration +import scala.concurrent.duration._ import scala.util.{Failure, Success, Try} +import spray.json._ object InstanceRegistry extends JsonSupport with AppLogging { @@ -22,8 +39,85 @@ object InstanceRegistry extends JsonSupport with AppLogging implicit val ec : ExecutionContext = system.dispatcher implicit val materializer : ActorMaterializer = Server.materializer + lazy val instanceIdFromEnv : Option[Long] = Try[Long](sys.env("INSTANCE_ID").toLong).toOption + + + def handleInstanceStart(configuration: Configuration) : Option[Long] = { + instanceIdFromEnv match { + case Some(id) => + reportStart(configuration) match { + case Success(_) => Some(id) + case Failure(_) => None + } + case None => + register(configuration) match { + case Success(id) => Some(id) + case Failure(_) => None + } + } + } - def register(configuration: Configuration) : Try[Long] = { + def handleInstanceStop(configuration: Configuration) : Try[Unit] = { + if(instanceIdFromEnv.isDefined) { + reportStop(configuration) + } else { + deregister(configuration) + } + } + + def handleInstanceFailure(configuration: Configuration) : Try[Unit] = { + if(instanceIdFromEnv.isDefined) { + reportFailure(configuration) + } else { + deregister(configuration) + } + } + + def reportStart(configuration: Configuration) : Try[Unit] = executeReportOperation(configuration, ReportOperationType.Start) + + def reportStop(configuration: Configuration) : Try[Unit] = { + if(configuration.usingInstanceRegistry) { + executeReportOperation(configuration, ReportOperationType.Stop) + } else { + Failure(new RuntimeException("Cannot report stop, no instance registry available.")) + } + } + + def reportFailure(configuration: Configuration) : Try[Unit] = { + if(configuration.usingInstanceRegistry){ + executeReportOperation(configuration, ReportOperationType.Failure) + } else { + Failure(new RuntimeException("Cannot report failure, no instance registry available.")) + } + } + + private def executeReportOperation(configuration: Configuration, operationType: ReportOperationType.Value) : Try[Unit] = { + instanceIdFromEnv match { + case Some(id) => + val request = HttpRequest( + method = HttpMethods.POST, + configuration.instanceRegistryUri + ReportOperationType.toOperationUriString(operationType, id)) + + Await.result(Http(system).singleRequest(request) map {response => + if(response.status == StatusCodes.OK){ + log.info(s"Successfully reported ${operationType.toString} to Instance Registry.") + Success() + } + else { + log.warning(s"Failed to report ${operationType.toString} to Instance Registry, server returned ${response.status}") + Failure(new RuntimeException(s"Failed to report ${operationType.toString} to Instance Registry, server returned ${response.status}")) + } + + } recover {case ex => + log.warning(s"Failed to report ${operationType.toString} to Instance Registry, exception: $ex") + Failure(new RuntimeException(s"Failed to report ${operationType.toString} to Instance Registry, exception: $ex")) + }, Duration.Inf) + case None => + log.warning(s"Cannot report ${operationType.toString} to Instance Registry, no instance id is present in env var 'INSTANCE_ID'.") + Failure(new RuntimeException(s"Cannot report ${operationType.toString} to Instance Registry, no instance id is present in env var 'INSTANCE_ID'.")) + } + } + def register(configuration: Configuration) :Try[Long] = { val instance = createInstance(None,configuration.bindPort, configuration.instanceName) Await.result(postInstance(instance, configuration.instanceRegistryUri + "/register") map {response => @@ -56,21 +150,26 @@ object InstanceRegistry extends JsonSupport with AppLogging val request = HttpRequest(method = HttpMethods.GET, configuration.instanceRegistryUri + "/matchingInstance?ComponentType=ElasticSearch") Await.result(Http(system).singleRequest(request) map {response => - val status = response.status - if(status == StatusCodes.OK) { - - Await.result(Unmarshal(response.entity).to[Instance] map {instance => - val elasticIP = instance.host - log.info(s"Instance Registry assigned ElasticSearch instance at $elasticIP ") - Success(instance) - } recover {case ex => - log.warning(s"Failed to read response from Instance Registry, exception: $ex") - Failure(ex) - }, Duration.Inf) - } - else{ - log.warning(s"Failed to read response from Instance Registry, server returned $status") - Failure(new RuntimeException(s"Failed to read response from Instance Registry, server returned $status")) + response.status match { + case StatusCodes.OK => + try { + val instanceString : String = Await.result(response.entity.dataBytes.runFold(ByteString(""))(_ ++ _).map(_.utf8String), 5 seconds) + val esInstance = instanceString.parseJson.convertTo[Instance](instanceFormat) + val elasticIP = esInstance.host + log.info(s"Instance Registry assigned ElasticSearch instance at $elasticIP") + Success(esInstance) + } catch { + case px: spray.json.JsonParser.ParsingException => + log.warning(s"Failed to read response from Instance Registry, exception: $px") + Failure(px) + } + case StatusCodes.NotFound => + log.warning(s"No matching instance of type 'ElasticSearch' is present at the instance registry.") + Failure(new RuntimeException(s"Instance Registry did not contain matching instance, server returned ${StatusCodes.NotFound}")) + case _ => + val status = response.status + log.warning(s"Failed to read matching instance from Instance Registry, server returned $status") + Failure(new RuntimeException(s"Failed to read matching instance from Instance Registry, server returned $status")) } } recover { case ex => log.warning(s"Failed to request ElasticSearch instance from Instance Registry, exception: $ex ") @@ -139,13 +238,89 @@ object InstanceRegistry extends JsonSupport with AppLogging } } - def postInstance(instance : Instance, uri: String) () : Future[HttpResponse] = - Marshal(instance).to[RequestEntity] flatMap { entity => - val request = HttpRequest(method = HttpMethods.POST, uri = uri, entity = entity) + def postInstance(instance : Instance, uri: String) () : Future[HttpResponse] = { + try { + val request = HttpRequest(method = HttpMethods.POST, uri = uri, entity = instance.toJson(instanceFormat).toString()) Http(system).singleRequest(request) + } catch { + case dx: DeserializationException => + log.warning(s"Failed to deregister to Instance Registry, exception: $dx") + Future.failed(dx) } + } private def createInstance(id: Option[Long], controlPort : Int, name : String) : Instance = - Instance(id, InetAddress.getLocalHost.getHostAddress, controlPort, name, ComponentType.WebApi) -} + Instance(id, InetAddress.getLocalHost.getHostAddress, + controlPort, name, ComponentType.WebApi, None, InstanceState.Running) + + def reportStart(id: String, configuration: Configuration):Try[ResponseEntity] ={ + val request = HttpRequest(method = HttpMethods.GET, configuration.instanceRegistryUri + "/reportStart") + Await.result(Http(system).singleRequest(request) map {response => + if(response.status == StatusCodes.OK){ + Success(response.entity) + } + else { + val statuscode = response.status + log.warning(s"Failed to perform reportStart, server returned $statuscode") + Failure(new RuntimeException(s"Failed to perform reportStart, server returned $statuscode")) + } + } recover {case ex => + log.warning(s"Failed to perform reportStart, exception: $ex") + Failure(new RuntimeException(s"Failed to perform reportStart, server returned, exception: $ex")) + }, Duration.Inf) + } + + def reportFailure(id: String, configuration: Configuration):Try[ResponseEntity] = { + + val request = HttpRequest(method = HttpMethods.GET, configuration.instanceRegistryUri + "/reportFailure") + Await.result(Http(system).singleRequest(request) map {response => + if(response.status == StatusCodes.OK){ + Success(response.entity) + } + else { + val statuscode = response.status + log.warning(s"Failed to perform reportFailure, server returned $statuscode") + Failure(new RuntimeException(s"Failed to perform reportFailure, server returned $statuscode")) + } + } recover {case ex => + log.warning(s"Failed to perform reportFailure, server returned, exception: $ex") + Failure(new RuntimeException(s"Failed to perform reportFailure, server returned, exception: $ex")) + }, Duration.Inf) + } + + def reportStop(id: String, configuration: Configuration):Try[ResponseEntity] = { + + val request = HttpRequest(method = HttpMethods.GET, configuration.instanceRegistryUri + "/reportStop") + Await.result(Http(system).singleRequest(request) map {response => + if(response.status == StatusCodes.OK){ + Success(response.entity) + } + else { + val statuscode = response.status + log.warning(s"Failed to perform reportStop, server returned $statuscode") + Failure(new RuntimeException(s"Failed to perform reportStop, server returned $statuscode")) + } + } recover {case ex => + log.warning(s"Failed to perform reportStop, server returned, exception: $ex") + Failure(new RuntimeException(s"Failed to perform reportStop, server returned, exception: $ex")) + }, Duration.Inf) + } + + object ReportOperationType extends Enumeration { + val Start : Value = Value("Start") + val Stop : Value = Value("Stop") + val Failure : Value = Value("Failure") + + def toOperationUriString(operation: ReportOperationType.Value, id: Long) : String = { + operation match { + case Start => + s"/reportStart?Id=$id" + case Stop => + s"/reportStop?Id=$id" + case _ => + s"/reportFailure?Id=$id" + } + } + } +} \ No newline at end of file diff --git a/src/main/scala/de/upb/cs/swt/delphi/webapi/Configuration.scala b/src/main/scala/de/upb/cs/swt/delphi/webapi/Configuration.scala index 741a312..8081d76 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/webapi/Configuration.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/webapi/Configuration.scala @@ -1,8 +1,23 @@ +// Copyright (C) 2018 The Delphi Team. +// See the LICENCE file distributed with this work for additional +// information regarding copyright ownership. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. package de.upb.cs.swt.delphi.webapi import com.sksamuel.elastic4s.http.ElasticDsl._ import com.sksamuel.elastic4s.{ElasticsearchClientUri, IndexAndType} -import de.upb.cs.swt.delphi.instancemanagement.InstanceEnums.ComponentType +import de.upb.cs.swt.delphi.instancemanagement.InstanceEnums.{ComponentType, InstanceState} import de.upb.cs.swt.delphi.instancemanagement.{Instance, InstanceRegistry} import scala.util.{Failure, Success, Try} @@ -23,14 +38,16 @@ class Configuration( //Server and Elasticsearch configuration lazy val elasticsearchClientUri: ElasticsearchClientUri = ElasticsearchClientUri( elasticsearchInstance.host + ":" + elasticsearchInstance.portNumber) - lazy val elasticsearchInstance : Instance = InstanceRegistry.retrieveElasticSearchInstance(this) match { + lazy val elasticsearchInstance : Instance = InstanceRegistry.retrieveElasticSearchInstance( configuration = this) match { case Success(instance) => instance case Failure(_) => Instance( None, fallbackElasticSearchHost, fallbackElasticSearchPort, "Default ElasticSearch instance", - ComponentType.ElasticSearch) + ComponentType.ElasticSearch, + None, + InstanceState.Running) } val defaultElasticSearchPort : Int = 9200 val defaultElasticSearchHost : String = "elasticsearch://localhost" @@ -40,7 +57,7 @@ class Configuration( //Server and Elasticsearch configuration case Some(_) => true case None => false } - lazy val assignedID : Option[Long] = InstanceRegistry.register(this) match { + lazy val assignedID : Option[Long] = InstanceRegistry.register(configuration = this) match { case Success(id) => Some(id) case Failure(_) => None } @@ -66,6 +83,7 @@ class Configuration( //Server and Elasticsearch configuration case None => defaultElasticSearchHost } + lazy val instanceId : Option[Long] = InstanceRegistry.handleInstanceStart(configuration = this) } From e4bf21c1b09e809e314baa0589872d76e8e60f9d Mon Sep 17 00:00:00 2001 From: ayybeeshafi Date: Thu, 18 Oct 2018 11:12:34 +0200 Subject: [PATCH 18/43] Implementation and testing of functions with respect to new API refs #23 --- .../scala/de/upb/cs/swt/delphi/webapi/Configuration.scala | 6 ++---- src/main/scala/de/upb/cs/swt/delphi/webapi/Server.scala | 8 +++++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/main/scala/de/upb/cs/swt/delphi/webapi/Configuration.scala b/src/main/scala/de/upb/cs/swt/delphi/webapi/Configuration.scala index 8081d76..e8e51e2 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/webapi/Configuration.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/webapi/Configuration.scala @@ -57,10 +57,8 @@ class Configuration( //Server and Elasticsearch configuration case Some(_) => true case None => false } - lazy val assignedID : Option[Long] = InstanceRegistry.register(configuration = this) match { - case Success(id) => Some(id) - case Failure(_) => None - } + lazy val assignedID : Option[Long] = InstanceRegistry.handleInstanceStart(configuration = this) + lazy val fallbackElasticSearchPort : Int = sys.env.get("DELPHI_ELASTIC_URI") match { case Some(hostString) => if(hostString.count(c => c == ':') == 3){ Try(hostString.split(":")(2).toInt) match { diff --git a/src/main/scala/de/upb/cs/swt/delphi/webapi/Server.scala b/src/main/scala/de/upb/cs/swt/delphi/webapi/Server.scala index f7f0e82..6fc9650 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/webapi/Server.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/webapi/Server.scala @@ -2,7 +2,9 @@ package de.upb.cs.swt.delphi.webapi import java.util.concurrent.TimeUnit -import akka.actor.ActorSystem +import akka.Done +import akka.actor.{ActorSystem, PoisonPill} +import akka.http.scaladsl.Http import akka.http.scaladsl.server.HttpApp import akka.pattern.ask import akka.stream.ActorMaterializer @@ -17,7 +19,7 @@ import spray.json._ import scala.concurrent.duration.Duration import scala.concurrent.{Await, ExecutionContext} -import scala.util.{Failure, Success} +import scala.util.{Failure, Success, Try} /** * Web server configuration for Delphi web API. @@ -110,7 +112,7 @@ object Server extends HttpApp with JsonSupport with AppLogging { Await.ready(f, Duration.Inf) Server.startServer(configuration.bindHost, configuration.bindPort) - InstanceRegistry.deregister(configuration) + InstanceRegistry.handleInstanceStop(configuration) system.terminate() } From af5371e9fa27e762d8dfd4f5f4651a3ab39d8d66 Mon Sep 17 00:00:00 2001 From: Ben Hermann Date: Mon, 22 Oct 2018 10:44:01 +0200 Subject: [PATCH 19/43] Removing incorrectly copied subproject --- delphi-webapi | 1 - 1 file changed, 1 deletion(-) delete mode 160000 delphi-webapi diff --git a/delphi-webapi b/delphi-webapi deleted file mode 160000 index ed901bf..0000000 --- a/delphi-webapi +++ /dev/null @@ -1 +0,0 @@ -Subproject commit ed901bf01811ac64126906ab10340da0f87fe0e8 From ed80dcfdc05bb5d6efb65a79395073b579737482 Mon Sep 17 00:00:00 2001 From: Ben Hermann Date: Mon, 22 Oct 2018 10:58:16 +0200 Subject: [PATCH 20/43] Added Licenses and reorganized imports --- .../swt/delphi/webapi/ElasticActorTest.scala | 1 - .../FeatureDescription.scala | 16 +++++++++++++ .../FeatureListMapping.scala | 16 +++++++++++++ .../delphi/instancemanagement/Instance.scala | 1 + .../upb/cs/swt/delphi/querylanguage/AST.scala | 16 +++++++++++++ .../cs/swt/delphi/querylanguage/Syntax.scala | 16 +++++++++++++ .../upb/cs/swt/delphi/webapi/AppLogging.scala | 16 +++++++++++++ .../cs/swt/delphi/webapi/Configuration.scala | 1 + .../cs/swt/delphi/webapi/ElasticActor.scala | 16 +++++++++++++ .../delphi/webapi/ElasticActorManager.scala | 16 +++++++++++++ .../webapi/ElasticPriorityMailbox.scala | 16 +++++++++++++ .../delphi/webapi/ElasticRequestLimiter.scala | 16 +++++++++++++ .../cs/swt/delphi/webapi/JsonSupport.scala | 16 +++++++++++++ .../de/upb/cs/swt/delphi/webapi/Server.scala | 24 +++++++++++++------ .../cs/swt/delphi/webapi/StartupCheck.scala | 16 +++++++++++++ .../swt/delphi/querylanguage/SyntaxTest.scala | 16 +++++++++++++ 16 files changed, 211 insertions(+), 8 deletions(-) diff --git a/src/it/scala/de/upb/cs/swt/delphi/webapi/ElasticActorTest.scala b/src/it/scala/de/upb/cs/swt/delphi/webapi/ElasticActorTest.scala index e144534..de3e8a1 100644 --- a/src/it/scala/de/upb/cs/swt/delphi/webapi/ElasticActorTest.scala +++ b/src/it/scala/de/upb/cs/swt/delphi/webapi/ElasticActorTest.scala @@ -1,4 +1,3 @@ - // Copyright (C) 2018 The Delphi Team. // See the LICENCE file distributed with this work for additional // information regarding copyright ownership. diff --git a/src/main/scala/de/upb/cs/swt/delphi/featuredefinitions/FeatureDescription.scala b/src/main/scala/de/upb/cs/swt/delphi/featuredefinitions/FeatureDescription.scala index 944d03a..8e7dbe4 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/featuredefinitions/FeatureDescription.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/featuredefinitions/FeatureDescription.scala @@ -1,3 +1,19 @@ +// Copyright (C) 2018 The Delphi Team. +// See the LICENCE file distributed with this work for additional +// information regarding copyright ownership. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package de.upb.cs.swt.delphi.featuredefinitions //Describes all relevant fields for conducting searches on this class diff --git a/src/main/scala/de/upb/cs/swt/delphi/featuredefinitions/FeatureListMapping.scala b/src/main/scala/de/upb/cs/swt/delphi/featuredefinitions/FeatureListMapping.scala index d4c165f..e099885 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/featuredefinitions/FeatureListMapping.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/featuredefinitions/FeatureListMapping.scala @@ -1,3 +1,19 @@ +// Copyright (C) 2018 The Delphi Team. +// See the LICENCE file distributed with this work for additional +// information regarding copyright ownership. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package de.upb.cs.swt.delphi.featuredefinitions object FeatureListMapping { diff --git a/src/main/scala/de/upb/cs/swt/delphi/instancemanagement/Instance.scala b/src/main/scala/de/upb/cs/swt/delphi/instancemanagement/Instance.scala index da34c22..db29ab8 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/instancemanagement/Instance.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/instancemanagement/Instance.scala @@ -13,6 +13,7 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. + package de.upb.cs.swt.delphi.instancemanagement import akka.http.scaladsl.marshallers.sprayjson.SprayJsonSupport import spray.json.{DefaultJsonProtocol, DeserializationException, JsString, JsValue, JsonFormat} diff --git a/src/main/scala/de/upb/cs/swt/delphi/querylanguage/AST.scala b/src/main/scala/de/upb/cs/swt/delphi/querylanguage/AST.scala index 57b8abe..57b2985 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/querylanguage/AST.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/querylanguage/AST.scala @@ -1,3 +1,19 @@ +// Copyright (C) 2018 The Delphi Team. +// See the LICENCE file distributed with this work for additional +// information regarding copyright ownership. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package de.upb.cs.swt.delphi.querylanguage trait CombinatorialExpr diff --git a/src/main/scala/de/upb/cs/swt/delphi/querylanguage/Syntax.scala b/src/main/scala/de/upb/cs/swt/delphi/querylanguage/Syntax.scala index 1779b56..016ec29 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/querylanguage/Syntax.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/querylanguage/Syntax.scala @@ -1,3 +1,19 @@ +// Copyright (C) 2018 The Delphi Team. +// See the LICENCE file distributed with this work for additional +// information regarding copyright ownership. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package de.upb.cs.swt.delphi.querylanguage import org.parboiled2.{CharPredicate, Parser, ParserInput, Rule1} diff --git a/src/main/scala/de/upb/cs/swt/delphi/webapi/AppLogging.scala b/src/main/scala/de/upb/cs/swt/delphi/webapi/AppLogging.scala index 673d2f2..514ec42 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/webapi/AppLogging.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/webapi/AppLogging.scala @@ -1,3 +1,19 @@ +// Copyright (C) 2018 The Delphi Team. +// See the LICENCE file distributed with this work for additional +// information regarding copyright ownership. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package de.upb.cs.swt.delphi.webapi import akka.actor.{ActorSystem, ExtendedActorSystem} diff --git a/src/main/scala/de/upb/cs/swt/delphi/webapi/Configuration.scala b/src/main/scala/de/upb/cs/swt/delphi/webapi/Configuration.scala index e8e51e2..42669c8 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/webapi/Configuration.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/webapi/Configuration.scala @@ -13,6 +13,7 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. + package de.upb.cs.swt.delphi.webapi import com.sksamuel.elastic4s.http.ElasticDsl._ diff --git a/src/main/scala/de/upb/cs/swt/delphi/webapi/ElasticActor.scala b/src/main/scala/de/upb/cs/swt/delphi/webapi/ElasticActor.scala index 2542d81..131a151 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/webapi/ElasticActor.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/webapi/ElasticActor.scala @@ -1,3 +1,19 @@ +// Copyright (C) 2018 The Delphi Team. +// See the LICENCE file distributed with this work for additional +// information regarding copyright ownership. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package de.upb.cs.swt.delphi.webapi import akka.actor.{Actor, ActorLogging, Props} import com.sksamuel.elastic4s.IndexAndType diff --git a/src/main/scala/de/upb/cs/swt/delphi/webapi/ElasticActorManager.scala b/src/main/scala/de/upb/cs/swt/delphi/webapi/ElasticActorManager.scala index aa8f6fc..d8530fc 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/webapi/ElasticActorManager.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/webapi/ElasticActorManager.scala @@ -1,3 +1,19 @@ +// Copyright (C) 2018 The Delphi Team. +// See the LICENCE file distributed with this work for additional +// information regarding copyright ownership. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package de.upb.cs.swt.delphi.webapi import akka.actor.{Actor, ActorLogging, Props, Terminated} diff --git a/src/main/scala/de/upb/cs/swt/delphi/webapi/ElasticPriorityMailbox.scala b/src/main/scala/de/upb/cs/swt/delphi/webapi/ElasticPriorityMailbox.scala index 5600c01..fc48390 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/webapi/ElasticPriorityMailbox.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/webapi/ElasticPriorityMailbox.scala @@ -1,3 +1,19 @@ +// Copyright (C) 2018 The Delphi Team. +// See the LICENCE file distributed with this work for additional +// information regarding copyright ownership. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package de.upb.cs.swt.delphi.webapi import akka.actor.ActorSystem diff --git a/src/main/scala/de/upb/cs/swt/delphi/webapi/ElasticRequestLimiter.scala b/src/main/scala/de/upb/cs/swt/delphi/webapi/ElasticRequestLimiter.scala index 4395630..60432b0 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/webapi/ElasticRequestLimiter.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/webapi/ElasticRequestLimiter.scala @@ -1,3 +1,19 @@ +// Copyright (C) 2018 The Delphi Team. +// See the LICENCE file distributed with this work for additional +// information regarding copyright ownership. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package de.upb.cs.swt.delphi.webapi diff --git a/src/main/scala/de/upb/cs/swt/delphi/webapi/JsonSupport.scala b/src/main/scala/de/upb/cs/swt/delphi/webapi/JsonSupport.scala index 4d8c771..082970f 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/webapi/JsonSupport.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/webapi/JsonSupport.scala @@ -1,3 +1,19 @@ +// Copyright (C) 2018 The Delphi Team. +// See the LICENCE file distributed with this work for additional +// information regarding copyright ownership. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package de.upb.cs.swt.delphi.webapi import akka.http.scaladsl.marshallers.sprayjson.SprayJsonSupport diff --git a/src/main/scala/de/upb/cs/swt/delphi/webapi/Server.scala b/src/main/scala/de/upb/cs/swt/delphi/webapi/Server.scala index 3f6ff68..d846793 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/webapi/Server.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/webapi/Server.scala @@ -1,10 +1,24 @@ +// Copyright (C) 2018 The Delphi Team. +// See the LICENCE file distributed with this work for additional +// information regarding copyright ownership. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package de.upb.cs.swt.delphi.webapi import java.util.concurrent.TimeUnit -import akka.Done -import akka.actor.{ActorSystem, PoisonPill} -import akka.http.scaladsl.Http +import akka.actor.ActorSystem import akka.http.scaladsl.server.HttpApp import akka.pattern.ask import akka.stream.ActorMaterializer @@ -15,10 +29,6 @@ import de.upb.cs.swt.delphi.webapi.ElasticActorManager.{Enqueue, Retrieve} import de.upb.cs.swt.delphi.webapi.ElasticRequestLimiter.Validate import spray.json._ -import scala.concurrent.duration.Duration -import scala.concurrent.{Await, ExecutionContext} -import scala.util.{Failure, Success, Try} - /** * Web server configuration for Delphi web API. */ diff --git a/src/main/scala/de/upb/cs/swt/delphi/webapi/StartupCheck.scala b/src/main/scala/de/upb/cs/swt/delphi/webapi/StartupCheck.scala index ba0812b..637e762 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/webapi/StartupCheck.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/webapi/StartupCheck.scala @@ -1,3 +1,19 @@ +// Copyright (C) 2018 The Delphi Team. +// See the LICENCE file distributed with this work for additional +// information regarding copyright ownership. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package de.upb.cs.swt.delphi.webapi import akka.actor.ActorSystem diff --git a/src/test/scala/de/upb/cs/swt/delphi/querylanguage/SyntaxTest.scala b/src/test/scala/de/upb/cs/swt/delphi/querylanguage/SyntaxTest.scala index 336d420..41d1e68 100644 --- a/src/test/scala/de/upb/cs/swt/delphi/querylanguage/SyntaxTest.scala +++ b/src/test/scala/de/upb/cs/swt/delphi/querylanguage/SyntaxTest.scala @@ -1,3 +1,19 @@ +// Copyright (C) 2018 The Delphi Team. +// See the LICENCE file distributed with this work for additional +// information regarding copyright ownership. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package de.upb.cs.swt.delphi.querylanguage import org.scalatest.{FlatSpec, Matchers} From f014bf0a8fd50f482e0a050b54b5f66c121926bb Mon Sep 17 00:00:00 2001 From: Ben Hermann Date: Mon, 22 Oct 2018 11:06:06 +0200 Subject: [PATCH 21/43] Renamed fields to conform to coding style --- .../upb/cs/swt/delphi/querylanguage/AST.scala | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/main/scala/de/upb/cs/swt/delphi/querylanguage/AST.scala b/src/main/scala/de/upb/cs/swt/delphi/querylanguage/AST.scala index 57b2985..54d2579 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/querylanguage/AST.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/querylanguage/AST.scala @@ -18,18 +18,18 @@ package de.upb.cs.swt.delphi.querylanguage trait CombinatorialExpr -case class AndExpr(Left: CombinatorialExpr, Right: CombinatorialExpr) extends CombinatorialExpr -case class OrExpr(Left: CombinatorialExpr, Right: CombinatorialExpr) extends CombinatorialExpr -case class NotExpr(Expr: CombinatorialExpr) extends CombinatorialExpr -case class XorExpr(Left: CombinatorialExpr, Right: CombinatorialExpr) extends CombinatorialExpr +case class AndExpr(left: CombinatorialExpr, right: CombinatorialExpr) extends CombinatorialExpr +case class OrExpr(left: CombinatorialExpr, right: CombinatorialExpr) extends CombinatorialExpr +case class NotExpr(expr: CombinatorialExpr) extends CombinatorialExpr +case class XorExpr(left: CombinatorialExpr, right: CombinatorialExpr) extends CombinatorialExpr trait SingularConditionExpr extends CombinatorialExpr -case class EqualExpr(Left: String, Right: String) extends SingularConditionExpr -case class NotEqualExpr(Left: String, Right: String) extends SingularConditionExpr -case class GreaterThanExpr(Left: String, Right: String) extends SingularConditionExpr -case class GreaterOrEqualExpr(Left: String, Right: String) extends SingularConditionExpr -case class LessThanExpr(Left: String, Right: String) extends SingularConditionExpr -case class LessOrEqualExpr(Left: String, Right: String) extends SingularConditionExpr -case class LikeExpr(Left: String, Right: String) extends SingularConditionExpr -case class TrueExpr(Expr: String) extends SingularConditionExpr +case class EqualExpr(left: String, right: String) extends SingularConditionExpr +case class NotEqualExpr(left: String, right: String) extends SingularConditionExpr +case class GreaterThanExpr(left: String, right: String) extends SingularConditionExpr +case class GreaterOrEqualExpr(left: String, right: String) extends SingularConditionExpr +case class LessThanExpr(left: String, right: String) extends SingularConditionExpr +case class LessOrEqualExpr(left: String, right: String) extends SingularConditionExpr +case class LikeExpr(left: String, right: String) extends SingularConditionExpr +case class TrueExpr(expr: String) extends SingularConditionExpr From bf2c78aa8520b979f9d4a939e96ec3dc48b375ed Mon Sep 17 00:00:00 2001 From: Ben Hermann Date: Mon, 22 Oct 2018 11:07:10 +0200 Subject: [PATCH 22/43] Reorganized imports --- .../upb/cs/swt/delphi/webapi/ElasticRequestLimiter.scala | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/main/scala/de/upb/cs/swt/delphi/webapi/ElasticRequestLimiter.scala b/src/main/scala/de/upb/cs/swt/delphi/webapi/ElasticRequestLimiter.scala index 60432b0..f131652 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/webapi/ElasticRequestLimiter.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/webapi/ElasticRequestLimiter.scala @@ -17,14 +17,13 @@ package de.upb.cs.swt.delphi.webapi -import akka.actor.{Actor, ActorLogging, ActorRef, Props} -import akka.actor.Timers +import akka.actor.{Actor, ActorLogging, ActorRef, Props, Timers} import akka.http.scaladsl.model.RemoteAddress -import de.upb.cs.swt.delphi.webapi.ElasticRequestLimiter._ import de.upb.cs.swt.delphi.webapi.ElasticActorManager.ElasticMessage +import de.upb.cs.swt.delphi.webapi.ElasticRequestLimiter._ -import scala.concurrent.duration._ import scala.collection.mutable +import scala.concurrent.duration._ //Limits the number of requests any given IP can make by tracking how many requests an IP has made within a given // window of time, and timing out any IP that exceeds a threshold by rejecting any further request for a period of time From 2324c226bff6797dc0b467fafc2e47a3247cb80a Mon Sep 17 00:00:00 2001 From: Ben Hermann Date: Mon, 22 Oct 2018 11:38:05 +0200 Subject: [PATCH 23/43] Added support for system termination (issue #22) --- .../de/upb/cs/swt/delphi/webapi/Server.scala | 29 ++++++++++++++----- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/src/main/scala/de/upb/cs/swt/delphi/webapi/Server.scala b/src/main/scala/de/upb/cs/swt/delphi/webapi/Server.scala index d846793..daaab63 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/webapi/Server.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/webapi/Server.scala @@ -19,7 +19,7 @@ package de.upb.cs.swt.delphi.webapi import java.util.concurrent.TimeUnit import akka.actor.ActorSystem -import akka.http.scaladsl.server.HttpApp +import akka.http.scaladsl.server.{HttpApp, Route} import akka.pattern.ask import akka.stream.ActorMaterializer import akka.util.Timeout @@ -29,6 +29,9 @@ import de.upb.cs.swt.delphi.webapi.ElasticActorManager.{Enqueue, Retrieve} import de.upb.cs.swt.delphi.webapi.ElasticRequestLimiter.Validate import spray.json._ +import scala.concurrent.{Await, ExecutionContext} +import scala.concurrent.duration._ + /** * Web server configuration for Delphi web API. */ @@ -41,7 +44,7 @@ object Server extends HttpApp with JsonSupport with AppLogging { implicit val timeout = Timeout(5, TimeUnit.SECONDS) implicit val materializer = ActorMaterializer() - override def routes = + override def routes: Route = path("version") { version } ~ @@ -69,7 +72,7 @@ object Server extends HttpApp with JsonSupport with AppLogging { } } - def retrieve(identifier: String) = { + def retrieve(identifier: String): Route = { get { pass { //TODO: Require authentication here complete( @@ -83,7 +86,7 @@ object Server extends HttpApp with JsonSupport with AppLogging { } } - def enqueue(identifier: String) = { + def enqueue(identifier: String): Route = { get { pass { //TODO: Require authorization here complete( @@ -93,7 +96,7 @@ object Server extends HttpApp with JsonSupport with AppLogging { } } - def search(query: String) = { + def search(query: String): Route = { get { complete { query @@ -102,10 +105,20 @@ object Server extends HttpApp with JsonSupport with AppLogging { } def main(args: Array[String]): Unit = { + sys.addShutdownHook({ + log.warning("Received shutdown signal.") + InstanceRegistry.handleInstanceStop(configuration) + }) + StartupCheck.check(configuration) - Server.startServer(configuration.bindHost, configuration.bindPort) - InstanceRegistry.handleInstanceStop(configuration) - system.terminate() + Server.startServer(configuration.bindHost, configuration.bindPort, system) + + implicit val ec : ExecutionContext = ExecutionContext.global + val terminationFuture = system.terminate() + + terminationFuture.onComplete { + sys.exit() + } } From 3fbbc6bb35727206c5ce9c646946e4d61d0b955a Mon Sep 17 00:00:00 2001 From: Ben Hermann Date: Mon, 22 Oct 2018 12:53:48 +0200 Subject: [PATCH 24/43] Pinned jackson-databind version --- build.sbt | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/build.sbt b/build.sbt index 2fbc8c4..428f433 100644 --- a/build.sbt +++ b/build.sbt @@ -51,3 +51,9 @@ configs(IntegrationTest). ) scalastyleConfig := baseDirectory.value / "project" / "scalastyle-config.xml" + +// Pinning secure versions of insecure transitive libraryDependencies +// Please update when updating dependencies above (including Play plugin) +libraryDependencies ++= Seq( + "com.fasterxml.jackson.core" % "jackson-databind" % "2.9.7" +) From cbd5d6b65032148285cf8dc2bd43723592b52968 Mon Sep 17 00:00:00 2001 From: Ben Hermann Date: Thu, 25 Oct 2018 11:34:19 +0200 Subject: [PATCH 25/43] Added statistics endpoint. Closes #27 --- .../delphi/webapi/StatisticsQueryCheck.scala | 11 ++++ .../cs/swt/delphi/webapi/Configuration.scala | 7 +- .../de/upb/cs/swt/delphi/webapi/Server.scala | 30 +++++---- .../swt/delphi/webapi/StatisticsQuery.scala | 65 +++++++++++++++++++ 4 files changed, 98 insertions(+), 15 deletions(-) create mode 100644 src/it/scala/de/upb/cs/swt/delphi/webapi/StatisticsQueryCheck.scala create mode 100644 src/main/scala/de/upb/cs/swt/delphi/webapi/StatisticsQuery.scala diff --git a/src/it/scala/de/upb/cs/swt/delphi/webapi/StatisticsQueryCheck.scala b/src/it/scala/de/upb/cs/swt/delphi/webapi/StatisticsQueryCheck.scala new file mode 100644 index 0000000..ef1af7b --- /dev/null +++ b/src/it/scala/de/upb/cs/swt/delphi/webapi/StatisticsQueryCheck.scala @@ -0,0 +1,11 @@ +package de.upb.cs.swt.delphi.webapi + +import org.scalatest.{FlatSpec, Matchers} + +class StatisticsQueryCheck extends FlatSpec with Matchers { + "Statics" should "be retrievable" in { + val configuration = new Configuration() + val stats = new StatisticsQuery(configuration) + println(stats.retrieveStandardStatistics) + } +} diff --git a/src/main/scala/de/upb/cs/swt/delphi/webapi/Configuration.scala b/src/main/scala/de/upb/cs/swt/delphi/webapi/Configuration.scala index 42669c8..d3d81e9 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/webapi/Configuration.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/webapi/Configuration.scala @@ -17,7 +17,7 @@ package de.upb.cs.swt.delphi.webapi import com.sksamuel.elastic4s.http.ElasticDsl._ -import com.sksamuel.elastic4s.{ElasticsearchClientUri, IndexAndType} +import com.sksamuel.elastic4s.{ElasticsearchClientUri, Index, IndexAndType} import de.upb.cs.swt.delphi.instancemanagement.InstanceEnums.{ComponentType, InstanceState} import de.upb.cs.swt.delphi.instancemanagement.{Instance, InstanceRegistry} @@ -29,13 +29,14 @@ import scala.util.{Failure, Success, Try} class Configuration( //Server and Elasticsearch configuration val bindHost: String = "0.0.0.0", val bindPort: Int = 8080, - val esProjectIndex: IndexAndType = "delphi" / "project", + val esIndex : String = "delphi", + val esType : String = "project", //Actor system configuration val elasticActorPoolSize: Int = 8 ) { - + lazy val esProjectIndex: IndexAndType = esIndex / esType lazy val elasticsearchClientUri: ElasticsearchClientUri = ElasticsearchClientUri( elasticsearchInstance.host + ":" + elasticsearchInstance.portNumber) diff --git a/src/main/scala/de/upb/cs/swt/delphi/webapi/Server.scala b/src/main/scala/de/upb/cs/swt/delphi/webapi/Server.scala index daaab63..09cebca 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/webapi/Server.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/webapi/Server.scala @@ -45,15 +45,13 @@ object Server extends HttpApp with JsonSupport with AppLogging { implicit val materializer = ActorMaterializer() override def routes: Route = - path("version") { - version - } ~ - path("features") { - features - } ~ + path("version") { version } ~ + path("features") { features } ~ + path("statistics") { statistics } ~ pathPrefix("search" / Remaining) { query => search(query) } ~ - pathPrefix("retrieve" / Remaining) { identifier => retrieve(identifier) } ~ - pathPrefix("enqueue" / Remaining) { identifier => enqueue(identifier) } + pathPrefix("retrieve" / Remaining) { identifier => retrieve(identifier) } + //~ + //pathPrefix("enqueue" / Remaining) { identifier => enqueue(identifier) } private def version = { @@ -72,9 +70,18 @@ object Server extends HttpApp with JsonSupport with AppLogging { } } - def retrieve(identifier: String): Route = { + private def statistics = { get { - pass { //TODO: Require authentication here + complete { + import StatisticsJson._ + new StatisticsQuery(configuration).retrieveStandardStatistics.toJson + } + } + } + + private def retrieve(identifier: String): Route = { + get { + pass { complete( (actorManager ? Retrieve(identifier)).mapTo[String] ) @@ -113,7 +120,7 @@ object Server extends HttpApp with JsonSupport with AppLogging { StartupCheck.check(configuration) Server.startServer(configuration.bindHost, configuration.bindPort, system) - implicit val ec : ExecutionContext = ExecutionContext.global + implicit val ec: ExecutionContext = ExecutionContext.global val terminationFuture = system.terminate() terminationFuture.onComplete { @@ -122,7 +129,6 @@ object Server extends HttpApp with JsonSupport with AppLogging { } - } diff --git a/src/main/scala/de/upb/cs/swt/delphi/webapi/StatisticsQuery.scala b/src/main/scala/de/upb/cs/swt/delphi/webapi/StatisticsQuery.scala new file mode 100644 index 0000000..4c7859b --- /dev/null +++ b/src/main/scala/de/upb/cs/swt/delphi/webapi/StatisticsQuery.scala @@ -0,0 +1,65 @@ +// Copyright (C) 2018 The Delphi Team. +// See the LICENCE file distributed with this work for additional +// information regarding copyright ownership. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package de.upb.cs.swt.delphi.webapi + +import com.sksamuel.elastic4s.http.{ElasticClient, RequestSuccess} +import com.sksamuel.elastic4s.http.ElasticDsl._ +import com.sksamuel.elastic4s.http.search.SearchResponse +import spray.json.DefaultJsonProtocol + +class StatisticsQuery(configuration: Configuration) { + def retrieveStandardStatistics = { + val client = ElasticClient(configuration.elasticsearchClientUri) + + val fullIndexSize = searchWithType(configuration.esProjectIndex) size 0 + val hermesEnabledProjects = searchWithType(configuration.esProjectIndex) query bool { + must { + existsQuery("hermes") + } + } size (0) + + + client.execute { + multi ( + fullIndexSize, + hermesEnabledProjects + ) + }.await match { + // TODO: These matchers are non exhaustive + case RequestSuccess(_, _, _, results) => { + assert(results.size == 2) + val total = results.items(0).response match { + case s: SearchResponse => { + s.hits.total + } + } + val hermesTotal = results.items(1).response match { + case s: SearchResponse => { + s.hits.total + } + } + Statistics(hermesTotal, hermesTotal) + } + } + } +} + +case class Statistics(total : Long, hermesEnabled : Long) + +object StatisticsJson extends DefaultJsonProtocol { + implicit val statisticsFormat = jsonFormat2(Statistics) +} From c06b77ccaf23701963aa2536d049ad2959644e00 Mon Sep 17 00:00:00 2001 From: Ben Hermann Date: Sat, 27 Oct 2018 02:22:50 +0200 Subject: [PATCH 26/43] Reimplemented `retrieve` endpoint --- .../upb/cs/swt/delphi/webapi/Identifier.scala | 22 +++ .../swt/delphi/webapi/MavenIdentifier.scala | 61 ++++++++ .../cs/swt/delphi/webapi/RetrieveQuery.scala | 135 ++++++++++++++++++ .../de/upb/cs/swt/delphi/webapi/Server.scala | 64 +++++---- .../swt/delphi/webapi/StatisticsQuery.scala | 41 ++++-- .../webapi/MavenIdentifierApplyTest.scala | 61 ++++++++ 6 files changed, 344 insertions(+), 40 deletions(-) create mode 100644 src/main/scala/de/upb/cs/swt/delphi/webapi/Identifier.scala create mode 100644 src/main/scala/de/upb/cs/swt/delphi/webapi/MavenIdentifier.scala create mode 100644 src/main/scala/de/upb/cs/swt/delphi/webapi/RetrieveQuery.scala create mode 100644 src/test/scala/de/upb/cs/swt/delphi/webapi/MavenIdentifierApplyTest.scala diff --git a/src/main/scala/de/upb/cs/swt/delphi/webapi/Identifier.scala b/src/main/scala/de/upb/cs/swt/delphi/webapi/Identifier.scala new file mode 100644 index 0000000..3dcc0f9 --- /dev/null +++ b/src/main/scala/de/upb/cs/swt/delphi/webapi/Identifier.scala @@ -0,0 +1,22 @@ +// Copyright (C) 2018 The Delphi Team. +// See the LICENCE file distributed with this work for additional +// information regarding copyright ownership. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package de.upb.cs.swt.delphi.webapi + +/** + * Represents an identifier for a software artifact + */ +trait Identifier diff --git a/src/main/scala/de/upb/cs/swt/delphi/webapi/MavenIdentifier.scala b/src/main/scala/de/upb/cs/swt/delphi/webapi/MavenIdentifier.scala new file mode 100644 index 0000000..71a7756 --- /dev/null +++ b/src/main/scala/de/upb/cs/swt/delphi/webapi/MavenIdentifier.scala @@ -0,0 +1,61 @@ +// Copyright (C) 2018 The Delphi Team. +// See the LICENCE file distributed with this work for additional +// information regarding copyright ownership. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package de.upb.cs.swt.delphi.webapi + +import java.net.{URI, URLEncoder} +import java.nio.charset.StandardCharsets + +case class MavenIdentifier(val repository: Option[String], val groupId: String, val artifactId: String, val version: Option[String]) extends Identifier { + + def toUniqueString = { + repository + ":" + groupId + ":" + artifactId + ":" + version + } + + override val toString: String = groupId + ":" + artifactId + ":" + version + + def toJarLocation : URI = { + constructArtifactBaseUri().resolve(encode(artifactId) + "-" + encode(version.getOrElse("")) + ".jar") + } + + def toPomLocation : URI = { + constructArtifactBaseUri().resolve(encode(artifactId) + "-" + encode(version.getOrElse("")) + ".pom") + } + + private def constructArtifactBaseUri(): URI = + new URI(repository.getOrElse("")) + .resolve(encode(groupId).replace('.', '/') + "/") + .resolve(encode(artifactId) + "/") + .resolve(encode(version.getOrElse("")) + "/") + + private def encode(input : String) : String = + URLEncoder.encode(input, StandardCharsets.UTF_8.toString()) +} + +object MavenIdentifier { + private implicit def wrapOption[A](value : A) : Option[A] = Some(value) + def apply(s: String): Option[MavenIdentifier] = { + val splitString: Array[String] = s.split(':') + if (splitString.length < 2 || splitString.length > 3) return None + //if (splitString(0).startsWith("http")) { + // if (splitString.length < 3) return None + // MavenIdentifier(splitString(0), splitString(1), splitString(2), if (splitString.length < 4) None else splitString(3)) + //} else { + //if (splitString.length > 3) return None + MavenIdentifier(None, splitString(0), splitString(1), if (splitString.length < 3) None else splitString(2)) + //} + } +} \ No newline at end of file diff --git a/src/main/scala/de/upb/cs/swt/delphi/webapi/RetrieveQuery.scala b/src/main/scala/de/upb/cs/swt/delphi/webapi/RetrieveQuery.scala new file mode 100644 index 0000000..5a59102 --- /dev/null +++ b/src/main/scala/de/upb/cs/swt/delphi/webapi/RetrieveQuery.scala @@ -0,0 +1,135 @@ +// Copyright (C) 2018 The Delphi Team. +// See the LICENCE file distributed with this work for additional +// information regarding copyright ownership. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package de.upb.cs.swt.delphi.webapi + +import com.sksamuel.elastic4s.http.{ElasticClient, RequestSuccess, Response} +import com.sksamuel.elastic4s.http.ElasticDsl.{termQuery, _} +import com.sksamuel.elastic4s.http.search.{SearchHit, SearchHits, SearchResponse} +import com.sksamuel.elastic4s.searches.queries.Query +import com.sksamuel.elastic4s.searches.queries.term.TermQuery +import org.joda.time.DateTime +import org.joda.time.format.{DateTimeFormatter, ISODateTimeFormat} +import spray.json.{DefaultJsonProtocol, DeserializationException, JsString, JsValue, RootJsonFormat} + +object RetrieveQuery { + def retrieve(identifier: String)(implicit configuration: Configuration) = { + val client = ElasticClient(configuration.elasticsearchClientUri) + + val parsedIdentifier = MavenIdentifier(identifier) + + + parsedIdentifier match { + case None => None + case Some(m) => { + val response: Response[SearchResponse] = client.execute { + searchWithType(configuration.esProjectIndex) query { + bool { + must( + constructIdQuery(m) + ) + } + } + }.await + + response match { + case RequestSuccess(_, body, _, results: SearchResponse) => { + results.totalHits match { + case 0L => None + case _ => { + Some(transformResults(results.hits)) + } + } + } + case _ => None + } + } + } + + + } + + private def constructIdQuery(m: MavenIdentifier): Iterable[Query] = { + val baseQuery: List[TermQuery] = List( + termQuery("identifier.groupId", m.groupId), + termQuery("identifier.artifactId", m.artifactId) + ) + m.version.isDefined match { + case true => { + baseQuery.+:(termQuery("identifier.version", m.version.getOrElse(""))) + } + case false => { + baseQuery + } + } + } + + private def getHermesResults(sourceMap: Map[String, AnyRef]): Map[String, Int] = { + if (!sourceMap.contains("hermes")) return Map() + if (!sourceMap("hermes").isInstanceOf[Map[String, AnyRef]]) return Map() + if (!sourceMap("hermes").asInstanceOf[Map[String, AnyRef]].contains("features")) return Map() + + val hermesMap = sourceMap("hermes").asInstanceOf[Map[String, AnyRef]] + + hermesMap("features").asInstanceOf[Map[String, Int]] + } + + def getMetadata(sourceMap: Map[String, AnyRef]): ArtifactMetadata = { + val identifier = sourceMap("identifier").asInstanceOf[Map[String, String]] + ArtifactMetadata(sourceMap("source").asInstanceOf[String], + ISODateTimeFormat.dateTime().parseDateTime(sourceMap("discovered").asInstanceOf[String]), + identifier("groupId"), identifier("artifactId"), identifier("version")) + } + + private def transformResult(h: SearchHit) = { + val sourceMap = h.sourceAsMap + Artifact(h.id, getMetadata(sourceMap), getHermesResults(sourceMap)) + } + + private def transformResults(hits: SearchHits) = { + hits.hits.map(h => transformResult(h)) + } +} + +case class Artifact(id: String, metadata: ArtifactMetadata, metricResults: Map[String, Int]) + +case class ArtifactMetadata(source: String, discovered: DateTime, groupId: String, artifactId: String, version: String) + +object ArtifactJson extends DefaultJsonProtocol { + + implicit object DateJsonFormat extends RootJsonFormat[DateTime] { + + private val parserISO: DateTimeFormatter = ISODateTimeFormat.dateTime() + + override def write(obj: DateTime) = JsString(parserISO.print(obj)) + + override def read(json: JsValue): DateTime = json match { + case JsString(s) => parserISO.parseDateTime(s) + case _ => throw new DeserializationException("Error info you want here ...") + } + } + + implicit val artifactMetadataFormat = jsonFormat5(ArtifactMetadata) + implicit val artifactFormat = jsonFormat3(Artifact) + + def prettyPrint(pretty: Option[_], value: JsValue): String = { + pretty.isDefined match { + case true => value.prettyPrint + case false => value.compactPrint + } + } +} + diff --git a/src/main/scala/de/upb/cs/swt/delphi/webapi/Server.scala b/src/main/scala/de/upb/cs/swt/delphi/webapi/Server.scala index 09cebca..c15de85 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/webapi/Server.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/webapi/Server.scala @@ -19,39 +19,46 @@ package de.upb.cs.swt.delphi.webapi import java.util.concurrent.TimeUnit import akka.actor.ActorSystem +import akka.http.scaladsl.model.{HttpResponse, StatusCodes} import akka.http.scaladsl.server.{HttpApp, Route} -import akka.pattern.ask import akka.stream.ActorMaterializer import akka.util.Timeout import de.upb.cs.swt.delphi.featuredefinitions.FeatureListMapping import de.upb.cs.swt.delphi.instancemanagement.InstanceRegistry -import de.upb.cs.swt.delphi.webapi.ElasticActorManager.{Enqueue, Retrieve} -import de.upb.cs.swt.delphi.webapi.ElasticRequestLimiter.Validate import spray.json._ -import scala.concurrent.{Await, ExecutionContext} -import scala.concurrent.duration._ +import ArtifactJson._ +import scala.concurrent.ExecutionContext /** * Web server configuration for Delphi web API. */ object Server extends HttpApp with JsonSupport with AppLogging { - private val configuration = new Configuration() implicit val system = ActorSystem("delphi-webapi") + implicit val materializer = ActorMaterializer() + + private implicit val configuration = new Configuration() private val actorManager = system.actorOf(ElasticActorManager.props(configuration)) private val requestLimiter = system.actorOf(ElasticRequestLimiter.props(configuration, actorManager)) - implicit val timeout = Timeout(5, TimeUnit.SECONDS) - implicit val materializer = ActorMaterializer() + private implicit val timeout = Timeout(5, TimeUnit.SECONDS) + override def routes: Route = - path("version") { version } ~ - path("features") { features } ~ - path("statistics") { statistics } ~ + path("version") { + version + } ~ + path("features") { + features + } ~ + path("statistics") { + statistics + } ~ pathPrefix("search" / Remaining) { query => search(query) } ~ pathPrefix("retrieve" / Remaining) { identifier => retrieve(identifier) } - //~ - //pathPrefix("enqueue" / Remaining) { identifier => enqueue(identifier) } + + //~ + //pathPrefix("enqueue" / Remaining) { identifier => enqueue(identifier) } private def version = { @@ -73,31 +80,26 @@ object Server extends HttpApp with JsonSupport with AppLogging { private def statistics = { get { complete { - import StatisticsJson._ - new StatisticsQuery(configuration).retrieveStandardStatistics.toJson + val result = new StatisticsQuery(configuration).retrieveStandardStatistics + result match { + case Some(stats) => { + import StatisticsJson._ + stats.toJson + } + case _ => "Failure" + } } } } private def retrieve(identifier: String): Route = { get { - pass { - complete( - (actorManager ? Retrieve(identifier)).mapTo[String] - ) - } ~ extractClientIP { ip => - complete( - (requestLimiter ? Validate(ip, Retrieve(identifier))).mapTo[String] - ) - } - } - } - - def enqueue(identifier: String): Route = { - get { - pass { //TODO: Require authorization here + parameter('pretty.?) { (pretty) => complete( - (actorManager ? Enqueue(identifier)).mapTo[String] + RetrieveQuery.retrieve(identifier) match { + case Some(result: Any) => prettyPrint(pretty, result.toJson) + case None => HttpResponse(StatusCodes.NotFound) + } ) } } diff --git a/src/main/scala/de/upb/cs/swt/delphi/webapi/StatisticsQuery.scala b/src/main/scala/de/upb/cs/swt/delphi/webapi/StatisticsQuery.scala index 4c7859b..d913369 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/webapi/StatisticsQuery.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/webapi/StatisticsQuery.scala @@ -16,12 +16,14 @@ package de.upb.cs.swt.delphi.webapi -import com.sksamuel.elastic4s.http.{ElasticClient, RequestSuccess} import com.sksamuel.elastic4s.http.ElasticDsl._ -import com.sksamuel.elastic4s.http.search.SearchResponse +import com.sksamuel.elastic4s.http.{ElasticClient, RequestSuccess} +import org.slf4j.LoggerFactory import spray.json.DefaultJsonProtocol class StatisticsQuery(configuration: Configuration) { + private val log = LoggerFactory.getLogger(this.getClass) + def retrieveStandardStatistics = { val client = ElasticClient(configuration.elasticsearchClientUri) @@ -42,17 +44,38 @@ class StatisticsQuery(configuration: Configuration) { // TODO: These matchers are non exhaustive case RequestSuccess(_, _, _, results) => { assert(results.size == 2) - val total = results.items(0).response match { - case s: SearchResponse => { - s.hits.total + val totalOpt = results.items(0).response match { + case Right(s) => { + Some(s.hits.total) + } + case _ => { + println(s"Received some other response: ${results.items(0).response}") + None } } - val hermesTotal = results.items(1).response match { - case s: SearchResponse => { - s.hits.total + val hermesTotalOpt = results.items(1).response match { + case Right(s) => { + Some(s.hits.total) + } + case _ => { + println(s"Received some other response: ${results.items(1).response}") + None } } - Statistics(hermesTotal, hermesTotal) + totalOpt match { + case Some(total) => { + hermesTotalOpt match { + case Some(hermesTotal) => Some(Statistics(total, hermesTotal)) + case _ => None + } + } + case _ => None + } + + } + case result => { + log.warn(s"Request failed: $result") + None } } } diff --git a/src/test/scala/de/upb/cs/swt/delphi/webapi/MavenIdentifierApplyTest.scala b/src/test/scala/de/upb/cs/swt/delphi/webapi/MavenIdentifierApplyTest.scala new file mode 100644 index 0000000..ae55a7c --- /dev/null +++ b/src/test/scala/de/upb/cs/swt/delphi/webapi/MavenIdentifierApplyTest.scala @@ -0,0 +1,61 @@ +// Copyright (C) 2018 The Delphi Team. +// See the LICENCE file distributed with this work for additional +// information regarding copyright ownership. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package de.upb.cs.swt.delphi.webapi + +import org.scalatest.{FlatSpec, Matchers} + +class MavenIdentifierApplyTest extends FlatSpec with Matchers { + "Valid identifiers with version" should "convert nicely" in { + val fullIdentifier = MavenIdentifier("log4j:log4j-test:1.4.2") + fullIdentifier match { + case Some(m) => { + m.repository.isDefined shouldBe false + m.groupId shouldBe "log4j" + m.artifactId shouldBe "log4j-test" + m.version.isDefined shouldBe true + m.version.getOrElse("") shouldBe "1.4.2" + } + case _ => fail("Identifier could not be properly parsed.") + } + } + "Valid identifiers without version" should "convert nicely" in { + val partial = MavenIdentifier("log4j:log4j-test") + partial match { + case Some(m) => { + m.repository.isDefined shouldBe false + m.groupId shouldBe "log4j" + m.artifactId shouldBe "log4j-test" + m.version.isDefined shouldBe false + } + case _ => fail("Identifier could not be properly parsed.") + } + } + + "Invalid identifiers" should "fail" in { + val invalid = MavenIdentifier("log4j") + invalid.isDefined shouldBe false + + val invalid2 = MavenIdentifier("log4j:::") + invalid2.isDefined shouldBe false + + val invalid3 = MavenIdentifier("log4j::") + invalid3.isDefined shouldBe false + + val invalid4 = MavenIdentifier("") + invalid4.isDefined shouldBe false + } +} From 4638cd6d56f19f1a34d616b7aedbcef5f044c08f Mon Sep 17 00:00:00 2001 From: Ben Hermann Date: Sat, 27 Oct 2018 02:49:37 +0200 Subject: [PATCH 27/43] Fixed termination inside of sbt console --- build.sbt | 2 ++ src/main/scala/de/upb/cs/swt/delphi/webapi/Server.scala | 8 ++++---- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/build.sbt b/build.sbt index 428f433..4948af6 100644 --- a/build.sbt +++ b/build.sbt @@ -57,3 +57,5 @@ scalastyleConfig := baseDirectory.value / "project" / "scalastyle-config.xml" libraryDependencies ++= Seq( "com.fasterxml.jackson.core" % "jackson-databind" % "2.9.7" ) + +trapExit := false diff --git a/src/main/scala/de/upb/cs/swt/delphi/webapi/Server.scala b/src/main/scala/de/upb/cs/swt/delphi/webapi/Server.scala index c15de85..7eb5cb5 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/webapi/Server.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/webapi/Server.scala @@ -26,9 +26,9 @@ import akka.util.Timeout import de.upb.cs.swt.delphi.featuredefinitions.FeatureListMapping import de.upb.cs.swt.delphi.instancemanagement.InstanceRegistry import spray.json._ - import ArtifactJson._ -import scala.concurrent.ExecutionContext + +import scala.concurrent.{ExecutionContext, Future} /** * Web server configuration for Delphi web API. @@ -122,11 +122,11 @@ object Server extends HttpApp with JsonSupport with AppLogging { StartupCheck.check(configuration) Server.startServer(configuration.bindHost, configuration.bindPort, system) - implicit val ec: ExecutionContext = ExecutionContext.global + implicit val ec: ExecutionContext = system.dispatcher val terminationFuture = system.terminate() terminationFuture.onComplete { - sys.exit() + sys.exit(0) } } From 35f7ca556efcd4ea8477a8957654b8b13de7c33f Mon Sep 17 00:00:00 2001 From: Ben Hermann Date: Sat, 27 Oct 2018 03:06:21 +0200 Subject: [PATCH 28/43] Changed prettyPrint to sortedPrint --- src/main/scala/de/upb/cs/swt/delphi/webapi/RetrieveQuery.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/scala/de/upb/cs/swt/delphi/webapi/RetrieveQuery.scala b/src/main/scala/de/upb/cs/swt/delphi/webapi/RetrieveQuery.scala index 5a59102..629897a 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/webapi/RetrieveQuery.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/webapi/RetrieveQuery.scala @@ -127,7 +127,7 @@ object ArtifactJson extends DefaultJsonProtocol { def prettyPrint(pretty: Option[_], value: JsValue): String = { pretty.isDefined match { - case true => value.prettyPrint + case true => value.sortedPrint case false => value.compactPrint } } From 76ec7515f4bec674fa0601b987afeb7c8ef272df Mon Sep 17 00:00:00 2001 From: Ben Hermann Date: Sat, 27 Oct 2018 16:35:13 +0200 Subject: [PATCH 29/43] Took feature list now directly from index mapping --- .../delphi/webapi/FeatureExtractorCheck.scala | 29 ++++++++++ .../featuredefinitions/FeatureExtractor.scala | 55 +++++++++++++++++++ .../FeatureListMapping.scala | 1 + .../de/upb/cs/swt/delphi/webapi/Server.scala | 29 ++++------ 4 files changed, 96 insertions(+), 18 deletions(-) create mode 100644 src/it/scala/de/upb/cs/swt/delphi/webapi/FeatureExtractorCheck.scala create mode 100644 src/main/scala/de/upb/cs/swt/delphi/featuredefinitions/FeatureExtractor.scala diff --git a/src/it/scala/de/upb/cs/swt/delphi/webapi/FeatureExtractorCheck.scala b/src/it/scala/de/upb/cs/swt/delphi/webapi/FeatureExtractorCheck.scala new file mode 100644 index 0000000..b75cddf --- /dev/null +++ b/src/it/scala/de/upb/cs/swt/delphi/webapi/FeatureExtractorCheck.scala @@ -0,0 +1,29 @@ +// Copyright (C) 2018 The Delphi Team. +// See the LICENCE file distributed with this work for additional +// information regarding copyright ownership. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package de.upb.cs.swt.delphi.webapi + +import de.upb.cs.swt.delphi.featuredefinitions.FeatureExtractor +import org.scalatest.{FlatSpec, Matchers} + +class FeatureExtractorCheck extends FlatSpec with Matchers { + "FeatureExtractor" should "provide a list of features" in { + val configuration = new Configuration() + val featureList = new FeatureExtractor(configuration).featureList + + featureList.size should be > 0 + } +} diff --git a/src/main/scala/de/upb/cs/swt/delphi/featuredefinitions/FeatureExtractor.scala b/src/main/scala/de/upb/cs/swt/delphi/featuredefinitions/FeatureExtractor.scala new file mode 100644 index 0000000..a79ef0f --- /dev/null +++ b/src/main/scala/de/upb/cs/swt/delphi/featuredefinitions/FeatureExtractor.scala @@ -0,0 +1,55 @@ +// Copyright (C) 2018 The Delphi Team. +// See the LICENCE file distributed with this work for additional +// information regarding copyright ownership. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package de.upb.cs.swt.delphi.featuredefinitions + +import com.sksamuel.elastic4s.http.{ElasticClient, RequestSuccess} +import com.sksamuel.elastic4s.http.ElasticDsl._ +import com.sksamuel.elastic4s.http.index.mappings.IndexMappings +import de.upb.cs.swt.delphi.webapi.Configuration +import org.slf4j.LoggerFactory +import spray.json._ + +class FeatureExtractor(configuration: Configuration) { + private val log = LoggerFactory.getLogger(this.getClass) + + lazy val featureList: Iterable[String] = { + val client = ElasticClient(configuration.elasticsearchClientUri) + val mappingRequest = client.execute { + getMapping(configuration.esProjectIndex) + }.await + + mappingRequest match { + case RequestSuccess(_, body, _, mappings) => { + val bodyJson = body.getOrElse("").parseJson.asJsObject + + bodyJson + .fields("delphi").asJsObject + .fields("mappings").asJsObject + .fields("project").asJsObject + .fields("properties").asJsObject + .fields("hermes").asJsObject + .fields("properties").asJsObject + .fields("features").asJsObject + .fields("properties").asJsObject.fields.keys + } + case _ => { + log.warn(s"Could not retrieve current feature list. Error was: $mappingRequest") + List() + } + } + } +} diff --git a/src/main/scala/de/upb/cs/swt/delphi/featuredefinitions/FeatureListMapping.scala b/src/main/scala/de/upb/cs/swt/delphi/featuredefinitions/FeatureListMapping.scala index e099885..9cd13e4 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/featuredefinitions/FeatureListMapping.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/featuredefinitions/FeatureListMapping.scala @@ -18,6 +18,7 @@ package de.upb.cs.swt.delphi.featuredefinitions object FeatureListMapping { + //Returns a list of all defined features def featureList: List[String] = (for {(key, des) <- featureMap if des != null} yield key).toList diff --git a/src/main/scala/de/upb/cs/swt/delphi/webapi/Server.scala b/src/main/scala/de/upb/cs/swt/delphi/webapi/Server.scala index 7eb5cb5..4689d1b 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/webapi/Server.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/webapi/Server.scala @@ -23,7 +23,7 @@ import akka.http.scaladsl.model.{HttpResponse, StatusCodes} import akka.http.scaladsl.server.{HttpApp, Route} import akka.stream.ActorMaterializer import akka.util.Timeout -import de.upb.cs.swt.delphi.featuredefinitions.FeatureListMapping +import de.upb.cs.swt.delphi.featuredefinitions.{FeatureExtractor, FeatureListMapping} import de.upb.cs.swt.delphi.instancemanagement.InstanceRegistry import spray.json._ import ArtifactJson._ @@ -39,27 +39,16 @@ object Server extends HttpApp with JsonSupport with AppLogging { implicit val materializer = ActorMaterializer() private implicit val configuration = new Configuration() - private val actorManager = system.actorOf(ElasticActorManager.props(configuration)) - private val requestLimiter = system.actorOf(ElasticRequestLimiter.props(configuration, actorManager)) private implicit val timeout = Timeout(5, TimeUnit.SECONDS) override def routes: Route = - path("version") { - version - } ~ - path("features") { - features - } ~ - path("statistics") { - statistics - } ~ + path("version") { version } ~ + path("features") { features } ~ + path("statistics") { statistics } ~ pathPrefix("search" / Remaining) { query => search(query) } ~ pathPrefix("retrieve" / Remaining) { identifier => retrieve(identifier) } - //~ - //pathPrefix("enqueue" / Remaining) { identifier => enqueue(identifier) } - private def version = { get { @@ -69,10 +58,14 @@ object Server extends HttpApp with JsonSupport with AppLogging { } } + private val featureExtractor = new FeatureExtractor(configuration) + private def features = { get { - complete { - FeatureListMapping.featureList.toJson + parameter('pretty.?) { (pretty) => + complete( + prettyPrint(pretty, featureExtractor.featureList.toJson) + ) } } } @@ -86,7 +79,7 @@ object Server extends HttpApp with JsonSupport with AppLogging { import StatisticsJson._ stats.toJson } - case _ => "Failure" + case _ => HttpResponse(StatusCodes.InternalServerError) } } } From 06bb1a930ce0d2ec65887ce18782cf7045c9e84d Mon Sep 17 00:00:00 2001 From: Ben Hermann Date: Sat, 27 Oct 2018 16:43:20 +0200 Subject: [PATCH 30/43] Reformatted and added pretty printing to the statistics endpoint --- .../de/upb/cs/swt/delphi/webapi/Server.scala | 28 ++++++++++++------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/src/main/scala/de/upb/cs/swt/delphi/webapi/Server.scala b/src/main/scala/de/upb/cs/swt/delphi/webapi/Server.scala index 4689d1b..71af2db 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/webapi/Server.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/webapi/Server.scala @@ -43,9 +43,15 @@ object Server extends HttpApp with JsonSupport with AppLogging { override def routes: Route = - path("version") { version } ~ - path("features") { features } ~ - path("statistics") { statistics } ~ + path("version") { + version + } ~ + path("features") { + features + } ~ + path("statistics") { + statistics + } ~ pathPrefix("search" / Remaining) { query => search(query) } ~ pathPrefix("retrieve" / Remaining) { identifier => retrieve(identifier) } @@ -72,14 +78,16 @@ object Server extends HttpApp with JsonSupport with AppLogging { private def statistics = { get { - complete { - val result = new StatisticsQuery(configuration).retrieveStandardStatistics - result match { - case Some(stats) => { - import StatisticsJson._ - stats.toJson + parameter('pretty.?) { (pretty) => + complete { + val result = new StatisticsQuery(configuration).retrieveStandardStatistics + result match { + case Some(stats) => { + import StatisticsJson._ + prettyPrint(pretty, stats.toJson) + } + case _ => HttpResponse(StatusCodes.InternalServerError) } - case _ => HttpResponse(StatusCodes.InternalServerError) } } } From dd3d624d1cf848f478ca7da23c8f65c5a15f2638 Mon Sep 17 00:00:00 2001 From: Ben Hermann Date: Sat, 27 Oct 2018 23:53:48 +0200 Subject: [PATCH 31/43] Implemented search endpoing --- .../swt/delphi/webapi/ElasticActorTest.scala | 3 +- .../swt/delphi/webapi/SearchQueryTest.scala | 32 +++++ .../upb/cs/swt/delphi/querylanguage/AST.scala | 17 +-- .../cs/swt/delphi/querylanguage/Syntax.scala | 19 +-- .../cs/swt/delphi/webapi/RetrieveQuery.scala | 51 +------- .../cs/swt/delphi/webapi/SearchQuery.scala | 121 ++++++++++++++++++ .../de/upb/cs/swt/delphi/webapi/Server.scala | 25 +++- .../delphi/webapi/artifacts/Artifact.scala | 19 +++ .../webapi/artifacts/ArtifactJson.scala | 46 +++++++ .../webapi/artifacts/ArtifactMetadata.scala | 21 +++ .../artifacts/ArtifactTransformer.scala | 51 ++++++++ .../swt/delphi/querylanguage/SyntaxTest.scala | 80 ++++++------ 12 files changed, 375 insertions(+), 110 deletions(-) create mode 100644 src/it/scala/de/upb/cs/swt/delphi/webapi/SearchQueryTest.scala create mode 100644 src/main/scala/de/upb/cs/swt/delphi/webapi/SearchQuery.scala create mode 100644 src/main/scala/de/upb/cs/swt/delphi/webapi/artifacts/Artifact.scala create mode 100644 src/main/scala/de/upb/cs/swt/delphi/webapi/artifacts/ArtifactJson.scala create mode 100644 src/main/scala/de/upb/cs/swt/delphi/webapi/artifacts/ArtifactMetadata.scala create mode 100644 src/main/scala/de/upb/cs/swt/delphi/webapi/artifacts/ArtifactTransformer.scala diff --git a/src/it/scala/de/upb/cs/swt/delphi/webapi/ElasticActorTest.scala b/src/it/scala/de/upb/cs/swt/delphi/webapi/ElasticActorTest.scala index de3e8a1..4857cca 100644 --- a/src/it/scala/de/upb/cs/swt/delphi/webapi/ElasticActorTest.scala +++ b/src/it/scala/de/upb/cs/swt/delphi/webapi/ElasticActorTest.scala @@ -24,7 +24,7 @@ import akka.stream.ActorMaterializer import com.sksamuel.elastic4s.RefreshPolicy import com.sksamuel.elastic4s.http.ElasticClient import com.sksamuel.elastic4s.http.ElasticDsl._ -import org.scalatest.{BeforeAndAfterAll, FlatSpec, Matchers} +import org.scalatest.{BeforeAndAfterAll, FlatSpec, Ignore, Matchers} import scala.concurrent.duration._ import scala.concurrent.{Await, Future} @@ -33,6 +33,7 @@ import scala.util.{Failure, Success} /** * @author Hariharan. */ +@Ignore class ElasticActorTest extends FlatSpec with Matchers with BeforeAndAfterAll { implicit val system = ActorSystem() implicit val materializer = ActorMaterializer()(system) diff --git a/src/it/scala/de/upb/cs/swt/delphi/webapi/SearchQueryTest.scala b/src/it/scala/de/upb/cs/swt/delphi/webapi/SearchQueryTest.scala new file mode 100644 index 0000000..3370900 --- /dev/null +++ b/src/it/scala/de/upb/cs/swt/delphi/webapi/SearchQueryTest.scala @@ -0,0 +1,32 @@ +// Copyright (C) 2018 The Delphi Team. +// See the LICENCE file distributed with this work for additional +// information regarding copyright ownership. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package de.upb.cs.swt.delphi.webapi + +import de.upb.cs.swt.delphi.featuredefinitions.FeatureExtractor +import org.scalatest.{FlatSpec, Matchers} +import de.upb.cs.swt.delphi.webapi.artifacts.ArtifactJson._ +import spray.json._ + +class SearchQueryTest extends FlatSpec with Matchers { + "Search query" should "check for fields" in { + val configuration = new Configuration() + val q = new SearchQuery(configuration, new FeatureExtractor(configuration)) + + val response = q.search("[if_icmpeq (opcode:159)]>1") + println(response.get.toJson) + } +} diff --git a/src/main/scala/de/upb/cs/swt/delphi/querylanguage/AST.scala b/src/main/scala/de/upb/cs/swt/delphi/querylanguage/AST.scala index 54d2579..64d0d09 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/querylanguage/AST.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/querylanguage/AST.scala @@ -25,11 +25,12 @@ case class XorExpr(left: CombinatorialExpr, right: CombinatorialExpr) extends Co trait SingularConditionExpr extends CombinatorialExpr -case class EqualExpr(left: String, right: String) extends SingularConditionExpr -case class NotEqualExpr(left: String, right: String) extends SingularConditionExpr -case class GreaterThanExpr(left: String, right: String) extends SingularConditionExpr -case class GreaterOrEqualExpr(left: String, right: String) extends SingularConditionExpr -case class LessThanExpr(left: String, right: String) extends SingularConditionExpr -case class LessOrEqualExpr(left: String, right: String) extends SingularConditionExpr -case class LikeExpr(left: String, right: String) extends SingularConditionExpr -case class TrueExpr(expr: String) extends SingularConditionExpr +case class EqualExpr(left: FieldReference, right: String) extends SingularConditionExpr +case class NotEqualExpr(left: FieldReference, right: String) extends SingularConditionExpr +case class GreaterThanExpr(left: FieldReference, right: String) extends SingularConditionExpr +case class GreaterOrEqualExpr(left: FieldReference, right: String) extends SingularConditionExpr +case class LessThanExpr(left: FieldReference, right: String) extends SingularConditionExpr +case class LessOrEqualExpr(left: FieldReference, right: String) extends SingularConditionExpr +case class LikeExpr(left: FieldReference, right: String) extends SingularConditionExpr +case class IsTrueExpr(fieldName: FieldReference) extends SingularConditionExpr +case class FieldReference(fieldName: String) extends CombinatorialExpr \ No newline at end of file diff --git a/src/main/scala/de/upb/cs/swt/delphi/querylanguage/Syntax.scala b/src/main/scala/de/upb/cs/swt/delphi/querylanguage/Syntax.scala index 016ec29..69d9a31 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/querylanguage/Syntax.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/querylanguage/Syntax.scala @@ -55,18 +55,19 @@ class Syntax(val input : ParserInput) extends Parser { // Singular conditions. def SingularConditionRule = rule { EqualRule | NotEqualRule | GreaterThanRule | GreaterOrEqual | - LessThan | LessOrEqual | Like | True + LessThan | LessOrEqual | Like | IsTrue } - def EqualRule = rule { Literal ~ "=" ~ Literal ~> EqualExpr } - def NotEqualRule = rule { Literal ~ "!=" ~ Literal ~> NotEqualExpr } - def GreaterThanRule = rule { Literal ~ ">" ~ Literal ~> GreaterThanExpr } - def GreaterOrEqual = rule { Literal ~ ">=" ~ Literal ~> GreaterOrEqualExpr } - def LessThan = rule { Literal ~ "<" ~ Literal ~> LessThanExpr } - def LessOrEqual = rule { Literal ~ "<=" ~ Literal ~> LessOrEqualExpr } - def Like = rule { Literal ~ "%" ~ Literal ~> LikeExpr } - def True = rule { Literal ~> TrueExpr } + def EqualRule = rule { FieldReferenceRule ~ "=" ~ Literal ~> EqualExpr } + def NotEqualRule = rule { FieldReferenceRule ~ "!=" ~ Literal ~> NotEqualExpr } + def GreaterThanRule = rule { FieldReferenceRule ~ ">" ~ Literal ~> GreaterThanExpr } + def GreaterOrEqual = rule { FieldReferenceRule ~ ">=" ~ Literal ~> GreaterOrEqualExpr } + def LessThan = rule { FieldReferenceRule ~ "<" ~ Literal ~> LessThanExpr } + def LessOrEqual = rule { FieldReferenceRule ~ "<=" ~ Literal ~> LessOrEqualExpr } + def Like = rule { FieldReferenceRule ~ "%" ~ Literal ~> LikeExpr } + def IsTrue = rule { FieldReferenceRule ~> IsTrueExpr } // Literals + def FieldReferenceRule = rule { "[" ~ capture(oneOrMore(CharPredicate.AlphaNum ++ ' ' ++ '_' ++ '(' ++ ':' ++')')) ~ "]" ~> FieldReference } def Literal = rule { capture(oneOrMore(CharPredicate.AlphaNum)) ~> (_.toString) } } diff --git a/src/main/scala/de/upb/cs/swt/delphi/webapi/RetrieveQuery.scala b/src/main/scala/de/upb/cs/swt/delphi/webapi/RetrieveQuery.scala index 629897a..f059727 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/webapi/RetrieveQuery.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/webapi/RetrieveQuery.scala @@ -21,6 +21,7 @@ import com.sksamuel.elastic4s.http.ElasticDsl.{termQuery, _} import com.sksamuel.elastic4s.http.search.{SearchHit, SearchHits, SearchResponse} import com.sksamuel.elastic4s.searches.queries.Query import com.sksamuel.elastic4s.searches.queries.term.TermQuery +import de.upb.cs.swt.delphi.webapi.artifacts.{Artifact, ArtifactMetadata, ArtifactTransformer} import org.joda.time.DateTime import org.joda.time.format.{DateTimeFormatter, ISODateTimeFormat} import spray.json.{DefaultJsonProtocol, DeserializationException, JsString, JsValue, RootJsonFormat} @@ -50,7 +51,7 @@ object RetrieveQuery { results.totalHits match { case 0L => None case _ => { - Some(transformResults(results.hits)) + Some(ArtifactTransformer.transformResults(results.hits)) } } } @@ -77,59 +78,11 @@ object RetrieveQuery { } } - private def getHermesResults(sourceMap: Map[String, AnyRef]): Map[String, Int] = { - if (!sourceMap.contains("hermes")) return Map() - if (!sourceMap("hermes").isInstanceOf[Map[String, AnyRef]]) return Map() - if (!sourceMap("hermes").asInstanceOf[Map[String, AnyRef]].contains("features")) return Map() - - val hermesMap = sourceMap("hermes").asInstanceOf[Map[String, AnyRef]] - - hermesMap("features").asInstanceOf[Map[String, Int]] - } - - def getMetadata(sourceMap: Map[String, AnyRef]): ArtifactMetadata = { - val identifier = sourceMap("identifier").asInstanceOf[Map[String, String]] - ArtifactMetadata(sourceMap("source").asInstanceOf[String], - ISODateTimeFormat.dateTime().parseDateTime(sourceMap("discovered").asInstanceOf[String]), - identifier("groupId"), identifier("artifactId"), identifier("version")) - } - - private def transformResult(h: SearchHit) = { - val sourceMap = h.sourceAsMap - Artifact(h.id, getMetadata(sourceMap), getHermesResults(sourceMap)) - } - - private def transformResults(hits: SearchHits) = { - hits.hits.map(h => transformResult(h)) - } } -case class Artifact(id: String, metadata: ArtifactMetadata, metricResults: Map[String, Int]) -case class ArtifactMetadata(source: String, discovered: DateTime, groupId: String, artifactId: String, version: String) -object ArtifactJson extends DefaultJsonProtocol { - implicit object DateJsonFormat extends RootJsonFormat[DateTime] { - private val parserISO: DateTimeFormatter = ISODateTimeFormat.dateTime() - override def write(obj: DateTime) = JsString(parserISO.print(obj)) - - override def read(json: JsValue): DateTime = json match { - case JsString(s) => parserISO.parseDateTime(s) - case _ => throw new DeserializationException("Error info you want here ...") - } - } - - implicit val artifactMetadataFormat = jsonFormat5(ArtifactMetadata) - implicit val artifactFormat = jsonFormat3(Artifact) - - def prettyPrint(pretty: Option[_], value: JsValue): String = { - pretty.isDefined match { - case true => value.sortedPrint - case false => value.compactPrint - } - } -} diff --git a/src/main/scala/de/upb/cs/swt/delphi/webapi/SearchQuery.scala b/src/main/scala/de/upb/cs/swt/delphi/webapi/SearchQuery.scala new file mode 100644 index 0000000..b32c7dd --- /dev/null +++ b/src/main/scala/de/upb/cs/swt/delphi/webapi/SearchQuery.scala @@ -0,0 +1,121 @@ +// Copyright (C) 2018 The Delphi Team. +// See the LICENCE file distributed with this work for additional +// information regarding copyright ownership. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package de.upb.cs.swt.delphi.webapi + +import com.sksamuel.elastic4s.http.{ElasticClient, RequestSuccess} +import de.upb.cs.swt.delphi.featuredefinitions.FeatureExtractor +import de.upb.cs.swt.delphi.querylanguage._ +import com.sksamuel.elastic4s.http.ElasticDsl._ +import com.sksamuel.elastic4s.http.search.SearchHits +import com.sksamuel.elastic4s.searches.queries.{NoopQuery, Query} +import de.upb.cs.swt.delphi.webapi.artifacts.ArtifactTransformer + +import scala.util.{Failure, Success, Try} + +class SearchQuery(configuration: Configuration, featureExtractor: FeatureExtractor) { + private val client = ElasticClient(configuration.elasticsearchClientUri) + + private def checkAndExecuteParsedQuery(ast: CombinatorialExpr): Try[SearchHits] = { + val fields = collectFieldNames(ast) + if (fields.diff(featureExtractor.featureList.toSeq).size > 0) return Failure(null) + + val query = searchWithType(configuration.esProjectIndex) query translate(ast) sourceInclude (ArtifactTransformer.baseFields ++ fields.intersect(featureExtractor.featureList.toSeq).map(i => addPrefix(i))) + + val response = client.execute { + query + }.await + + response match { + case RequestSuccess(_, body, _, result) => Success(result.hits) + case r => Failure(new IllegalArgumentException(r.toString)) + } + } + + private def addPrefix(fieldName: String): String = s"hermes.features.$fieldName" + + private def translate(node: CombinatorialExpr): Query = { + node match { + case AndExpr(left, right) => bool { + must( + translate(left), + translate(right) + ) + } + case OrExpr(left, right) => bool { + should( + translate(left), + translate(right) + ) + } + case NotExpr(expr) => bool { + not(translate(expr)) + } + case XorExpr(left, right) => bool { + should( + must( + translate(left), + not(translate(right)) + ), + must( + not(translate(right)), + translate(left) + ) + ) + } + case EqualExpr(field, value) => matchQuery(addPrefix(field.fieldName), value) + case NotEqualExpr(field, value) => bool(not(matchQuery(addPrefix(field.fieldName), value))) + case GreaterThanExpr(field, value) => rangeQuery(addPrefix(field.fieldName)).gt(value.toLong) + case GreaterOrEqualExpr(field, value) => rangeQuery(addPrefix(field.fieldName)).gte(value.toLong) + case LessThanExpr(field, value) => rangeQuery(addPrefix(field.fieldName)).lt(value.toLong) + case LessOrEqualExpr(field, value) => rangeQuery(addPrefix(field.fieldName)).lte(value.toLong) + case LikeExpr(field, value) => prefixQuery(addPrefix(field.fieldName), value) + case _ => NoopQuery + } + } + + private def collectFieldNames(node: CombinatorialExpr): Seq[String] = { + node match { + case AndExpr(left, right) => collectFieldNames(left) ++ collectFieldNames(right) + case OrExpr(left, right) => collectFieldNames(left) ++ collectFieldNames(right) + case NotExpr(expr) => collectFieldNames(expr) + case XorExpr(left, right) => collectFieldNames(left) ++ collectFieldNames(right) + case EqualExpr(field, _) => Seq(field.fieldName) + case NotEqualExpr(field, _) => Seq(field.fieldName) + case GreaterThanExpr(field, _) => Seq(field.fieldName) + case GreaterOrEqualExpr(field, _) => Seq(field.fieldName) + case LessThanExpr(field, _) => Seq(field.fieldName) + case LessOrEqualExpr(field, _) => Seq(field.fieldName) + case LikeExpr(field, _) => Seq(field.fieldName) + case IsTrueExpr(field) => Seq(field.fieldName) + case FieldReference(name) => Seq(name) + case _ => Seq() + } + } + + def search(query: String) = { + val parserResult = new Syntax(query).QueryRule.run() + parserResult match { + case Failure(e) => Failure(e) + case Success(ast) => { + checkAndExecuteParsedQuery(ast) match { + case Failure(e) => Failure(e) + case Success(hits) => Success(ArtifactTransformer.transformResults(hits)) + } + } + } + } +} diff --git a/src/main/scala/de/upb/cs/swt/delphi/webapi/Server.scala b/src/main/scala/de/upb/cs/swt/delphi/webapi/Server.scala index 71af2db..9f9fa22 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/webapi/Server.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/webapi/Server.scala @@ -26,9 +26,10 @@ import akka.util.Timeout import de.upb.cs.swt.delphi.featuredefinitions.{FeatureExtractor, FeatureListMapping} import de.upb.cs.swt.delphi.instancemanagement.InstanceRegistry import spray.json._ -import ArtifactJson._ +import de.upb.cs.swt.delphi.webapi.artifacts.ArtifactJson._ import scala.concurrent.{ExecutionContext, Future} +import scala.util.{Failure, Success} /** * Web server configuration for Delphi web API. @@ -52,7 +53,7 @@ object Server extends HttpApp with JsonSupport with AppLogging { path("statistics") { statistics } ~ - pathPrefix("search" / Remaining) { query => search(query) } ~ + pathPrefix("search" ) { search } ~ pathPrefix("retrieve" / Remaining) { identifier => retrieve(identifier) } @@ -70,6 +71,7 @@ object Server extends HttpApp with JsonSupport with AppLogging { get { parameter('pretty.?) { (pretty) => complete( + //TODO: Introduce failure concept for feature extractor prettyPrint(pretty, featureExtractor.featureList.toJson) ) } @@ -98,7 +100,7 @@ object Server extends HttpApp with JsonSupport with AppLogging { parameter('pretty.?) { (pretty) => complete( RetrieveQuery.retrieve(identifier) match { - case Some(result: Any) => prettyPrint(pretty, result.toJson) + case Some(result) => prettyPrint(pretty, result.toJson) case None => HttpResponse(StatusCodes.NotFound) } ) @@ -106,10 +108,19 @@ object Server extends HttpApp with JsonSupport with AppLogging { } } - def search(query: String): Route = { - get { - complete { - query + def search: Route = { + post { + parameter('pretty.?) { (pretty) => + entity(as[JsValue]) { input => + val query = input.asJsObject.fields("query").convertTo[String] + log.info(s"Received search query: $query") + complete( + new SearchQuery(configuration, featureExtractor).search(query) match { + case Success(result) => prettyPrint(pretty, result.toJson) + case Failure(e) => e.toString + } + ) + } } } } diff --git a/src/main/scala/de/upb/cs/swt/delphi/webapi/artifacts/Artifact.scala b/src/main/scala/de/upb/cs/swt/delphi/webapi/artifacts/Artifact.scala new file mode 100644 index 0000000..265409b --- /dev/null +++ b/src/main/scala/de/upb/cs/swt/delphi/webapi/artifacts/Artifact.scala @@ -0,0 +1,19 @@ +// Copyright (C) 2018 The Delphi Team. +// See the LICENCE file distributed with this work for additional +// information regarding copyright ownership. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package de.upb.cs.swt.delphi.webapi.artifacts + +case class Artifact(id: String, metadata: ArtifactMetadata, metricResults: Map[String, Int]) diff --git a/src/main/scala/de/upb/cs/swt/delphi/webapi/artifacts/ArtifactJson.scala b/src/main/scala/de/upb/cs/swt/delphi/webapi/artifacts/ArtifactJson.scala new file mode 100644 index 0000000..617815c --- /dev/null +++ b/src/main/scala/de/upb/cs/swt/delphi/webapi/artifacts/ArtifactJson.scala @@ -0,0 +1,46 @@ +// Copyright (C) 2018 The Delphi Team. +// See the LICENCE file distributed with this work for additional +// information regarding copyright ownership. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package de.upb.cs.swt.delphi.webapi.artifacts + +import org.joda.time.DateTime +import org.joda.time.format.{DateTimeFormatter, ISODateTimeFormat} +import spray.json.{DefaultJsonProtocol, DeserializationException, JsString, JsValue, RootJsonFormat} + +object ArtifactJson extends DefaultJsonProtocol { + + implicit object DateJsonFormat extends RootJsonFormat[DateTime] { + + private val parserISO: DateTimeFormatter = ISODateTimeFormat.dateTime() + + override def write(obj: DateTime) = JsString(parserISO.print(obj)) + + override def read(json: JsValue): DateTime = json match { + case JsString(s) => parserISO.parseDateTime(s) + case _ => throw new DeserializationException("Error info you want here ...") + } + } + + implicit val artifactMetadataFormat = jsonFormat5(ArtifactMetadata) + implicit val artifactFormat = jsonFormat3(Artifact) + + def prettyPrint(pretty: Option[_], value: JsValue): String = { + pretty.isDefined match { + case true => value.sortedPrint + case false => value.compactPrint + } + } +} diff --git a/src/main/scala/de/upb/cs/swt/delphi/webapi/artifacts/ArtifactMetadata.scala b/src/main/scala/de/upb/cs/swt/delphi/webapi/artifacts/ArtifactMetadata.scala new file mode 100644 index 0000000..f35dbfb --- /dev/null +++ b/src/main/scala/de/upb/cs/swt/delphi/webapi/artifacts/ArtifactMetadata.scala @@ -0,0 +1,21 @@ +// Copyright (C) 2018 The Delphi Team. +// See the LICENCE file distributed with this work for additional +// information regarding copyright ownership. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package de.upb.cs.swt.delphi.webapi.artifacts + +import org.joda.time.DateTime + +case class ArtifactMetadata(source: String, discovered: DateTime, groupId: String, artifactId: String, version: String) diff --git a/src/main/scala/de/upb/cs/swt/delphi/webapi/artifacts/ArtifactTransformer.scala b/src/main/scala/de/upb/cs/swt/delphi/webapi/artifacts/ArtifactTransformer.scala new file mode 100644 index 0000000..6e429d4 --- /dev/null +++ b/src/main/scala/de/upb/cs/swt/delphi/webapi/artifacts/ArtifactTransformer.scala @@ -0,0 +1,51 @@ +// Copyright (C) 2018 The Delphi Team. +// See the LICENCE file distributed with this work for additional +// information regarding copyright ownership. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package de.upb.cs.swt.delphi.webapi.artifacts + +import com.sksamuel.elastic4s.http.search.{SearchHit, SearchHits} +import org.joda.time.format.ISODateTimeFormat + +object ArtifactTransformer { + + private def getHermesResults(sourceMap: Map[String, AnyRef]): Map[String, Int] = { + if (!sourceMap.contains("hermes")) return Map() + if (!sourceMap("hermes").isInstanceOf[Map[String, AnyRef]]) return Map() + if (!sourceMap("hermes").asInstanceOf[Map[String, AnyRef]].contains("features")) return Map() + + val hermesMap = sourceMap("hermes").asInstanceOf[Map[String, AnyRef]] + + hermesMap("features").asInstanceOf[Map[String, Int]] + } + + private def getMetadata(sourceMap: Map[String, AnyRef]): ArtifactMetadata = { + val identifier = sourceMap("identifier").asInstanceOf[Map[String, String]] + ArtifactMetadata(sourceMap("source").asInstanceOf[String], + ISODateTimeFormat.dateTime().parseDateTime(sourceMap("discovered").asInstanceOf[String]), + identifier("groupId"), identifier("artifactId"), identifier("version")) + } + + private def transformResult(h: SearchHit) = { + val sourceMap = h.sourceAsMap + Artifact(h.id, getMetadata(sourceMap), getHermesResults(sourceMap)) + } + + def transformResults(hits: SearchHits) = { + hits.hits.map(h => transformResult(h)) + } + + val baseFields = Seq("source", "discovered", "identifier.groupId", "identifier.artifactId", "identifier.version") +} diff --git a/src/test/scala/de/upb/cs/swt/delphi/querylanguage/SyntaxTest.scala b/src/test/scala/de/upb/cs/swt/delphi/querylanguage/SyntaxTest.scala index 41d1e68..179c75a 100644 --- a/src/test/scala/de/upb/cs/swt/delphi/querylanguage/SyntaxTest.scala +++ b/src/test/scala/de/upb/cs/swt/delphi/querylanguage/SyntaxTest.scala @@ -27,141 +27,149 @@ import scala.util.{Failure, Success} class SyntaxTest extends FlatSpec with Matchers { "Syntax.singularConditionWithOperator" should "be valid" in { - val parseResult = new Syntax("Filter1=abc").QueryRule.run() + val parseResult = new Syntax("[Filter1]=abc").QueryRule.run() parseResult shouldBe a [Success[_]] parseResult match { case Success(ast) => { - ast.toString shouldEqual "EqualExpr(Filter1,abc)" + ast.toString shouldEqual "EqualExpr(FieldReference(Filter1),abc)" } } } + "Syntax.singularConditionNoOperator" should "be valid" in { - val parseResult = new Syntax("Filter1").QueryRule.run() + val parseResult = new Syntax("[Filter1]").QueryRule.run() parseResult shouldBe a [Success[_]] parseResult match { case Success(ast) => { - ast.toString shouldEqual "TrueExpr(Filter1)" + ast.toString shouldEqual "IsTrueExpr(FieldReference(Filter1))" } } } + "Syntax.singularConditionTypo" should "be valid" in { - val parseResult = new Syntax("Filter1=<3").QueryRule.run() + val parseResult = new Syntax("[Filter1]=<3").QueryRule.run() parseResult shouldBe a [Failure[_]] } "Syntax.singularConditionOddCharacters" should "be valid" in { - val parseResult = new Syntax("Filter1%Filter2%'}.:").QueryRule.run() + val parseResult = new Syntax("[Filter1]%[Filter2]%'}.:").QueryRule.run() parseResult shouldBe a [Failure[_]] } + "Syntax.combinatoryConditionSimple" should "be valid" in { - val parseResult = new Syntax("Filter1&&Filter2=3").QueryRule.run() + val parseResult = new Syntax("[Filter1]&&[Filter2]=3").QueryRule.run() parseResult shouldBe a [Success[_]] parseResult match { case Success(ast) => { - ast.toString shouldEqual "AndExpr(TrueExpr(Filter1),EqualExpr(Filter2,3))" + ast.toString shouldEqual "AndExpr(IsTrueExpr(FieldReference(Filter1)),EqualExpr(FieldReference(Filter2),3))" } } } + "Syntax.combinatoryConditionParentheses" should "be valid" in { - val parseResult = new Syntax("Filter1||(Filter2&&(Filter3<3||Filter4>0))").QueryRule.run() + val parseResult = new Syntax("[Filter1]||([Filter2]&&([Filter3]<3||[Filter4]>0))").QueryRule.run() parseResult shouldBe a [Success[_]] parseResult match { case Success(ast) => { - ast.toString shouldEqual "OrExpr(TrueExpr(Filter1),AndExpr(TrueExpr(Filter2)," + - "OrExpr(LessThanExpr(Filter3,3),GreaterThanExpr(Filter4,0))))" + ast.toString shouldEqual "OrExpr(IsTrueExpr(FieldReference(Filter1)),AndExpr(IsTrueExpr(FieldReference(Filter2))," + + "OrExpr(LessThanExpr(FieldReference(Filter3),3),GreaterThanExpr(FieldReference(Filter4),0))))" } } } + "Syntax.combinatoryConditionParenthesesComplex" should "be valid" in { - val parseResult = new Syntax("Filter1&&((Filter2<3||Filter2>0)%%(Filter4&&Filter5))").QueryRule.run() + val parseResult = new Syntax("[Filter1]&&(([Filter2]<3||[Filter2]>0)%%([Filter4]&&[Filter5]))").QueryRule.run() parseResult shouldBe a [Success[_]] parseResult match { case Success(ast) => { - ast.toString shouldEqual "AndExpr(TrueExpr(Filter1),XorExpr(OrExpr(LessThanExpr(Filter2,3)," + - "GreaterThanExpr(Filter2,0)),AndExpr(TrueExpr(Filter4),TrueExpr(Filter5))))" + ast.toString shouldEqual "AndExpr(IsTrueExpr(FieldReference(Filter1)),XorExpr(OrExpr(LessThanExpr(FieldReference(Filter2),3)," + + "GreaterThanExpr(FieldReference(Filter2),0)),AndExpr(IsTrueExpr(FieldReference(Filter4)),IsTrueExpr(FieldReference(Filter5)))))" } } } + "Syntax.combinatoryConditionNonMatchingParentheses" should "be valid" in { - val parseResult = new Syntax("Filter1&&(Filter2<3||Filter2>0%%(Filter4)").QueryRule.run() + val parseResult = new Syntax("[Filter1]&&([Filter2]<3||[Filter2]>0%%([Filter4])").QueryRule.run() parseResult shouldBe a [Failure[_]] } "Syntax.combinatoryConditionTypo" should "be valid" in { - val parseResult = new Syntax("Filter1&Filter2<3)").QueryRule.run() + val parseResult = new Syntax("[Filter1]&[Filter2]<3)").QueryRule.run() parseResult shouldBe a [Failure[_]] } "Syntax.combinatoryConditionLeftToRightPriority" should "be valid" in { - val parseResult = new Syntax("Filter1&&Filter2&&Filter3").QueryRule.run() + val parseResult = new Syntax("[Filter1]&&[Filter2]&&[Filter3]").QueryRule.run() parseResult shouldBe a [Success[_]] parseResult match { case Success(ast) => { - ast.toString shouldEqual "AndExpr(AndExpr(TrueExpr(Filter1)," + - "TrueExpr(Filter2)),TrueExpr(Filter3))" + ast.toString shouldEqual "AndExpr(AndExpr(IsTrueExpr(FieldReference(Filter1))," + + "IsTrueExpr(FieldReference(Filter2))),IsTrueExpr(FieldReference(Filter3)))" } } } "Syntax.combinatoryConditionOperatorPriorities" should "be valid" in { - val parseResult = new Syntax("Filter1||Filter2%%!Filter3&&Filter4").QueryRule.run() + val parseResult = new Syntax("[Filter1]||[Filter2]%%![Filter3]&&[Filter4]").QueryRule.run() parseResult shouldBe a [Success[_]] parseResult match { case Success(ast) => { - ast.toString shouldEqual "OrExpr(TrueExpr(Filter1),AndExpr(XorExpr(" + - "TrueExpr(Filter2),NotExpr(TrueExpr(Filter3))),TrueExpr(Filter4)))" + ast.toString shouldEqual "OrExpr(IsTrueExpr(FieldReference(Filter1)),AndExpr(XorExpr(" + + "IsTrueExpr(FieldReference(Filter2)),NotExpr(IsTrueExpr(FieldReference(Filter3)))),IsTrueExpr(FieldReference(Filter4))))" } } } + "Syntax.combinatoryConditionOperatorPrioritiesParentheses" should "be valid" in { - val parseResult = new Syntax("(Filter1||Filter2)&&!Filter3%%!(Filter4&&Filter5)").QueryRule.run() + val parseResult = new Syntax("([Filter1]||[Filter2])&&![Filter3]%%!([Filter4]&&[Filter5])").QueryRule.run() parseResult shouldBe a [Success[_]] parseResult match { case Success(ast) => { - ast.toString shouldEqual "AndExpr(OrExpr(TrueExpr(Filter1),TrueExpr(Filter2))," + - "XorExpr(NotExpr(TrueExpr(Filter3)),NotExpr(AndExpr(TrueExpr(Filter4)," + - "TrueExpr(Filter5)))))" + ast.toString shouldEqual "AndExpr(OrExpr(IsTrueExpr(FieldReference(Filter1)),IsTrueExpr(FieldReference(Filter2)))," + + "XorExpr(NotExpr(IsTrueExpr(FieldReference(Filter3))),NotExpr(AndExpr(IsTrueExpr(FieldReference(Filter4))," + + "IsTrueExpr(FieldReference(Filter5))))))" } } } + "Syntax.notConditionSimple" should "be valid" in { - val parseResult = new Syntax("!Filter1&&!(Filter2)").QueryRule.run() + val parseResult = new Syntax("![Filter1]&&!([Filter2])").QueryRule.run() parseResult shouldBe a [Success[_]] parseResult match { case Success(ast) => { - ast.toString shouldEqual "AndExpr(NotExpr(TrueExpr(Filter1))," + - "NotExpr(TrueExpr(Filter2)))" + ast.toString shouldEqual "AndExpr(NotExpr(IsTrueExpr(FieldReference(Filter1)))," + + "NotExpr(IsTrueExpr(FieldReference(Filter2))))" } } } "Syntax.notConditionSimpleParentheses" should "be valid" in { - val parseResult = new Syntax("!(Filter1&&!Filter2)").QueryRule.run() + val parseResult = new Syntax("!([Filter1]&&![Filter2])").QueryRule.run() parseResult shouldBe a [Success[_]] parseResult match { case Success(ast) => { - ast.toString shouldEqual "NotExpr(AndExpr(TrueExpr(Filter1)," + - "NotExpr(TrueExpr(Filter2))))" + ast.toString shouldEqual "NotExpr(AndExpr(IsTrueExpr(FieldReference(Filter1))," + + "NotExpr(IsTrueExpr(FieldReference(Filter2)))))" } } } "Syntax.notConditionComplex" should "be valid" in { - val parseResult = new Syntax("!!(Filter1)&&!(Filter2<=0||!(Filter3&&!Filter4%abc))").QueryRule.run() + val parseResult = new Syntax("!!([Filter1])&&!([Filter2]<=0||!([Filter3]&&![Filter4]%abc))").QueryRule.run() parseResult shouldBe a [Success[_]] parseResult match { case Success(ast) => { - ast.toString shouldEqual "AndExpr(NotExpr(NotExpr(TrueExpr(Filter1)))," + - "NotExpr(OrExpr(LessOrEqualExpr(Filter2,0),NotExpr(AndExpr(TrueExpr(Filter3)," + - "NotExpr(LikeExpr(Filter4,abc)))))))" + ast.toString shouldEqual "AndExpr(NotExpr(NotExpr(IsTrueExpr(FieldReference(Filter1))))," + + "NotExpr(OrExpr(LessOrEqualExpr(FieldReference(Filter2),0),NotExpr(AndExpr(IsTrueExpr(FieldReference(Filter3))," + + "NotExpr(LikeExpr(FieldReference(Filter4),abc)))))))" } } } From d27a248edc13ca7496c321582008467a4f3a3725 Mon Sep 17 00:00:00 2001 From: Ben Hermann Date: Sat, 27 Oct 2018 23:58:32 +0200 Subject: [PATCH 32/43] Gave sense to the test --- .../scala/de/upb/cs/swt/delphi/webapi/SearchQueryTest.scala | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/it/scala/de/upb/cs/swt/delphi/webapi/SearchQueryTest.scala b/src/it/scala/de/upb/cs/swt/delphi/webapi/SearchQueryTest.scala index 3370900..24203a1 100644 --- a/src/it/scala/de/upb/cs/swt/delphi/webapi/SearchQueryTest.scala +++ b/src/it/scala/de/upb/cs/swt/delphi/webapi/SearchQueryTest.scala @@ -21,12 +21,14 @@ import org.scalatest.{FlatSpec, Matchers} import de.upb.cs.swt.delphi.webapi.artifacts.ArtifactJson._ import spray.json._ +import scala.util.Success + class SearchQueryTest extends FlatSpec with Matchers { "Search query" should "check for fields" in { val configuration = new Configuration() val q = new SearchQuery(configuration, new FeatureExtractor(configuration)) val response = q.search("[if_icmpeq (opcode:159)]>1") - println(response.get.toJson) + response shouldBe a [Success[_]] } } From 3b1587a09f1a2ddc701ba5c13e4f966559ac66f8 Mon Sep 17 00:00:00 2001 From: Ben Hermann Date: Sat, 27 Oct 2018 23:59:34 +0200 Subject: [PATCH 33/43] Organized imports --- src/it/scala/de/upb/cs/swt/delphi/webapi/SearchQueryTest.scala | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/it/scala/de/upb/cs/swt/delphi/webapi/SearchQueryTest.scala b/src/it/scala/de/upb/cs/swt/delphi/webapi/SearchQueryTest.scala index 24203a1..70d1e7a 100644 --- a/src/it/scala/de/upb/cs/swt/delphi/webapi/SearchQueryTest.scala +++ b/src/it/scala/de/upb/cs/swt/delphi/webapi/SearchQueryTest.scala @@ -18,8 +18,6 @@ package de.upb.cs.swt.delphi.webapi import de.upb.cs.swt.delphi.featuredefinitions.FeatureExtractor import org.scalatest.{FlatSpec, Matchers} -import de.upb.cs.swt.delphi.webapi.artifacts.ArtifactJson._ -import spray.json._ import scala.util.Success From 7b26a4f66e932f279517e5082e6a2e8261b3fef2 Mon Sep 17 00:00:00 2001 From: Ben Hermann Date: Sun, 28 Oct 2018 16:29:44 +0100 Subject: [PATCH 34/43] Added `-` to syntax as a valid character in field references --- src/main/scala/de/upb/cs/swt/delphi/querylanguage/Syntax.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/scala/de/upb/cs/swt/delphi/querylanguage/Syntax.scala b/src/main/scala/de/upb/cs/swt/delphi/querylanguage/Syntax.scala index 69d9a31..3a78ebc 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/querylanguage/Syntax.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/querylanguage/Syntax.scala @@ -67,7 +67,7 @@ class Syntax(val input : ParserInput) extends Parser { def IsTrue = rule { FieldReferenceRule ~> IsTrueExpr } // Literals - def FieldReferenceRule = rule { "[" ~ capture(oneOrMore(CharPredicate.AlphaNum ++ ' ' ++ '_' ++ '(' ++ ':' ++')')) ~ "]" ~> FieldReference } + def FieldReferenceRule = rule { "[" ~ capture(oneOrMore(CharPredicate.AlphaNum ++ '-' ++ ' ' ++ '_' ++ '(' ++ ':' ++')')) ~ "]" ~> FieldReference } def Literal = rule { capture(oneOrMore(CharPredicate.AlphaNum)) ~> (_.toString) } } From 40ed0fa493f25ca4e2b8cda8d9c6fe23f622a28f Mon Sep 17 00:00:00 2001 From: Ben Hermann Date: Tue, 30 Oct 2018 10:22:03 +0100 Subject: [PATCH 35/43] Refactorings --- .../delphi/webapi/FeatureExtractorCheck.scala | 2 +- .../swt/delphi/webapi/SearchQueryTest.scala | 2 +- .../cs/swt/delphi/webapi/Configuration.scala | 38 +++++++++---------- .../cs/swt/delphi/webapi/SearchQuery.scala | 10 +++-- .../de/upb/cs/swt/delphi/webapi/Server.scala | 2 +- .../FeatureDescription.scala | 2 +- .../featuredefinitions/FeatureExtractor.scala | 2 +- .../FeatureListMapping.scala | 2 +- .../{ => webapi}/querylanguage/AST.scala | 2 +- .../{ => webapi}/querylanguage/Syntax.scala | 2 +- .../querylanguage/SyntaxTest.scala | 2 +- 11 files changed, 35 insertions(+), 31 deletions(-) rename src/main/scala/de/upb/cs/swt/delphi/{ => webapi}/featuredefinitions/FeatureDescription.scala (93%) rename src/main/scala/de/upb/cs/swt/delphi/{ => webapi}/featuredefinitions/FeatureExtractor.scala (97%) rename src/main/scala/de/upb/cs/swt/delphi/{ => webapi}/featuredefinitions/FeatureListMapping.scala (99%) rename src/main/scala/de/upb/cs/swt/delphi/{ => webapi}/querylanguage/AST.scala (97%) rename src/main/scala/de/upb/cs/swt/delphi/{ => webapi}/querylanguage/Syntax.scala (98%) rename src/test/scala/de/upb/cs/swt/delphi/{ => webapi}/querylanguage/SyntaxTest.scala (99%) diff --git a/src/it/scala/de/upb/cs/swt/delphi/webapi/FeatureExtractorCheck.scala b/src/it/scala/de/upb/cs/swt/delphi/webapi/FeatureExtractorCheck.scala index b75cddf..0d2ddad 100644 --- a/src/it/scala/de/upb/cs/swt/delphi/webapi/FeatureExtractorCheck.scala +++ b/src/it/scala/de/upb/cs/swt/delphi/webapi/FeatureExtractorCheck.scala @@ -16,7 +16,7 @@ package de.upb.cs.swt.delphi.webapi -import de.upb.cs.swt.delphi.featuredefinitions.FeatureExtractor +import de.upb.cs.swt.delphi.webapi.featuredefinitions.FeatureExtractor import org.scalatest.{FlatSpec, Matchers} class FeatureExtractorCheck extends FlatSpec with Matchers { diff --git a/src/it/scala/de/upb/cs/swt/delphi/webapi/SearchQueryTest.scala b/src/it/scala/de/upb/cs/swt/delphi/webapi/SearchQueryTest.scala index 70d1e7a..066ceab 100644 --- a/src/it/scala/de/upb/cs/swt/delphi/webapi/SearchQueryTest.scala +++ b/src/it/scala/de/upb/cs/swt/delphi/webapi/SearchQueryTest.scala @@ -16,7 +16,7 @@ package de.upb.cs.swt.delphi.webapi -import de.upb.cs.swt.delphi.featuredefinitions.FeatureExtractor +import de.upb.cs.swt.delphi.webapi.featuredefinitions.FeatureExtractor import org.scalatest.{FlatSpec, Matchers} import scala.util.Success diff --git a/src/main/scala/de/upb/cs/swt/delphi/webapi/Configuration.scala b/src/main/scala/de/upb/cs/swt/delphi/webapi/Configuration.scala index d3d81e9..3874191 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/webapi/Configuration.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/webapi/Configuration.scala @@ -26,21 +26,21 @@ import scala.util.{Failure, Success, Try} /** * @author Ben Hermann */ -class Configuration( //Server and Elasticsearch configuration - val bindHost: String = "0.0.0.0", - val bindPort: Int = 8080, - val esIndex : String = "delphi", - val esType : String = "project", +class Configuration( //Server and Elasticsearch configuration + val bindHost: String = "0.0.0.0", + val bindPort: Int = 8080, + val esIndex: String = "delphi", + val esType: String = "project", - //Actor system configuration - val elasticActorPoolSize: Int = 8 + //Actor system configuration + val elasticActorPoolSize: Int = 8 ) { lazy val esProjectIndex: IndexAndType = esIndex / esType lazy val elasticsearchClientUri: ElasticsearchClientUri = ElasticsearchClientUri( elasticsearchInstance.host + ":" + elasticsearchInstance.portNumber) - lazy val elasticsearchInstance : Instance = InstanceRegistry.retrieveElasticSearchInstance( configuration = this) match { + lazy val elasticsearchInstance: Instance = InstanceRegistry.retrieveElasticSearchInstance(configuration = this) match { case Success(instance) => instance case Failure(_) => Instance( None, @@ -51,18 +51,18 @@ class Configuration( //Server and Elasticsearch configuration None, InstanceState.Running) } - val defaultElasticSearchPort : Int = 9200 - val defaultElasticSearchHost : String = "elasticsearch://localhost" + val defaultElasticSearchPort: Int = 9200 + val defaultElasticSearchHost: String = "elasticsearch://localhost" val instanceName = "MyWebApiInstance" - val instanceRegistryUri : String = sys.env.getOrElse("DELPHI_IR_URI", "http://localhost:8087") - lazy val usingInstanceRegistry : Boolean = assignedID match { + val instanceRegistryUri: String = sys.env.getOrElse("DELPHI_IR_URI", "http://localhost:8087") + lazy val usingInstanceRegistry: Boolean = assignedID match { case Some(_) => true case None => false } - lazy val assignedID : Option[Long] = InstanceRegistry.handleInstanceStart(configuration = this) + lazy val assignedID: Option[Long] = InstanceRegistry.handleInstanceStart(configuration = this) - lazy val fallbackElasticSearchPort : Int = sys.env.get("DELPHI_ELASTIC_URI") match { - case Some(hostString) => if(hostString.count(c => c == ':') == 3){ + lazy val fallbackElasticSearchPort: Int = sys.env.get("DELPHI_ELASTIC_URI") match { + case Some(hostString) => if (hostString.count(c => c == ':') == 3) { Try(hostString.split(":")(2).toInt) match { case Success(port) => port case Failure(_) => defaultElasticSearchPort @@ -73,17 +73,17 @@ class Configuration( //Server and Elasticsearch configuration case None => defaultElasticSearchPort } - lazy val fallbackElasticSearchHost : String = sys.env.get("DELPHI_ELASTIC_URI") match { + lazy val fallbackElasticSearchHost: String = sys.env.get("DELPHI_ELASTIC_URI") match { case Some(hostString) => - if(hostString.count(c => c == ':') == 2){ - hostString.substring(0,hostString.lastIndexOf(":")) + if (hostString.count(c => c == ':') == 2) { + hostString.substring(0, hostString.lastIndexOf(":")) } else { defaultElasticSearchHost } case None => defaultElasticSearchHost } - lazy val instanceId : Option[Long] = InstanceRegistry.handleInstanceStart(configuration = this) + lazy val instanceId: Option[Long] = InstanceRegistry.handleInstanceStart(configuration = this) } diff --git a/src/main/scala/de/upb/cs/swt/delphi/webapi/SearchQuery.scala b/src/main/scala/de/upb/cs/swt/delphi/webapi/SearchQuery.scala index b32c7dd..9990934 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/webapi/SearchQuery.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/webapi/SearchQuery.scala @@ -17,8 +17,8 @@ package de.upb.cs.swt.delphi.webapi import com.sksamuel.elastic4s.http.{ElasticClient, RequestSuccess} -import de.upb.cs.swt.delphi.featuredefinitions.FeatureExtractor -import de.upb.cs.swt.delphi.querylanguage._ +import de.upb.cs.swt.delphi.webapi.featuredefinitions.FeatureExtractor +import de.upb.cs.swt.delphi.webapi.querylanguage._ import com.sksamuel.elastic4s.http.ElasticDsl._ import com.sksamuel.elastic4s.http.search.SearchHits import com.sksamuel.elastic4s.searches.queries.{NoopQuery, Query} @@ -33,7 +33,11 @@ class SearchQuery(configuration: Configuration, featureExtractor: FeatureExtract val fields = collectFieldNames(ast) if (fields.diff(featureExtractor.featureList.toSeq).size > 0) return Failure(null) - val query = searchWithType(configuration.esProjectIndex) query translate(ast) sourceInclude (ArtifactTransformer.baseFields ++ fields.intersect(featureExtractor.featureList.toSeq).map(i => addPrefix(i))) + val query = searchWithType(configuration.esProjectIndex) + .query(translate(ast)) + .sourceInclude(ArtifactTransformer.baseFields ++ fields.intersect(featureExtractor.featureList.toSeq).map(i => addPrefix(i))) + + val response = client.execute { query diff --git a/src/main/scala/de/upb/cs/swt/delphi/webapi/Server.scala b/src/main/scala/de/upb/cs/swt/delphi/webapi/Server.scala index 9f9fa22..71b555a 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/webapi/Server.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/webapi/Server.scala @@ -23,7 +23,7 @@ import akka.http.scaladsl.model.{HttpResponse, StatusCodes} import akka.http.scaladsl.server.{HttpApp, Route} import akka.stream.ActorMaterializer import akka.util.Timeout -import de.upb.cs.swt.delphi.featuredefinitions.{FeatureExtractor, FeatureListMapping} +import de.upb.cs.swt.delphi.webapi.featuredefinitions.{FeatureExtractor, FeatureListMapping} import de.upb.cs.swt.delphi.instancemanagement.InstanceRegistry import spray.json._ import de.upb.cs.swt.delphi.webapi.artifacts.ArtifactJson._ diff --git a/src/main/scala/de/upb/cs/swt/delphi/featuredefinitions/FeatureDescription.scala b/src/main/scala/de/upb/cs/swt/delphi/webapi/featuredefinitions/FeatureDescription.scala similarity index 93% rename from src/main/scala/de/upb/cs/swt/delphi/featuredefinitions/FeatureDescription.scala rename to src/main/scala/de/upb/cs/swt/delphi/webapi/featuredefinitions/FeatureDescription.scala index 8e7dbe4..6854919 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/featuredefinitions/FeatureDescription.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/webapi/featuredefinitions/FeatureDescription.scala @@ -14,7 +14,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package de.upb.cs.swt.delphi.featuredefinitions +package de.upb.cs.swt.delphi.webapi.featuredefinitions //Describes all relevant fields for conducting searches on this class case class FeatureDescription(fType: String) diff --git a/src/main/scala/de/upb/cs/swt/delphi/featuredefinitions/FeatureExtractor.scala b/src/main/scala/de/upb/cs/swt/delphi/webapi/featuredefinitions/FeatureExtractor.scala similarity index 97% rename from src/main/scala/de/upb/cs/swt/delphi/featuredefinitions/FeatureExtractor.scala rename to src/main/scala/de/upb/cs/swt/delphi/webapi/featuredefinitions/FeatureExtractor.scala index a79ef0f..543e84d 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/featuredefinitions/FeatureExtractor.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/webapi/featuredefinitions/FeatureExtractor.scala @@ -14,7 +14,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package de.upb.cs.swt.delphi.featuredefinitions +package de.upb.cs.swt.delphi.webapi.featuredefinitions import com.sksamuel.elastic4s.http.{ElasticClient, RequestSuccess} import com.sksamuel.elastic4s.http.ElasticDsl._ diff --git a/src/main/scala/de/upb/cs/swt/delphi/featuredefinitions/FeatureListMapping.scala b/src/main/scala/de/upb/cs/swt/delphi/webapi/featuredefinitions/FeatureListMapping.scala similarity index 99% rename from src/main/scala/de/upb/cs/swt/delphi/featuredefinitions/FeatureListMapping.scala rename to src/main/scala/de/upb/cs/swt/delphi/webapi/featuredefinitions/FeatureListMapping.scala index 9cd13e4..bee0dc8 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/featuredefinitions/FeatureListMapping.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/webapi/featuredefinitions/FeatureListMapping.scala @@ -14,7 +14,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package de.upb.cs.swt.delphi.featuredefinitions +package de.upb.cs.swt.delphi.webapi.featuredefinitions object FeatureListMapping { diff --git a/src/main/scala/de/upb/cs/swt/delphi/querylanguage/AST.scala b/src/main/scala/de/upb/cs/swt/delphi/webapi/querylanguage/AST.scala similarity index 97% rename from src/main/scala/de/upb/cs/swt/delphi/querylanguage/AST.scala rename to src/main/scala/de/upb/cs/swt/delphi/webapi/querylanguage/AST.scala index 64d0d09..4dcd583 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/querylanguage/AST.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/webapi/querylanguage/AST.scala @@ -14,7 +14,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package de.upb.cs.swt.delphi.querylanguage +package de.upb.cs.swt.delphi.webapi.querylanguage trait CombinatorialExpr diff --git a/src/main/scala/de/upb/cs/swt/delphi/querylanguage/Syntax.scala b/src/main/scala/de/upb/cs/swt/delphi/webapi/querylanguage/Syntax.scala similarity index 98% rename from src/main/scala/de/upb/cs/swt/delphi/querylanguage/Syntax.scala rename to src/main/scala/de/upb/cs/swt/delphi/webapi/querylanguage/Syntax.scala index 3a78ebc..7b78b4f 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/querylanguage/Syntax.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/webapi/querylanguage/Syntax.scala @@ -14,7 +14,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package de.upb.cs.swt.delphi.querylanguage +package de.upb.cs.swt.delphi.webapi.querylanguage import org.parboiled2.{CharPredicate, Parser, ParserInput, Rule1} diff --git a/src/test/scala/de/upb/cs/swt/delphi/querylanguage/SyntaxTest.scala b/src/test/scala/de/upb/cs/swt/delphi/webapi/querylanguage/SyntaxTest.scala similarity index 99% rename from src/test/scala/de/upb/cs/swt/delphi/querylanguage/SyntaxTest.scala rename to src/test/scala/de/upb/cs/swt/delphi/webapi/querylanguage/SyntaxTest.scala index 179c75a..88de92e 100644 --- a/src/test/scala/de/upb/cs/swt/delphi/querylanguage/SyntaxTest.scala +++ b/src/test/scala/de/upb/cs/swt/delphi/webapi/querylanguage/SyntaxTest.scala @@ -14,7 +14,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package de.upb.cs.swt.delphi.querylanguage +package de.upb.cs.swt.delphi.webapi.querylanguage import org.scalatest.{FlatSpec, Matchers} import scala.util.{Failure, Success} From 78b3a2d5ee08f3224177ac78dbca5771235cac2d Mon Sep 17 00:00:00 2001 From: Ben Hermann Date: Tue, 30 Oct 2018 12:43:39 +0100 Subject: [PATCH 36/43] Refactorings --- src/main/resources/application.conf | 19 +--- .../cs/swt/delphi/webapi/ElasticActor.scala | 72 -------------- .../delphi/webapi/ElasticActorManager.scala | 60 ------------ .../webapi/ElasticPriorityMailbox.scala | 30 ------ .../delphi/webapi/ElasticRequestLimiter.scala | 97 ------------------- .../swt/delphi/webapi/MavenIdentifier.scala | 2 + .../webapi/{ => artifacts}/Identifier.scala | 2 +- .../webapi/{ => search}/SearchQuery.scala | 0 .../{ => search}/querylanguage/AST.scala | 2 +- .../{ => search}/querylanguage/Syntax.scala | 2 +- .../querylanguage/SyntaxTest.scala | 2 +- 11 files changed, 7 insertions(+), 281 deletions(-) delete mode 100644 src/main/scala/de/upb/cs/swt/delphi/webapi/ElasticActor.scala delete mode 100644 src/main/scala/de/upb/cs/swt/delphi/webapi/ElasticActorManager.scala delete mode 100644 src/main/scala/de/upb/cs/swt/delphi/webapi/ElasticPriorityMailbox.scala delete mode 100644 src/main/scala/de/upb/cs/swt/delphi/webapi/ElasticRequestLimiter.scala rename src/main/scala/de/upb/cs/swt/delphi/webapi/{ => artifacts}/Identifier.scala (94%) rename src/main/scala/de/upb/cs/swt/delphi/webapi/{ => search}/SearchQuery.scala (100%) rename src/main/scala/de/upb/cs/swt/delphi/webapi/{ => search}/querylanguage/AST.scala (97%) rename src/main/scala/de/upb/cs/swt/delphi/webapi/{ => search}/querylanguage/Syntax.scala (97%) rename src/test/scala/de/upb/cs/swt/delphi/webapi/{ => search}/querylanguage/SyntaxTest.scala (99%) diff --git a/src/main/resources/application.conf b/src/main/resources/application.conf index 935d3b3..2840c06 100644 --- a/src/main/resources/application.conf +++ b/src/main/resources/application.conf @@ -1,11 +1,5 @@ -es-priority-mailbox { - mailbox-type = "de.upb.cs.swt.delphi.webapi.ElasticPriorityMailbox" -} - akka.actor.deployment { - /espriomailboxactor { - mailbox = es-priority-mailbox - } + } akka { @@ -15,14 +9,3 @@ akka { } } } - -# Use this dispatcher for actors that make blocking calls to the Elasticsearch database -elasticsearch-handling-dispatcher { - type = Dispatcher - executor = "thread-pool-executor" - thread-pool-executor { - fixed-pool-size = 4 - # This thread pool is intended for development purposes, and should be increased for production - } - throughput = 1 -} \ No newline at end of file diff --git a/src/main/scala/de/upb/cs/swt/delphi/webapi/ElasticActor.scala b/src/main/scala/de/upb/cs/swt/delphi/webapi/ElasticActor.scala deleted file mode 100644 index 131a151..0000000 --- a/src/main/scala/de/upb/cs/swt/delphi/webapi/ElasticActor.scala +++ /dev/null @@ -1,72 +0,0 @@ -// Copyright (C) 2018 The Delphi Team. -// See the LICENCE file distributed with this work for additional -// information regarding copyright ownership. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package de.upb.cs.swt.delphi.webapi -import akka.actor.{Actor, ActorLogging, Props} -import com.sksamuel.elastic4s.IndexAndType -import com.sksamuel.elastic4s.http.{ElasticClient, RequestFailure, RequestSuccess} -import com.sksamuel.elastic4s.http.ElasticDsl._ -import de.upb.cs.swt.delphi.webapi.ElasticActorManager.{Enqueue, Retrieve} - -import spray.json._ -import scala.concurrent.ExecutionContext -import scala.concurrent.duration._ - -class ElasticActor(configuration: Configuration, index: IndexAndType) extends Actor with ActorLogging { - - implicit val executionContext: ExecutionContext = context.system.dispatchers.lookup("elasticsearch-handling-dispatcher") - val client = ElasticClient(configuration.elasticsearchClientUri) - - override def preStart(): Unit = log.info("Search actor started") - - override def postStop(): Unit = log.info("Search actor shut down") - - context.setReceiveTimeout(2 seconds) - - override def receive = { - case Enqueue(id) => getSource(id) - case Retrieve(id) => getSource(id) - } - - private def getSource(id: String) = { - log.info("Executing get on entry {}", id) - val searchByName = searchWithType(index) query must( - matchQuery("name", id) - ) - log.info(s"Query {}",client.show(searchByName)) - def queryResponse = client.execute { - log.info(s"Got retrieve request for $id.") - searchByName - }.await - - val source = queryResponse match { - case results: RequestSuccess[_] => { - val resObj = results.body.get.parseJson.asJsObject - val hitsObj=resObj.fields.getOrElse("hits", JsObject.empty).asJsObject - val hitsArr=hitsObj.fields.getOrElse("hits",JsArray.empty).asInstanceOf[JsArray] - val source=hitsArr.elements.map(m=>m.asJsObject.fields.get("_source")) - source.head.getOrElse(JsObject.empty).toString() - } - case failure: RequestFailure => Option.empty - } - sender().tell(source, context.self) - } -} - -object ElasticActor { - def props(configuration: Configuration, index: IndexAndType): Props = Props(new ElasticActor(configuration, index)) - .withMailbox("es-priority-mailbox") -} diff --git a/src/main/scala/de/upb/cs/swt/delphi/webapi/ElasticActorManager.scala b/src/main/scala/de/upb/cs/swt/delphi/webapi/ElasticActorManager.scala deleted file mode 100644 index d8530fc..0000000 --- a/src/main/scala/de/upb/cs/swt/delphi/webapi/ElasticActorManager.scala +++ /dev/null @@ -1,60 +0,0 @@ -// Copyright (C) 2018 The Delphi Team. -// See the LICENCE file distributed with this work for additional -// information regarding copyright ownership. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package de.upb.cs.swt.delphi.webapi - -import akka.actor.{Actor, ActorLogging, Props, Terminated} -import akka.routing.{ActorRefRoutee, RoundRobinRoutingLogic, Router} -import de.upb.cs.swt.delphi.webapi.ElasticActorManager.ElasticMessage - -class ElasticActorManager(configuration: Configuration) extends Actor with ActorLogging{ - - private val index = configuration.esProjectIndex - private var elasticRouter = { - val routees = Vector.fill(configuration.elasticActorPoolSize) { - val r = context.actorOf(ElasticActor.props(configuration, index)) - context watch r - ActorRefRoutee(r) - } - Router(RoundRobinRoutingLogic(), routees) - } - - override def preStart(): Unit = log.info("Actor manager started") - override def postStop(): Unit = log.info("Actor manager shut down") - - override def receive = { - case em: ElasticMessage => { - log.info("Forwarding request {} to ElasticActor", em) - elasticRouter.route(em, sender()) - } - case Terminated(id) => { - elasticRouter.removeRoutee(id) - val r = context.actorOf(ElasticActor.props(configuration, index)) - context watch r - elasticRouter = elasticRouter.addRoutee(r) - } - } -} - -object ElasticActorManager{ - def props(configuration: Configuration) : Props = Props(new ElasticActorManager(configuration)) - .withMailbox("es-priority-mailbox") - - sealed trait ElasticMessage - - final case class Retrieve(id: String) extends ElasticMessage - final case class Enqueue(id: String) extends ElasticMessage -} \ No newline at end of file diff --git a/src/main/scala/de/upb/cs/swt/delphi/webapi/ElasticPriorityMailbox.scala b/src/main/scala/de/upb/cs/swt/delphi/webapi/ElasticPriorityMailbox.scala deleted file mode 100644 index fc48390..0000000 --- a/src/main/scala/de/upb/cs/swt/delphi/webapi/ElasticPriorityMailbox.scala +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright (C) 2018 The Delphi Team. -// See the LICENCE file distributed with this work for additional -// information regarding copyright ownership. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package de.upb.cs.swt.delphi.webapi - -import akka.actor.ActorSystem -import akka.dispatch.{PriorityGenerator, UnboundedStablePriorityMailbox} -import de.upb.cs.swt.delphi.webapi.ElasticActorManager.{Enqueue, Retrieve} -import com.typesafe.config.Config - -class ElasticPriorityMailbox (settings: ActorSystem.Settings, config: Config) - extends UnboundedStablePriorityMailbox( - PriorityGenerator{ - case Retrieve(_) => 5 - case Enqueue(_) => 1 - case _ => 2 - }) diff --git a/src/main/scala/de/upb/cs/swt/delphi/webapi/ElasticRequestLimiter.scala b/src/main/scala/de/upb/cs/swt/delphi/webapi/ElasticRequestLimiter.scala deleted file mode 100644 index f131652..0000000 --- a/src/main/scala/de/upb/cs/swt/delphi/webapi/ElasticRequestLimiter.scala +++ /dev/null @@ -1,97 +0,0 @@ -// Copyright (C) 2018 The Delphi Team. -// See the LICENCE file distributed with this work for additional -// information regarding copyright ownership. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package de.upb.cs.swt.delphi.webapi - - -import akka.actor.{Actor, ActorLogging, ActorRef, Props, Timers} -import akka.http.scaladsl.model.RemoteAddress -import de.upb.cs.swt.delphi.webapi.ElasticActorManager.ElasticMessage -import de.upb.cs.swt.delphi.webapi.ElasticRequestLimiter._ - -import scala.collection.mutable -import scala.concurrent.duration._ - -//Limits the number of requests any given IP can make by tracking how many requests an IP has made within a given -// window of time, and timing out any IP that exceeds a threshold by rejecting any further request for a period of time -class ElasticRequestLimiter(configuration: Configuration, nextActor: ActorRef) extends Actor with ActorLogging with Timers { - - private val window = 1 second - private val threshold = 10 - private val timeout = 2 hours - - private var recentIPs: mutable.Map[String, Int] = mutable.Map() - private var blockedIPs: mutable.Set[String] = mutable.Set() - - override def preStart(): Unit = { - log.info("Request limiter started") - timers.startPeriodicTimer(ClearTimer, ClearLogs, window) - } - override def postStop(): Unit = log.info("Request limiter shut down") - - override def receive = { - case Validate(rawIp, message) => { - val ip = rawIp.toOption.map(_.getHostAddress).getOrElse("unknown") - //First, reject IPs marked as blocked - if (blockedIPs.contains(ip)) { - rejectRequest() - } else { - //Check if this IP has made any requests recently - if (recentIPs.contains(ip)) { - //If so, increment their counter and test if they have exceeded the request threshold - recentIPs.update(ip, recentIPs(ip) + 1) - if (recentIPs(ip) > threshold) { - //If the threshold has been exceeded, mark this IP as blocked and reject it, and set up a message to unblock it after a period - blockedIPs += ip - log.info("Blocked IP {} due to exceeding request frequency threshold", ip) - timers.startSingleTimer(ForgiveTimer(ip), Forgive(ip), timeout) - rejectRequest() - } else { - //Else, forward this message - nextActor forward message - } - } else { - //Else, register their request in the map and pass it to the next actor - recentIPs += (ip -> 1) - nextActor forward message - } - } - } - case ClearLogs => - recentIPs.clear() - case Forgive(ip) => { - blockedIPs -= ip - log.info("Forgave IP {} after timeout", ip) - } - } - - //Rejects requests from blocked IPs - private def rejectRequest() = - sender() ! "Sorry, you have exceeded the limit on request frequency for unregistered users.\n" + - "As a result, you have been timed out.\n" + - "Please wait a while or register an account with us to continue using this service." -} - -object ElasticRequestLimiter{ - def props(configuration: Configuration, nextActor: ActorRef) : Props = Props(new ElasticRequestLimiter(configuration, nextActor)) - - final case class Validate(rawIp: RemoteAddress, message: ElasticMessage) - final case object ClearLogs - final case class Forgive(ip: String) - - final case object ClearTimer - final case class ForgiveTimer(ip: String) -} \ No newline at end of file diff --git a/src/main/scala/de/upb/cs/swt/delphi/webapi/MavenIdentifier.scala b/src/main/scala/de/upb/cs/swt/delphi/webapi/MavenIdentifier.scala index 71a7756..3d88347 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/webapi/MavenIdentifier.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/webapi/MavenIdentifier.scala @@ -19,6 +19,8 @@ package de.upb.cs.swt.delphi.webapi import java.net.{URI, URLEncoder} import java.nio.charset.StandardCharsets +import de.upb.cs.swt.delphi.webapi.artifacts.Identifier + case class MavenIdentifier(val repository: Option[String], val groupId: String, val artifactId: String, val version: Option[String]) extends Identifier { def toUniqueString = { diff --git a/src/main/scala/de/upb/cs/swt/delphi/webapi/Identifier.scala b/src/main/scala/de/upb/cs/swt/delphi/webapi/artifacts/Identifier.scala similarity index 94% rename from src/main/scala/de/upb/cs/swt/delphi/webapi/Identifier.scala rename to src/main/scala/de/upb/cs/swt/delphi/webapi/artifacts/Identifier.scala index 3dcc0f9..ed419b3 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/webapi/Identifier.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/webapi/artifacts/Identifier.scala @@ -14,7 +14,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package de.upb.cs.swt.delphi.webapi +package de.upb.cs.swt.delphi.webapi.artifacts /** * Represents an identifier for a software artifact diff --git a/src/main/scala/de/upb/cs/swt/delphi/webapi/SearchQuery.scala b/src/main/scala/de/upb/cs/swt/delphi/webapi/search/SearchQuery.scala similarity index 100% rename from src/main/scala/de/upb/cs/swt/delphi/webapi/SearchQuery.scala rename to src/main/scala/de/upb/cs/swt/delphi/webapi/search/SearchQuery.scala diff --git a/src/main/scala/de/upb/cs/swt/delphi/webapi/querylanguage/AST.scala b/src/main/scala/de/upb/cs/swt/delphi/webapi/search/querylanguage/AST.scala similarity index 97% rename from src/main/scala/de/upb/cs/swt/delphi/webapi/querylanguage/AST.scala rename to src/main/scala/de/upb/cs/swt/delphi/webapi/search/querylanguage/AST.scala index 4dcd583..ccece70 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/webapi/querylanguage/AST.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/webapi/search/querylanguage/AST.scala @@ -14,7 +14,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package de.upb.cs.swt.delphi.webapi.querylanguage +package de.upb.cs.swt.delphi.webapi.search.querylanguage trait CombinatorialExpr diff --git a/src/main/scala/de/upb/cs/swt/delphi/webapi/querylanguage/Syntax.scala b/src/main/scala/de/upb/cs/swt/delphi/webapi/search/querylanguage/Syntax.scala similarity index 97% rename from src/main/scala/de/upb/cs/swt/delphi/webapi/querylanguage/Syntax.scala rename to src/main/scala/de/upb/cs/swt/delphi/webapi/search/querylanguage/Syntax.scala index 7b78b4f..2df2cb4 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/webapi/querylanguage/Syntax.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/webapi/search/querylanguage/Syntax.scala @@ -14,7 +14,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package de.upb.cs.swt.delphi.webapi.querylanguage +package de.upb.cs.swt.delphi.webapi.search.querylanguage import org.parboiled2.{CharPredicate, Parser, ParserInput, Rule1} diff --git a/src/test/scala/de/upb/cs/swt/delphi/webapi/querylanguage/SyntaxTest.scala b/src/test/scala/de/upb/cs/swt/delphi/webapi/search/querylanguage/SyntaxTest.scala similarity index 99% rename from src/test/scala/de/upb/cs/swt/delphi/webapi/querylanguage/SyntaxTest.scala rename to src/test/scala/de/upb/cs/swt/delphi/webapi/search/querylanguage/SyntaxTest.scala index 88de92e..94a79d0 100644 --- a/src/test/scala/de/upb/cs/swt/delphi/webapi/querylanguage/SyntaxTest.scala +++ b/src/test/scala/de/upb/cs/swt/delphi/webapi/search/querylanguage/SyntaxTest.scala @@ -14,7 +14,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package de.upb.cs.swt.delphi.webapi.querylanguage +package de.upb.cs.swt.delphi.webapi.search.querylanguage import org.scalatest.{FlatSpec, Matchers} import scala.util.{Failure, Success} From cb1bb231d8e4deb9d78a57a30c5623d7afa2fc91 Mon Sep 17 00:00:00 2001 From: Ben Hermann Date: Tue, 30 Oct 2018 12:44:20 +0100 Subject: [PATCH 37/43] Better error messages --- src/main/scala/de/upb/cs/swt/delphi/webapi/Server.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/scala/de/upb/cs/swt/delphi/webapi/Server.scala b/src/main/scala/de/upb/cs/swt/delphi/webapi/Server.scala index 71b555a..47aab61 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/webapi/Server.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/webapi/Server.scala @@ -117,7 +117,7 @@ object Server extends HttpApp with JsonSupport with AppLogging { complete( new SearchQuery(configuration, featureExtractor).search(query) match { case Success(result) => prettyPrint(pretty, result.toJson) - case Failure(e) => e.toString + case Failure(e) => e.getMessage } ) } From f0408a37f3737728f32a8c81f5ec4582fe219ca9 Mon Sep 17 00:00:00 2001 From: Ben Hermann Date: Tue, 30 Oct 2018 12:44:54 +0100 Subject: [PATCH 38/43] Changed query interface to accomodate for new parameters --- .../swt/delphi/webapi/SearchQueryTest.scala | 3 ++- .../cs/swt/delphi/webapi/RetrieveQuery.scala | 9 +++----- .../de/upb/cs/swt/delphi/webapi/Server.scala | 15 +++++++------ .../delphi/webapi/search/QueryRequest.scala | 3 +++ .../webapi/search/QueryRequestJson.scala | 7 ++++++ .../delphi/webapi/search/SearchQuery.scala | 22 +++++++++---------- 6 files changed, 34 insertions(+), 25 deletions(-) create mode 100644 src/main/scala/de/upb/cs/swt/delphi/webapi/search/QueryRequest.scala create mode 100644 src/main/scala/de/upb/cs/swt/delphi/webapi/search/QueryRequestJson.scala diff --git a/src/it/scala/de/upb/cs/swt/delphi/webapi/SearchQueryTest.scala b/src/it/scala/de/upb/cs/swt/delphi/webapi/SearchQueryTest.scala index 066ceab..61b194b 100644 --- a/src/it/scala/de/upb/cs/swt/delphi/webapi/SearchQueryTest.scala +++ b/src/it/scala/de/upb/cs/swt/delphi/webapi/SearchQueryTest.scala @@ -17,6 +17,7 @@ package de.upb.cs.swt.delphi.webapi import de.upb.cs.swt.delphi.webapi.featuredefinitions.FeatureExtractor +import de.upb.cs.swt.delphi.webapi.search.{QueryRequest, SearchQuery} import org.scalatest.{FlatSpec, Matchers} import scala.util.Success @@ -26,7 +27,7 @@ class SearchQueryTest extends FlatSpec with Matchers { val configuration = new Configuration() val q = new SearchQuery(configuration, new FeatureExtractor(configuration)) - val response = q.search("[if_icmpeq (opcode:159)]>1") + val response = q.search(QueryRequest("[if_icmpeq (opcode:159)]>1")) response shouldBe a [Success[_]] } } diff --git a/src/main/scala/de/upb/cs/swt/delphi/webapi/RetrieveQuery.scala b/src/main/scala/de/upb/cs/swt/delphi/webapi/RetrieveQuery.scala index f059727..dd149ed 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/webapi/RetrieveQuery.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/webapi/RetrieveQuery.scala @@ -16,15 +16,12 @@ package de.upb.cs.swt.delphi.webapi -import com.sksamuel.elastic4s.http.{ElasticClient, RequestSuccess, Response} import com.sksamuel.elastic4s.http.ElasticDsl.{termQuery, _} -import com.sksamuel.elastic4s.http.search.{SearchHit, SearchHits, SearchResponse} +import com.sksamuel.elastic4s.http.search.SearchResponse +import com.sksamuel.elastic4s.http.{ElasticClient, RequestSuccess, Response} import com.sksamuel.elastic4s.searches.queries.Query import com.sksamuel.elastic4s.searches.queries.term.TermQuery -import de.upb.cs.swt.delphi.webapi.artifacts.{Artifact, ArtifactMetadata, ArtifactTransformer} -import org.joda.time.DateTime -import org.joda.time.format.{DateTimeFormatter, ISODateTimeFormat} -import spray.json.{DefaultJsonProtocol, DeserializationException, JsString, JsValue, RootJsonFormat} +import de.upb.cs.swt.delphi.webapi.artifacts.ArtifactTransformer object RetrieveQuery { def retrieve(identifier: String)(implicit configuration: Configuration) = { diff --git a/src/main/scala/de/upb/cs/swt/delphi/webapi/Server.scala b/src/main/scala/de/upb/cs/swt/delphi/webapi/Server.scala index 47aab61..315f6e9 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/webapi/Server.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/webapi/Server.scala @@ -23,12 +23,14 @@ import akka.http.scaladsl.model.{HttpResponse, StatusCodes} import akka.http.scaladsl.server.{HttpApp, Route} import akka.stream.ActorMaterializer import akka.util.Timeout -import de.upb.cs.swt.delphi.webapi.featuredefinitions.{FeatureExtractor, FeatureListMapping} import de.upb.cs.swt.delphi.instancemanagement.InstanceRegistry -import spray.json._ import de.upb.cs.swt.delphi.webapi.artifacts.ArtifactJson._ +import de.upb.cs.swt.delphi.webapi.search.QueryRequestJson._ +import de.upb.cs.swt.delphi.webapi.featuredefinitions.FeatureExtractor +import de.upb.cs.swt.delphi.webapi.search.{QueryRequest, SearchQuery} +import spray.json._ -import scala.concurrent.{ExecutionContext, Future} +import scala.concurrent.ExecutionContext import scala.util.{Failure, Success} /** @@ -111,11 +113,10 @@ object Server extends HttpApp with JsonSupport with AppLogging { def search: Route = { post { parameter('pretty.?) { (pretty) => - entity(as[JsValue]) { input => - val query = input.asJsObject.fields("query").convertTo[String] - log.info(s"Received search query: $query") + entity(as[QueryRequest]) { input => + log.info(s"Received search query: ${input.query}") complete( - new SearchQuery(configuration, featureExtractor).search(query) match { + new SearchQuery(configuration, featureExtractor).search(input) match { case Success(result) => prettyPrint(pretty, result.toJson) case Failure(e) => e.getMessage } diff --git a/src/main/scala/de/upb/cs/swt/delphi/webapi/search/QueryRequest.scala b/src/main/scala/de/upb/cs/swt/delphi/webapi/search/QueryRequest.scala new file mode 100644 index 0000000..3479577 --- /dev/null +++ b/src/main/scala/de/upb/cs/swt/delphi/webapi/search/QueryRequest.scala @@ -0,0 +1,3 @@ +package de.upb.cs.swt.delphi.webapi.search + +case class QueryRequest (query : String, limit : Option[Int] = Some(50)) diff --git a/src/main/scala/de/upb/cs/swt/delphi/webapi/search/QueryRequestJson.scala b/src/main/scala/de/upb/cs/swt/delphi/webapi/search/QueryRequestJson.scala new file mode 100644 index 0000000..6ae239e --- /dev/null +++ b/src/main/scala/de/upb/cs/swt/delphi/webapi/search/QueryRequestJson.scala @@ -0,0 +1,7 @@ +package de.upb.cs.swt.delphi.webapi.search + +import spray.json.DefaultJsonProtocol + +object QueryRequestJson extends DefaultJsonProtocol { + implicit val queryRequestFormat = jsonFormat2(QueryRequest) +} diff --git a/src/main/scala/de/upb/cs/swt/delphi/webapi/search/SearchQuery.scala b/src/main/scala/de/upb/cs/swt/delphi/webapi/search/SearchQuery.scala index 9990934..b86b4d6 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/webapi/search/SearchQuery.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/webapi/search/SearchQuery.scala @@ -14,30 +14,30 @@ // See the License for the specific language governing permissions and // limitations under the License. -package de.upb.cs.swt.delphi.webapi +package de.upb.cs.swt.delphi.webapi.search -import com.sksamuel.elastic4s.http.{ElasticClient, RequestSuccess} -import de.upb.cs.swt.delphi.webapi.featuredefinitions.FeatureExtractor -import de.upb.cs.swt.delphi.webapi.querylanguage._ import com.sksamuel.elastic4s.http.ElasticDsl._ import com.sksamuel.elastic4s.http.search.SearchHits +import com.sksamuel.elastic4s.http.{ElasticClient, RequestSuccess} import com.sksamuel.elastic4s.searches.queries.{NoopQuery, Query} +import de.upb.cs.swt.delphi.webapi.Configuration import de.upb.cs.swt.delphi.webapi.artifacts.ArtifactTransformer +import de.upb.cs.swt.delphi.webapi.featuredefinitions.FeatureExtractor +import de.upb.cs.swt.delphi.webapi.search.querylanguage._ import scala.util.{Failure, Success, Try} class SearchQuery(configuration: Configuration, featureExtractor: FeatureExtractor) { private val client = ElasticClient(configuration.elasticsearchClientUri) - private def checkAndExecuteParsedQuery(ast: CombinatorialExpr): Try[SearchHits] = { + private def checkAndExecuteParsedQuery(ast: CombinatorialExpr, limit : Int): Try[SearchHits] = { val fields = collectFieldNames(ast) - if (fields.diff(featureExtractor.featureList.toSeq).size > 0) return Failure(null) + if (fields.diff(featureExtractor.featureList.toSeq).size > 0) return Failure(new IllegalArgumentException("Unknown field name used.")) val query = searchWithType(configuration.esProjectIndex) .query(translate(ast)) .sourceInclude(ArtifactTransformer.baseFields ++ fields.intersect(featureExtractor.featureList.toSeq).map(i => addPrefix(i))) - - + .limit(limit) val response = client.execute { query @@ -110,12 +110,12 @@ class SearchQuery(configuration: Configuration, featureExtractor: FeatureExtract } } - def search(query: String) = { - val parserResult = new Syntax(query).QueryRule.run() + def search(query: QueryRequest) = { + val parserResult = new Syntax(query.query).QueryRule.run() parserResult match { case Failure(e) => Failure(e) case Success(ast) => { - checkAndExecuteParsedQuery(ast) match { + checkAndExecuteParsedQuery(ast, query.limit.getOrElse(50)) match { case Failure(e) => Failure(e) case Success(hits) => Success(ArtifactTransformer.transformResults(hits)) } From 9aa80bc16812e1b189601bce931443c270986249 Mon Sep 17 00:00:00 2001 From: Ben Hermann Date: Tue, 30 Oct 2018 12:51:28 +0100 Subject: [PATCH 39/43] Refactorings --- .../swt/delphi/webapi/ElasticActorTest.scala | 111 ----- ...torCheck.scala => FeatureQueryCheck.scala} | 5 +- .../swt/delphi/webapi/SearchQueryTest.scala | 3 +- ...tureExtractor.scala => FeatureQuery.scala} | 8 +- .../de/upb/cs/swt/delphi/webapi/Server.scala | 3 +- .../FeatureDescription.scala | 20 - .../FeatureListMapping.scala | 421 ------------------ .../delphi/webapi/search/SearchQuery.scala | 5 +- 8 files changed, 9 insertions(+), 567 deletions(-) delete mode 100644 src/it/scala/de/upb/cs/swt/delphi/webapi/ElasticActorTest.scala rename src/it/scala/de/upb/cs/swt/delphi/webapi/{FeatureExtractorCheck.scala => FeatureQueryCheck.scala} (82%) rename src/main/scala/de/upb/cs/swt/delphi/webapi/{featuredefinitions/FeatureExtractor.scala => FeatureQuery.scala} (88%) delete mode 100644 src/main/scala/de/upb/cs/swt/delphi/webapi/featuredefinitions/FeatureDescription.scala delete mode 100644 src/main/scala/de/upb/cs/swt/delphi/webapi/featuredefinitions/FeatureListMapping.scala diff --git a/src/it/scala/de/upb/cs/swt/delphi/webapi/ElasticActorTest.scala b/src/it/scala/de/upb/cs/swt/delphi/webapi/ElasticActorTest.scala deleted file mode 100644 index 4857cca..0000000 --- a/src/it/scala/de/upb/cs/swt/delphi/webapi/ElasticActorTest.scala +++ /dev/null @@ -1,111 +0,0 @@ -// Copyright (C) 2018 The Delphi Team. -// See the LICENCE file distributed with this work for additional -// information regarding copyright ownership. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at - -// http://www.apache.org/licenses/LICENSE-2.0 - -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package de.upb.cs.swt.delphi.webapi - -import akka.actor.ActorSystem -import akka.http.scaladsl.Http -import akka.http.scaladsl.model.{HttpRequest, HttpResponse} -import akka.http.scaladsl.unmarshalling.Unmarshal -import akka.stream.ActorMaterializer -import com.sksamuel.elastic4s.RefreshPolicy -import com.sksamuel.elastic4s.http.ElasticClient -import com.sksamuel.elastic4s.http.ElasticDsl._ -import org.scalatest.{BeforeAndAfterAll, FlatSpec, Ignore, Matchers} - -import scala.concurrent.duration._ -import scala.concurrent.{Await, Future} -import scala.util.{Failure, Success} - -/** - * @author Hariharan. - */ -@Ignore -class ElasticActorTest extends FlatSpec with Matchers with BeforeAndAfterAll { - implicit val system = ActorSystem() - implicit val materializer = ActorMaterializer()(system) - implicit val executionContext = system.dispatcher - val configuration = new Configuration() - val client = ElasticClient(configuration.elasticsearchClientUri) - - override def beforeAll(): Unit = { - client.execute { - indexInto("delphi" / "project").fields( - "name" -> "test:elastic-actor-test:1.0" - ).refresh(RefreshPolicy.IMMEDIATE) - }.await - } - - - override def afterAll(): Unit = { - client.execute { - deleteByQuery("delphi", "project", matchQuery("name", "test:elastic-actor-test:1.0")) - }.await - client.close() - system.terminate() - - } - - - "Version no.." should - "match version from build.sbt" in { - val res: Future[HttpResponse] = Http().singleRequest(HttpRequest(uri = "http://localhost:8080/version")) - res.onComplete { - case Success(ver) => { - assert(ver.status.isSuccess()); - val res2Str: Future[String] = Unmarshal(ver.entity).to[String] - res2Str.onComplete { - case Success(value) => { - assert(value.equals(BuildInfo.version)) - } - case Failure(e) => { - assertThrows(e); - } - } - } - case Failure(e) => { - assertThrows(e); - } - } - Await.result(res, 2.seconds) - } - - "Retrive endpoint" should - "get test:elastic-actor-test:1.0 artifact" in { - val mavenId = "test:elastic-actor-test:1.0" - val url = s"http://localhost:8080/retrieve/${mavenId}" - val res: Future[HttpResponse] = Http().singleRequest(HttpRequest(uri = url)) - res.onComplete { - case Success(data) => { - assert(data.status.isSuccess()) - val res2Str: Future[String] = Unmarshal(data.entity).to[String] - res2Str.onComplete { - case Success(value) => { - assert(value.contains(mavenId)) - } - case Failure(e) => { - assertThrows(e); - } - } - } - case Failure(exception) => { - assertThrows(exception) - } - } - Await.result(res, 2.seconds) - } - -} diff --git a/src/it/scala/de/upb/cs/swt/delphi/webapi/FeatureExtractorCheck.scala b/src/it/scala/de/upb/cs/swt/delphi/webapi/FeatureQueryCheck.scala similarity index 82% rename from src/it/scala/de/upb/cs/swt/delphi/webapi/FeatureExtractorCheck.scala rename to src/it/scala/de/upb/cs/swt/delphi/webapi/FeatureQueryCheck.scala index 0d2ddad..ff9a5bc 100644 --- a/src/it/scala/de/upb/cs/swt/delphi/webapi/FeatureExtractorCheck.scala +++ b/src/it/scala/de/upb/cs/swt/delphi/webapi/FeatureQueryCheck.scala @@ -16,13 +16,12 @@ package de.upb.cs.swt.delphi.webapi -import de.upb.cs.swt.delphi.webapi.featuredefinitions.FeatureExtractor import org.scalatest.{FlatSpec, Matchers} -class FeatureExtractorCheck extends FlatSpec with Matchers { +class FeatureQueryCheck extends FlatSpec with Matchers { "FeatureExtractor" should "provide a list of features" in { val configuration = new Configuration() - val featureList = new FeatureExtractor(configuration).featureList + val featureList = new FeatureQuery(configuration).featureList featureList.size should be > 0 } diff --git a/src/it/scala/de/upb/cs/swt/delphi/webapi/SearchQueryTest.scala b/src/it/scala/de/upb/cs/swt/delphi/webapi/SearchQueryTest.scala index 61b194b..1f157d2 100644 --- a/src/it/scala/de/upb/cs/swt/delphi/webapi/SearchQueryTest.scala +++ b/src/it/scala/de/upb/cs/swt/delphi/webapi/SearchQueryTest.scala @@ -16,7 +16,6 @@ package de.upb.cs.swt.delphi.webapi -import de.upb.cs.swt.delphi.webapi.featuredefinitions.FeatureExtractor import de.upb.cs.swt.delphi.webapi.search.{QueryRequest, SearchQuery} import org.scalatest.{FlatSpec, Matchers} @@ -25,7 +24,7 @@ import scala.util.Success class SearchQueryTest extends FlatSpec with Matchers { "Search query" should "check for fields" in { val configuration = new Configuration() - val q = new SearchQuery(configuration, new FeatureExtractor(configuration)) + val q = new SearchQuery(configuration, new FeatureQuery(configuration)) val response = q.search(QueryRequest("[if_icmpeq (opcode:159)]>1")) response shouldBe a [Success[_]] diff --git a/src/main/scala/de/upb/cs/swt/delphi/webapi/featuredefinitions/FeatureExtractor.scala b/src/main/scala/de/upb/cs/swt/delphi/webapi/FeatureQuery.scala similarity index 88% rename from src/main/scala/de/upb/cs/swt/delphi/webapi/featuredefinitions/FeatureExtractor.scala rename to src/main/scala/de/upb/cs/swt/delphi/webapi/FeatureQuery.scala index 543e84d..623e64f 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/webapi/featuredefinitions/FeatureExtractor.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/webapi/FeatureQuery.scala @@ -14,16 +14,14 @@ // See the License for the specific language governing permissions and // limitations under the License. -package de.upb.cs.swt.delphi.webapi.featuredefinitions +package de.upb.cs.swt.delphi.webapi -import com.sksamuel.elastic4s.http.{ElasticClient, RequestSuccess} import com.sksamuel.elastic4s.http.ElasticDsl._ -import com.sksamuel.elastic4s.http.index.mappings.IndexMappings -import de.upb.cs.swt.delphi.webapi.Configuration +import com.sksamuel.elastic4s.http.{ElasticClient, RequestSuccess} import org.slf4j.LoggerFactory import spray.json._ -class FeatureExtractor(configuration: Configuration) { +class FeatureQuery(configuration: Configuration) { private val log = LoggerFactory.getLogger(this.getClass) lazy val featureList: Iterable[String] = { diff --git a/src/main/scala/de/upb/cs/swt/delphi/webapi/Server.scala b/src/main/scala/de/upb/cs/swt/delphi/webapi/Server.scala index 315f6e9..d59dd29 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/webapi/Server.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/webapi/Server.scala @@ -26,7 +26,6 @@ import akka.util.Timeout import de.upb.cs.swt.delphi.instancemanagement.InstanceRegistry import de.upb.cs.swt.delphi.webapi.artifacts.ArtifactJson._ import de.upb.cs.swt.delphi.webapi.search.QueryRequestJson._ -import de.upb.cs.swt.delphi.webapi.featuredefinitions.FeatureExtractor import de.upb.cs.swt.delphi.webapi.search.{QueryRequest, SearchQuery} import spray.json._ @@ -67,7 +66,7 @@ object Server extends HttpApp with JsonSupport with AppLogging { } } - private val featureExtractor = new FeatureExtractor(configuration) + private val featureExtractor = new FeatureQuery(configuration) private def features = { get { diff --git a/src/main/scala/de/upb/cs/swt/delphi/webapi/featuredefinitions/FeatureDescription.scala b/src/main/scala/de/upb/cs/swt/delphi/webapi/featuredefinitions/FeatureDescription.scala deleted file mode 100644 index 6854919..0000000 --- a/src/main/scala/de/upb/cs/swt/delphi/webapi/featuredefinitions/FeatureDescription.scala +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright (C) 2018 The Delphi Team. -// See the LICENCE file distributed with this work for additional -// information regarding copyright ownership. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package de.upb.cs.swt.delphi.webapi.featuredefinitions - -//Describes all relevant fields for conducting searches on this class -case class FeatureDescription(fType: String) diff --git a/src/main/scala/de/upb/cs/swt/delphi/webapi/featuredefinitions/FeatureListMapping.scala b/src/main/scala/de/upb/cs/swt/delphi/webapi/featuredefinitions/FeatureListMapping.scala deleted file mode 100644 index bee0dc8..0000000 --- a/src/main/scala/de/upb/cs/swt/delphi/webapi/featuredefinitions/FeatureListMapping.scala +++ /dev/null @@ -1,421 +0,0 @@ -// Copyright (C) 2018 The Delphi Team. -// See the LICENCE file distributed with this work for additional -// information regarding copyright ownership. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package de.upb.cs.swt.delphi.webapi.featuredefinitions - -object FeatureListMapping { - - - //Returns a list of all defined features - def featureList: List[String] = - (for {(key, des) <- featureMap if des != null} yield key).toList - - //Maps all field names onto descriptions of that field. Null fields cannot be searched in the current version - private val featureMap = Map[String, FeatureDescription]( - "ProjectPackages" -> null, - "⟨SizeOfInheritanceTree⟩" -> null, - "ProjectFields" -> null, - "LibraryMethods" -> null, - "ProjectInstructions" -> null, - "ProjectMethods" -> null, - "LibraryClassFiles" -> null, - "LibraryFields" -> null, - "LibraryPackages" -> null, - "ProjectClassFiles" -> null, - "0 FPC" -> null, - "1-3 FPC" -> null, - "4-10 FPC" -> null, - ">10 FPC" -> null, - "0 MPC" -> null, - "1-3 MPC" -> null, - "4-10 MPC" -> null, - ">10 MPC" -> null, - "1-3 CPP" -> null, - "4-10 CPP" -> null, - ">10 CPP" -> null, - "0 NOC" -> null, - "1-3 NOC" -> null, - "4-10 NOC" -> null, - ">10 NOC" -> null, - "linear methods (McCabe)" -> null, - "2-3 McCabe" -> null, - "4-10 McCabe" -> null, - ">10 McCabe" -> null, - "Designator" -> null, - "Taxonomy" -> null, - "Joiner" -> null, - "Pool" -> null, - "Function Pointer" -> null, - "Function Object" -> null, - "Cobol Like" -> null, - "Stateless" -> null, - "Common State" -> null, - "Immutable" -> null, - "Restricted Creation" -> null, - "Sampler" -> null, - "Box" -> null, - "Compound Box" -> null, - "Canopy" -> null, - "Record" -> null, - "Data Manager" -> null, - "Sink" -> null, - "Outline" -> null, - "Trait" -> null, - "State Machine" -> null, - "Pure Type" -> null, - "Augmented Type" -> null, - "Pseudo Class" -> null, - "Implementor" -> null, - "Overrider" -> null, - "Extender" -> null, - "unused private fields" -> null, - "unused package visible fields" -> null, - "unused protected fields" -> null, - "unused public fields" -> null, - "package visible fields only used by defining type" -> null, - "protected fields only used by defining type" -> null, - "public fields only used by defininig type " -> null, - "Trivial Class.forName Usage" -> null, - "Nontrivial Class.forName Usage" -> null, - "nop (opcode:0)" -> null, - "aconst_null (opcode:1)" -> null, - "iconst_m1 (opcode:2)" -> null, - "iconst_0 (opcode:3)" -> null, - "iconst_1 (opcode:4)" -> null, - "iconst_2 (opcode:5)" -> null, - "iconst_3 (opcode:6)" -> null, - "iconst_4 (opcode:7)" -> null, - "iconst_5 (opcode:8)" -> null, - "lconst_0 (opcode:9)" -> null, - "lconst_1 (opcode:10)" -> null, - "fconst_0 (opcode:11)" -> null, - "fconst_1 (opcode:12)" -> null, - "fconst_2 (opcode:13)" -> null, - "dconst_0 (opcode:14)" -> null, - "dconst_1 (opcode:15)" -> null, - "bipush (opcode:16)" -> null, - "sipush (opcode:17)" -> null, - "ldc (opcode:18)" -> null, - "ldc_w (opcode:19)" -> null, - "ldc2_w (opcode:20)" -> null, - "iload (opcode:21)" -> null, - "lload (opcode:22)" -> null, - "fload (opcode:23)" -> null, - "dload (opcode:24)" -> null, - "aload (opcode:25)" -> null, - "iload_0 (opcode:26)" -> null, - "iload_1 (opcode:27)" -> null, - "iload_2 (opcode:28)" -> null, - "iload_3 (opcode:29)" -> null, - "lload_0 (opcode:30)" -> null, - "lload_1 (opcode:31)" -> null, - "lload_2 (opcode:32)" -> null, - "lload_3 (opcode:33)" -> null, - "fload_0 (opcode:34)" -> null, - "fload_1 (opcode:35)" -> null, - "fload_2 (opcode:36)" -> null, - "fload_3 (opcode:37)" -> null, - "dload_0 (opcode:38)" -> null, - "dload_1 (opcode:39)" -> null, - "dload_2 (opcode:40)" -> null, - "dload_3 (opcode:41)" -> null, - "aload_0 (opcode:42)" -> null, - "aload_1 (opcode:43)" -> null, - "aload_2 (opcode:44)" -> null, - "aload_3 (opcode:45)" -> null, - "iaload (opcode:46)" -> null, - "laload (opcode:47)" -> null, - "faload (opcode:48)" -> null, - "daload (opcode:49)" -> null, - "aaload (opcode:50)" -> null, - "baload (opcode:51)" -> null, - "caload (opcode:52)" -> null, - "saload (opcode:53)" -> null, - "istore (opcode:54)" -> null, - "lstore (opcode:55)" -> null, - "fstore (opcode:56)" -> null, - "dstore (opcode:57)" -> null, - "astore (opcode:58)" -> null, - "istore_0 (opcode:59)" -> null, - "istore_1 (opcode:60)" -> null, - "istore_2 (opcode:61)" -> null, - "istore_3 (opcode:62)" -> null, - "lstore_0 (opcode:63)" -> null, - "lstore_1 (opcode:64)" -> null, - "lstore_2 (opcode:65)" -> null, - "lstore_3 (opcode:66)" -> null, - "fstore_0 (opcode:67)" -> null, - "fstore_1 (opcode:68)" -> null, - "fstore_2 (opcode:69)" -> null, - "fstore_3 (opcode:70)" -> null, - "dstore_0 (opcode:71)" -> null, - "dstore_1 (opcode:72)" -> null, - "dstore_2 (opcode:73)" -> null, - "dstore_3 (opcode:74)" -> null, - "astore_0 (opcode:75)" -> null, - "astore_1 (opcode:76)" -> null, - "astore_2 (opcode:77)" -> null, - "astore_3 (opcode:78)" -> null, - "iastore (opcode:79)" -> null, - "lastore (opcode:80)" -> null, - "fastore (opcode:81)" -> null, - "dastore (opcode:82)" -> null, - "aastore (opcode:83)" -> null, - "bastore (opcode:84)" -> null, - "castore (opcode:85)" -> null, - "sastore (opcode:86)" -> null, - "pop (opcode:87)" -> null, - "pop2 (opcode:88)" -> null, - "dup (opcode:89)" -> null, - "dup_x1 (opcode:90)" -> null, - "dup_x2 (opcode:91)" -> null, - "dup2 (opcode:92)" -> null, - "dup2_x1 (opcode:93)" -> null, - "dup2_x2 (opcode:94)" -> null, - "swap (opcode:95)" -> null, - "iadd (opcode:96)" -> null, - "ladd (opcode:97)" -> null, - "fadd (opcode:98)" -> null, - "dadd (opcode:99)" -> null, - "isub (opcode:100)" -> null, - "lsub (opcode:101)" -> null, - "fsub (opcode:102)" -> null, - "dsub (opcode:103)" -> null, - "imul (opcode:104)" -> null, - "lmul (opcode:105)" -> null, - "fmul (opcode:106)" -> null, - "dmul (opcode:107)" -> null, - "idiv (opcode:108)" -> null, - "ldiv (opcode:109)" -> null, - "fdiv (opcode:110)" -> null, - "ddiv (opcode:111)" -> null, - "irem (opcode:112)" -> null, - "lrem (opcode:113)" -> null, - "frem (opcode:114)" -> null, - "drem (opcode:115)" -> null, - "ineg (opcode:116)" -> null, - "lneg (opcode:117)" -> null, - "fneg (opcode:118)" -> null, - "dneg (opcode:119)" -> null, - "ishl (opcode:120)" -> null, - "lshl (opcode:121)" -> null, - "ishr (opcode:122)" -> null, - "lshr (opcode:123)" -> null, - "iushr (opcode:124)" -> null, - "lushr (opcode:125)" -> null, - "iand (opcode:126)" -> null, - "land (opcode:127)" -> null, - "ior (opcode:128)" -> null, - "lor (opcode:129)" -> null, - "ixor (opcode:130)" -> null, - "lxor (opcode:131)" -> null, - "iinc (opcode:132)" -> null, - "i2l (opcode:133)" -> null, - "i2f (opcode:134)" -> null, - "i2d (opcode:135)" -> null, - "l2i (opcode:136)" -> null, - "l2f (opcode:137)" -> null, - "l2d (opcode:138)" -> null, - "f2i (opcode:139)" -> null, - "f2l (opcode:140)" -> null, - "f2d (opcode:141)" -> null, - "d2i (opcode:142)" -> null, - "d2l (opcode:143)" -> null, - "d2f (opcode:144)" -> null, - "i2b (opcode:145)" -> null, - "i2c (opcode:146)" -> null, - "i2s (opcode:147)" -> null, - "lcmp (opcode:148)" -> null, - "fcmpl (opcode:149)" -> null, - "fcmpg (opcode:150)" -> null, - "dcmpl (opcode:151)" -> null, - "dcmpg (opcode:152)" -> null, - "ifeq (opcode:153)" -> null, - "ifne (opcode:154)" -> null, - "iflt (opcode:155)" -> null, - "ifge (opcode:156)" -> null, - "ifgt (opcode:157)" -> null, - "ifle (opcode:158)" -> null, - "if_icmpeq (opcode:159)" -> null, - "if_icmpne (opcode:160)" -> null, - "if_icmplt (opcode:161)" -> null, - "if_icmpge (opcode:162)" -> null, - "if_icmpgt (opcode:163)" -> null, - "if_icmple (opcode:164)" -> null, - "if_acmpeq (opcode:165)" -> null, - "if_acmpne (opcode:166)" -> null, - "goto (opcode:167)" -> null, - "jsr (opcode:168)" -> null, - "ret (opcode:169)" -> null, - "tableswitch (opcode:170)" -> null, - "lookupswitch (opcode:171)" -> null, - "ireturn (opcode:172)" -> null, - "lreturn (opcode:173)" -> null, - "freturn (opcode:174)" -> null, - "dreturn (opcode:175)" -> null, - "areturn (opcode:176)" -> null, - "return (opcode:177)" -> null, - "getstatic (opcode:178)" -> null, - "putstatic (opcode:179)" -> null, - "getfield (opcode:180)" -> null, - "putfield (opcode:181)" -> null, - "invokevirtual (opcode:182)" -> null, - "invokespecial (opcode:183)" -> null, - "invokestatic (opcode:184)" -> null, - "invokeinterface (opcode:185)" -> null, - "invokedynamic (opcode:186)" -> null, - "new (opcode:187)" -> null, - "newarray (opcode:188)" -> null, - "anewarray (opcode:189)" -> null, - "arraylength (opcode:190)" -> null, - "athrow (opcode:191)" -> null, - "checkcast (opcode:192)" -> null, - "instanceof (opcode:193)" -> null, - "monitorenter (opcode:194)" -> null, - "monitorexit (opcode:195)" -> null, - "wide (opcode:196)" -> null, - "multianewarray (opcode:197)" -> null, - "ifnull (opcode:198)" -> null, - "ifnonnull (opcode:199)" -> null, - "goto_w (opcode:200)" -> null, - "jsr_w (opcode:201)" -> null, - "Self-recursive Data Structure" -> null, - "Mutually-recursive Data Structure 2 Types" -> null, - "Mutually-recursive Data Structure 3 Types" -> null, - "Mutually-recursive Data Structure 4 Types" -> null, - "Mutually-recursive Data Structure more than 4 Types" -> null, - "Never Returns Normally" -> null, - "Method with Infinite Loop" -> null, - "Class File With Source Attribute" -> null, - "Method With Line Number Table" -> null, - "Method With Local Variable Table" -> null, - "Method With Local Variable Type Table" -> null, - "FanOut - Category 1" -> null, - "FanOut - Category 2" -> null, - "FanOut - Category 3" -> null, - "FanOut - Category 4" -> null, - "FanOut - Category 5" -> null, - "FanOut - Category 6" -> null, - "FanIn - Category 1" -> null, - "FanIn - Category 2" -> null, - "FanIn - Category 3" -> null, - "FanIn - Category 4" -> null, - "FanIn - Category 5" -> null, - "FanIn - Category 6" -> null, - "FanIn/FanOut - Category 1" -> null, - "FanIn/FanOut - Category 2" -> null, - "FanIn/FanOut - Category 3" -> null, - "FanIn/FanOut - Category 4" -> null, - "FanIn/FanOut - Category 5" -> null, - "FanIn/FanOut - Category 6" -> null, - "custom ClassLoader implementation" -> null, - "Retrieving the SystemClassLoader" -> null, - "Retrieving some ClassLoader" -> null, - "define new classes/packages" -> null, - "accessing resources" -> null, - "javax.crypto.Cipher getInstance" -> null, - "using SecureRandom" -> null, - "using MessageDigest" -> null, - "using Signature" -> null, - "using Mac" -> null, - "cryptographic key handling" -> null, - "using KeyStore" -> null, - "using Certificates" -> null, - "native methods" -> null, - "synthetic methods" -> null, - "bridge methods" -> null, - "synchronized methods" -> null, - "varargs methods" -> null, - "static initializers" -> null, - "static methods (not including static initializers)" -> null, - "constructors" -> null, - "instance methods" -> null, - "java.lang.Class forName" -> null, - "reflective instance creation" -> null, - "reflective field write" -> null, - "reflective field read" -> null, - "makes fields accessible" -> null, - "makes methods or constructors accessible" -> null, - "makes an AccessibleObject accessible (exact type unknown)" -> null, - "java.lang.reflect.Method Object invoke(Object, Object[])" -> null, - "java.lang.invoke.MethodHandles lookup" -> null, - "java.lang.invoke.MethodHandles publicLookup" -> null, - "method handle invocation" -> null, - "java.lang.reflect.Proxy newProxyInstance" -> null, - "Process" -> null, - "JVM exit" -> null, - "Native Libraries" -> null, - "java.lang.System getSecurityManager" -> null, - "java.lang.System setSecurityManager" -> null, - "Environment" -> null, - "Sound" -> null, - "Network sockets" -> null, - "Object-based Thread Notification" -> null, - "Usage of Thread API" -> null, - "Usage of ThreadGroup API" -> null, - "sun.misc.Unsafe sun.misc.Unsafe getUnsafe()" -> null, - "Unsafe - Alloc" -> null, - "Unsafe - Array" -> null, - "Unsafe - compareAndSwap" -> null, - "Unsafe - Class" -> null, - "Unsafe - Fence" -> null, - "Unsafe - Fetch & Add" -> null, - "Unsafe - Heap" -> null, - "Unsafe - Heap Get" -> null, - "Unsafe - Heap Put" -> null, - "Misc" -> null, - "Unsafe - Monitor" -> null, - "Unsafe - Off-Heap" -> null, - "Unsafe - Offset" -> null, - "Unsafe - Ordered Put" -> null, - "Unsafe - Park" -> null, - "Unsafe - Throw" -> null, - "Unsafe - Volatile Get" -> null, - "Unsafe - Volatile Put" -> null, - "java.sql.DriverManager getConnection" -> null, - "java.sql.Connection rollback" -> null, - "creation and execution of java.sql.Statement" -> null, - "creation and execution of java.sql.PreparedStatement" -> null, - "creation and execution of java.sql.CallableStatement" -> null, - "class file retransformation" -> null, - "instrumenting native methods" -> null, - "appending class loader search" -> null, - "retrieve classes information" -> null, - "(concrete) classes" -> null, - "abstract classes" -> null, - "annotations" -> null, - "enumerations" -> null, - "marker interfaces" -> null, - "simple functional interfaces (single abstract method (SAM) interface)" -> null, - "non-functional interface with default methods (Java >8)" -> null, - "non-functional interface with static methods (Java >8)" -> null, - "(standard) interface" -> null, - "module (Java >9)" -> null, - "Very Small Inheritance Tree" -> null, - "Small Inheritance Tree" -> null, - "Medium Inheritance Tree" -> null, - "Large Inheritance Tree" -> null, - "Very Large Inheritance Tree" -> null, - "Huge Inheritance Tree" -> null, - "Size of the Inheritance Tree Unknown" -> null, - "Class File JDK 1.1 (JDK 1.0.2)" -> null, - "Class File Java 5" -> null, - "Class File Java 6" -> null, - "Class File Java 7" -> null, - "Class File Java 8" -> null, - "Class File Java 9" -> null - ) -} diff --git a/src/main/scala/de/upb/cs/swt/delphi/webapi/search/SearchQuery.scala b/src/main/scala/de/upb/cs/swt/delphi/webapi/search/SearchQuery.scala index b86b4d6..80c11b1 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/webapi/search/SearchQuery.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/webapi/search/SearchQuery.scala @@ -20,14 +20,13 @@ import com.sksamuel.elastic4s.http.ElasticDsl._ import com.sksamuel.elastic4s.http.search.SearchHits import com.sksamuel.elastic4s.http.{ElasticClient, RequestSuccess} import com.sksamuel.elastic4s.searches.queries.{NoopQuery, Query} -import de.upb.cs.swt.delphi.webapi.Configuration +import de.upb.cs.swt.delphi.webapi.{Configuration, FeatureQuery} import de.upb.cs.swt.delphi.webapi.artifacts.ArtifactTransformer -import de.upb.cs.swt.delphi.webapi.featuredefinitions.FeatureExtractor import de.upb.cs.swt.delphi.webapi.search.querylanguage._ import scala.util.{Failure, Success, Try} -class SearchQuery(configuration: Configuration, featureExtractor: FeatureExtractor) { +class SearchQuery(configuration: Configuration, featureExtractor: FeatureQuery) { private val client = ElasticClient(configuration.elasticsearchClientUri) private def checkAndExecuteParsedQuery(ast: CombinatorialExpr, limit : Int): Try[SearchHits] = { From 529e9548b8b2525ad94db593c919d3a729f01258 Mon Sep 17 00:00:00 2001 From: Ben Hermann Date: Tue, 30 Oct 2018 12:53:14 +0100 Subject: [PATCH 40/43] Changed scalastyle configuration --- project/scalastyle-config.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/project/scalastyle-config.xml b/project/scalastyle-config.xml index c220edc..88ff84c 100644 --- a/project/scalastyle-config.xml +++ b/project/scalastyle-config.xml @@ -8,8 +8,8 @@ - Date: Tue, 30 Oct 2018 13:59:48 +0100 Subject: [PATCH 41/43] Fixed logging --- build.sbt | 4 +++- src/main/resources/application.conf | 8 +++++++ src/main/resources/logback.xml | 36 +++++++++++++++++++++++++++++ 3 files changed, 47 insertions(+), 1 deletion(-) create mode 100644 src/main/resources/logback.xml diff --git a/build.sbt b/build.sbt index 4948af6..5737398 100644 --- a/build.sbt +++ b/build.sbt @@ -11,11 +11,13 @@ libraryDependencies += "org.parboiled" %% "parboiled" % "2.1.4" libraryDependencies += "com.typesafe.akka" %% "akka-http" % akkaHttpVersion libraryDependencies += "com.typesafe.akka" %% "akka-http-testkit" % akkaHttpVersion libraryDependencies += "com.typesafe.akka" %% "akka-stream" % "2.5.12" +libraryDependencies += "com.typesafe.akka" %% "akka-slf4j" % "2.5.12" libraryDependencies += "com.typesafe.akka" %% "akka-http-spray-json" % akkaHttpVersion libraryDependencies += "io.spray" %% "spray-json" % "1.3.3" libraryDependencies += "org.scalactic" %% "scalactic" % "3.0.4" libraryDependencies += "org.scalatest" %% "scalatest" % "3.0.4" % "it,test" -libraryDependencies += "org.apache.logging.log4j" % "log4j-core" % "2.11.1" + +libraryDependencies += "ch.qos.logback" % "logback-classic" % "1.2.3" % Runtime val elastic4sVersion = "6.3.0" libraryDependencies ++= Seq( diff --git a/src/main/resources/application.conf b/src/main/resources/application.conf index 2840c06..fef50f4 100644 --- a/src/main/resources/application.conf +++ b/src/main/resources/application.conf @@ -3,6 +3,14 @@ akka.actor.deployment { } akka { + + loggers = ["akka.event.slf4j.Slf4jLogger"] + logLevel = "INFO" + event-handlers = ["akka.event.slf4j.Slf4jEventHandler"] + log-dead-letters = 0 + log-dead-letters-during-shutdown = off + log-config-on-start = off + http { server { remote-address-header = on diff --git a/src/main/resources/logback.xml b/src/main/resources/logback.xml new file mode 100644 index 0000000..e6c0015 --- /dev/null +++ b/src/main/resources/logback.xml @@ -0,0 +1,36 @@ + + + + + + + ${application.home:-.}/logs/application.log + + [%level] [%date] from %logger in %thread - %message%n%xException + + + + + + [%date] %logger{15} - %message%n%xException{10} + + + + + + + + + + + + + + + + + + + + + From e91fd60ac21abb15cebe5ab9faa16c430dd0a0eb Mon Sep 17 00:00:00 2001 From: Ben Hermann Date: Tue, 30 Oct 2018 14:32:56 +0100 Subject: [PATCH 42/43] Organized dependencies --- build.sbt | 35 ++++++++++++++++------------------- 1 file changed, 16 insertions(+), 19 deletions(-) diff --git a/build.sbt b/build.sbt index 5737398..bc0ac1d 100644 --- a/build.sbt +++ b/build.sbt @@ -4,34 +4,31 @@ version := "1.0.0-SNAPSHOT" scalaVersion := "2.12.4" -val akkaHttpVersion = "10.1.5" - -libraryDependencies += "org.parboiled" %% "parboiled" % "2.1.4" - -libraryDependencies += "com.typesafe.akka" %% "akka-http" % akkaHttpVersion -libraryDependencies += "com.typesafe.akka" %% "akka-http-testkit" % akkaHttpVersion -libraryDependencies += "com.typesafe.akka" %% "akka-stream" % "2.5.12" -libraryDependencies += "com.typesafe.akka" %% "akka-slf4j" % "2.5.12" -libraryDependencies += "com.typesafe.akka" %% "akka-http-spray-json" % akkaHttpVersion -libraryDependencies += "io.spray" %% "spray-json" % "1.3.3" -libraryDependencies += "org.scalactic" %% "scalactic" % "3.0.4" -libraryDependencies += "org.scalatest" %% "scalatest" % "3.0.4" % "it,test" +val akkaVersion = "2.5.14" +libraryDependencies ++= Seq ( + "com.typesafe.akka" %% "akka-stream" % akkaVersion, + "com.typesafe.akka" %% "akka-slf4j" % akkaVersion +) -libraryDependencies += "ch.qos.logback" % "logback-classic" % "1.2.3" % Runtime +val akkaHttpVersion = "10.1.5" +libraryDependencies ++= Seq ( + "com.typesafe.akka" %% "akka-http" % akkaHttpVersion, + "com.typesafe.akka" %% "akka-http-testkit" % akkaHttpVersion, + "com.typesafe.akka" %% "akka-http-spray-json" % akkaHttpVersion +) val elastic4sVersion = "6.3.0" libraryDependencies ++= Seq( "com.sksamuel.elastic4s" %% "elastic4s-core" % elastic4sVersion, - - - // for the http client "com.sksamuel.elastic4s" %% "elastic4s-http" % elastic4sVersion, - - // if you want to use reactive streams "com.sksamuel.elastic4s" %% "elastic4s-http-streams" % elastic4sVersion, - ) +libraryDependencies += "org.parboiled" %% "parboiled" % "2.1.4" +libraryDependencies += "io.spray" %% "spray-json" % "1.3.3" +libraryDependencies += "org.scalactic" %% "scalactic" % "3.0.4" +libraryDependencies += "org.scalatest" %% "scalatest" % "3.0.4" % "it,test" +libraryDependencies += "ch.qos.logback" % "logback-classic" % "1.2.3" % Runtime lazy val webapi = (project in file(".")). //https://www.scala-sbt.org/1.x/docs/Testing.html From 7b5e1f291dc2f1daf8f36d6faa6fbf09b3957343 Mon Sep 17 00:00:00 2001 From: Ben Hermann Date: Thu, 1 Nov 2018 09:58:27 +0100 Subject: [PATCH 43/43] Fixed retrieve query --- .../swt/delphi/webapi/MavenIdentifier.scala | 19 ++++--- .../cs/swt/delphi/webapi/RetrieveQuery.scala | 49 +++++-------------- .../artifacts/ArtifactTransformer.scala | 13 +++-- 3 files changed, 30 insertions(+), 51 deletions(-) diff --git a/src/main/scala/de/upb/cs/swt/delphi/webapi/MavenIdentifier.scala b/src/main/scala/de/upb/cs/swt/delphi/webapi/MavenIdentifier.scala index 3d88347..81be0c1 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/webapi/MavenIdentifier.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/webapi/MavenIdentifier.scala @@ -23,8 +23,8 @@ import de.upb.cs.swt.delphi.webapi.artifacts.Identifier case class MavenIdentifier(val repository: Option[String], val groupId: String, val artifactId: String, val version: Option[String]) extends Identifier { - def toUniqueString = { - repository + ":" + groupId + ":" + artifactId + ":" + version + def toUniqueString: String = { + repository.getOrElse(MavenIdentifier.DefaultRepository) + ":" + groupId + ":" + artifactId + ":" + version.getOrElse("") } override val toString: String = groupId + ":" + artifactId + ":" + version @@ -38,7 +38,7 @@ case class MavenIdentifier(val repository: Option[String], val groupId: String, } private def constructArtifactBaseUri(): URI = - new URI(repository.getOrElse("")) + new URI(repository.getOrElse(MavenIdentifier.DefaultRepository)) .resolve(encode(groupId).replace('.', '/') + "/") .resolve(encode(artifactId) + "/") .resolve(encode(version.getOrElse("")) + "/") @@ -48,16 +48,15 @@ case class MavenIdentifier(val repository: Option[String], val groupId: String, } object MavenIdentifier { + private val DefaultRepository = "http://repo1.maven.org/maven2/" + private implicit def wrapOption[A](value : A) : Option[A] = Some(value) + def apply(s: String): Option[MavenIdentifier] = { val splitString: Array[String] = s.split(':') if (splitString.length < 2 || splitString.length > 3) return None - //if (splitString(0).startsWith("http")) { - // if (splitString.length < 3) return None - // MavenIdentifier(splitString(0), splitString(1), splitString(2), if (splitString.length < 4) None else splitString(3)) - //} else { - //if (splitString.length > 3) return None - MavenIdentifier(None, splitString(0), splitString(1), if (splitString.length < 3) None else splitString(2)) - //} + + MavenIdentifier(None, splitString(0), splitString(1), if (splitString.length < 3) None else splitString(2)) + } } \ No newline at end of file diff --git a/src/main/scala/de/upb/cs/swt/delphi/webapi/RetrieveQuery.scala b/src/main/scala/de/upb/cs/swt/delphi/webapi/RetrieveQuery.scala index dd149ed..6c8a942 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/webapi/RetrieveQuery.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/webapi/RetrieveQuery.scala @@ -16,39 +16,32 @@ package de.upb.cs.swt.delphi.webapi -import com.sksamuel.elastic4s.http.ElasticDsl.{termQuery, _} -import com.sksamuel.elastic4s.http.search.SearchResponse +import com.sksamuel.elastic4s.http.ElasticDsl._ +import com.sksamuel.elastic4s.http.get.GetResponse import com.sksamuel.elastic4s.http.{ElasticClient, RequestSuccess, Response} -import com.sksamuel.elastic4s.searches.queries.Query -import com.sksamuel.elastic4s.searches.queries.term.TermQuery -import de.upb.cs.swt.delphi.webapi.artifacts.ArtifactTransformer +import de.upb.cs.swt.delphi.webapi.artifacts.{Artifact, ArtifactTransformer} object RetrieveQuery { - def retrieve(identifier: String)(implicit configuration: Configuration) = { + + def retrieve(identifier: String)(implicit configuration: Configuration): Option[List[Artifact]] = { val client = ElasticClient(configuration.elasticsearchClientUri) val parsedIdentifier = MavenIdentifier(identifier) - parsedIdentifier match { case None => None case Some(m) => { - val response: Response[SearchResponse] = client.execute { - searchWithType(configuration.esProjectIndex) query { - bool { - must( - constructIdQuery(m) - ) - } - } + + val response: Response[GetResponse] = client.execute { + get(configuration.esProjectIndex.index, configuration.esProjectIndex.`type`, m.toUniqueString) }.await response match { - case RequestSuccess(_, body, _, results: SearchResponse) => { - results.totalHits match { - case 0L => None + case RequestSuccess(_, body, _, results: GetResponse) => { + results.found match { + case false => None case _ => { - Some(ArtifactTransformer.transformResults(results.hits)) + Some(List(ArtifactTransformer.transformResult(results.id, results.sourceAsMap))) } } } @@ -56,25 +49,7 @@ object RetrieveQuery { } } } - - } - - private def constructIdQuery(m: MavenIdentifier): Iterable[Query] = { - val baseQuery: List[TermQuery] = List( - termQuery("identifier.groupId", m.groupId), - termQuery("identifier.artifactId", m.artifactId) - ) - m.version.isDefined match { - case true => { - baseQuery.+:(termQuery("identifier.version", m.version.getOrElse(""))) - } - case false => { - baseQuery - } - } - } - } diff --git a/src/main/scala/de/upb/cs/swt/delphi/webapi/artifacts/ArtifactTransformer.scala b/src/main/scala/de/upb/cs/swt/delphi/webapi/artifacts/ArtifactTransformer.scala index 6e429d4..a34fd12 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/webapi/artifacts/ArtifactTransformer.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/webapi/artifacts/ArtifactTransformer.scala @@ -38,12 +38,17 @@ object ArtifactTransformer { identifier("groupId"), identifier("artifactId"), identifier("version")) } - private def transformResult(h: SearchHit) = { - val sourceMap = h.sourceAsMap - Artifact(h.id, getMetadata(sourceMap), getHermesResults(sourceMap)) + def transformResult(id : String, sourceMap : Map[String, AnyRef]) :Artifact = { + Artifact(id, getMetadata(sourceMap), getHermesResults(sourceMap)) } - def transformResults(hits: SearchHits) = { + + private def transformResult(h: SearchHit): Artifact = { + transformResult(h.id, h.sourceAsMap) + } + + + def transformResults(hits: SearchHits): Array[Artifact] = { hits.hits.map(h => transformResult(h)) }