diff --git a/README.md b/README.md index 4c912c8..5c5b545 100644 --- a/README.md +++ b/README.md @@ -5,10 +5,10 @@ The web API implementation for the Delphi platform. We are currently in pre-alpha state! There is no release and the code in this repository is purely experimental! -|branch | status | codacy | -| :---: | :---: | :---: | -| master | [![Build Status](https://travis-ci.org/delphi-hub/delphi-webapi.svg?branch=master)](https://travis-ci.org/delphi-hub/delphi-webapi) | [![Codacy Badge](https://api.codacy.com/project/badge/Grade/8ebe27850ffb4139af6280fd1cd6d540)](https://www.codacy.com/app/delphi-hub/delphi-webapi?utm_source=github.com&utm_medium=referral&utm_content=delphi-hub/delphi-webapi&utm_campaign=Badge_Grade)| -| develop | [![Build Status](https://travis-ci.org/delphi-hub/delphi-webapi.svg?branch=develop)](https://travis-ci.org/delphi-hub/delphi-webapi) | [![Codacy Badge](https://api.codacy.com/project/badge/Grade/8ebe27850ffb4139af6280fd1cd6d540?branch=develop)](https://www.codacy.com/app/delphi-hub/delphi-webapi?branch=develop&utm_source=github.com&utm_medium=referral&utm_content=delphi-hub/delphi-webapi&utm_campaign=Badge_Grade) | +|branch | status | codacy | snyk | +| :---: | :---: | :---: | :---: | +| master | [![Build Status](https://travis-ci.org/delphi-hub/delphi-webapi.svg?branch=master)](https://travis-ci.org/delphi-hub/delphi-webapi) | [![Codacy Badge](https://api.codacy.com/project/badge/Grade/8ebe27850ffb4139af6280fd1cd6d540)](https://www.codacy.com/app/delphi-hub/delphi-webapi?utm_source=github.com&utm_medium=referral&utm_content=delphi-hub/delphi-webapi&utm_campaign=Badge_Grade)| [![Known Vulnerabilities](https://snyk.io/test/github/delphi-hub/delphi-webapi/badge.svg?targetFile=build.sbt)](https://snyk.io/test/github/delphi-hub/delphi-webapi/?targetFile=build.sbt) | +| develop | [![Build Status](https://travis-ci.org/delphi-hub/delphi-webapi.svg?branch=develop)](https://travis-ci.org/delphi-hub/delphi-webapi) | [![Codacy Badge](https://api.codacy.com/project/badge/Grade/8ebe27850ffb4139af6280fd1cd6d540?branch=develop)](https://www.codacy.com/app/delphi-hub/delphi-webapi?branch=develop&utm_source=github.com&utm_medium=referral&utm_content=delphi-hub/delphi-webapi&utm_campaign=Badge_Grade) | [![Known Vulnerabilities](https://snyk.io/test/github/delphi-hub/delphi-webapi/develop/badge.svg?targetFile=build.sbt)](https://snyk.io/test/github/delphi-hub/delphi-webapi/develop/?targetFile=build.sbt) | ## What is the Delphi Web API? diff --git a/build.sbt b/build.sbt index c9ab5ec..bc0ac1d 100644 --- a/build.sbt +++ b/build.sbt @@ -4,36 +4,42 @@ version := "1.0.0-SNAPSHOT" scalaVersion := "2.12.4" -libraryDependencies += "org.parboiled" %% "parboiled" % "2.1.4" -libraryDependencies += "com.typesafe.akka" %% "akka-http" % "10.0.11" -libraryDependencies += "com.typesafe.akka" %% "akka-stream" % "2.5.12" -libraryDependencies += "com.typesafe.akka" %% "akka-http-spray-json" % "10.1.1" -libraryDependencies += "io.spray" %% "spray-json" % "1.3.3" -libraryDependencies += "org.parboiled" %% "parboiled" % "2.1.4" -libraryDependencies += "org.scalactic" %% "scalactic" % "3.0.4" -libraryDependencies += "org.scalatest" %% "scalatest" % "3.0.4" % "test" +val akkaVersion = "2.5.14" +libraryDependencies ++= Seq ( + "com.typesafe.akka" %% "akka-stream" % akkaVersion, + "com.typesafe.akka" %% "akka-slf4j" % akkaVersion +) + +val akkaHttpVersion = "10.1.5" +libraryDependencies ++= Seq ( + "com.typesafe.akka" %% "akka-http" % akkaHttpVersion, + "com.typesafe.akka" %% "akka-http-testkit" % akkaHttpVersion, + "com.typesafe.akka" %% "akka-http-spray-json" % akkaHttpVersion +) val elastic4sVersion = "6.3.0" libraryDependencies ++= Seq( "com.sksamuel.elastic4s" %% "elastic4s-core" % elastic4sVersion, - - // for the http client "com.sksamuel.elastic4s" %% "elastic4s-http" % elastic4sVersion, - - // if you want to use reactive streams "com.sksamuel.elastic4s" %% "elastic4s-http-streams" % elastic4sVersion, - - // testing - "com.sksamuel.elastic4s" %% "elastic4s-testkit" % elastic4sVersion % "test", - "com.sksamuel.elastic4s" %% "elastic4s-embedded" % elastic4sVersion % "test" ) +libraryDependencies += "org.parboiled" %% "parboiled" % "2.1.4" +libraryDependencies += "io.spray" %% "spray-json" % "1.3.3" +libraryDependencies += "org.scalactic" %% "scalactic" % "3.0.4" +libraryDependencies += "org.scalatest" %% "scalatest" % "3.0.4" % "it,test" +libraryDependencies += "ch.qos.logback" % "logback-classic" % "1.2.3" % Runtime lazy val webapi = (project in file(".")). + //https://www.scala-sbt.org/1.x/docs/Testing.html +configs(IntegrationTest). + settings( + Defaults.itSettings, + ). enablePlugins(JavaAppPackaging). enablePlugins(DockerPlugin). enablePlugins(ScalastylePlugin). - settings ( + settings( dockerBaseImage := "openjdk:jre-alpine" ). enablePlugins(AshScriptPlugin). @@ -44,3 +50,11 @@ lazy val webapi = (project in file(".")). ) scalastyleConfig := baseDirectory.value / "project" / "scalastyle-config.xml" + +// Pinning secure versions of insecure transitive libraryDependencies +// Please update when updating dependencies above (including Play plugin) +libraryDependencies ++= Seq( + "com.fasterxml.jackson.core" % "jackson-databind" % "2.9.7" +) + +trapExit := false diff --git a/project/scalastyle-config.xml b/project/scalastyle-config.xml index c220edc..88ff84c 100644 --- a/project/scalastyle-config.xml +++ b/project/scalastyle-config.xml @@ -8,8 +8,8 @@ - 0 + } +} diff --git a/src/it/scala/de/upb/cs/swt/delphi/webapi/SearchQueryTest.scala b/src/it/scala/de/upb/cs/swt/delphi/webapi/SearchQueryTest.scala new file mode 100644 index 0000000..1f157d2 --- /dev/null +++ b/src/it/scala/de/upb/cs/swt/delphi/webapi/SearchQueryTest.scala @@ -0,0 +1,32 @@ +// Copyright (C) 2018 The Delphi Team. +// See the LICENCE file distributed with this work for additional +// information regarding copyright ownership. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package de.upb.cs.swt.delphi.webapi + +import de.upb.cs.swt.delphi.webapi.search.{QueryRequest, SearchQuery} +import org.scalatest.{FlatSpec, Matchers} + +import scala.util.Success + +class SearchQueryTest extends FlatSpec with Matchers { + "Search query" should "check for fields" in { + val configuration = new Configuration() + val q = new SearchQuery(configuration, new FeatureQuery(configuration)) + + val response = q.search(QueryRequest("[if_icmpeq (opcode:159)]>1")) + response shouldBe a [Success[_]] + } +} diff --git a/src/it/scala/de/upb/cs/swt/delphi/webapi/StatisticsQueryCheck.scala b/src/it/scala/de/upb/cs/swt/delphi/webapi/StatisticsQueryCheck.scala new file mode 100644 index 0000000..ef1af7b --- /dev/null +++ b/src/it/scala/de/upb/cs/swt/delphi/webapi/StatisticsQueryCheck.scala @@ -0,0 +1,11 @@ +package de.upb.cs.swt.delphi.webapi + +import org.scalatest.{FlatSpec, Matchers} + +class StatisticsQueryCheck extends FlatSpec with Matchers { + "Statics" should "be retrievable" in { + val configuration = new Configuration() + val stats = new StatisticsQuery(configuration) + println(stats.retrieveStandardStatistics) + } +} diff --git a/src/main/resources/application.conf b/src/main/resources/application.conf index 935d3b3..fef50f4 100644 --- a/src/main/resources/application.conf +++ b/src/main/resources/application.conf @@ -1,28 +1,19 @@ -es-priority-mailbox { - mailbox-type = "de.upb.cs.swt.delphi.webapi.ElasticPriorityMailbox" -} - akka.actor.deployment { - /espriomailboxactor { - mailbox = es-priority-mailbox - } + } akka { + + loggers = ["akka.event.slf4j.Slf4jLogger"] + logLevel = "INFO" + event-handlers = ["akka.event.slf4j.Slf4jEventHandler"] + log-dead-letters = 0 + log-dead-letters-during-shutdown = off + log-config-on-start = off + http { server { remote-address-header = on } } } - -# Use this dispatcher for actors that make blocking calls to the Elasticsearch database -elasticsearch-handling-dispatcher { - type = Dispatcher - executor = "thread-pool-executor" - thread-pool-executor { - fixed-pool-size = 4 - # This thread pool is intended for development purposes, and should be increased for production - } - throughput = 1 -} \ No newline at end of file diff --git a/src/main/resources/logback.xml b/src/main/resources/logback.xml new file mode 100644 index 0000000..e6c0015 --- /dev/null +++ b/src/main/resources/logback.xml @@ -0,0 +1,36 @@ + + + + + + + ${application.home:-.}/logs/application.log + + [%level] [%date] from %logger in %thread - %message%n%xException + + + + + + [%date] %logger{15} - %message%n%xException{10} + + + + + + + + + + + + + + + + + + + + + diff --git a/src/main/scala/de/upb/cs/swt/delphi/featuredefinitions/FeatureDescription.scala b/src/main/scala/de/upb/cs/swt/delphi/featuredefinitions/FeatureDescription.scala deleted file mode 100644 index 944d03a..0000000 --- a/src/main/scala/de/upb/cs/swt/delphi/featuredefinitions/FeatureDescription.scala +++ /dev/null @@ -1,4 +0,0 @@ -package de.upb.cs.swt.delphi.featuredefinitions - -//Describes all relevant fields for conducting searches on this class -case class FeatureDescription(fType: String) diff --git a/src/main/scala/de/upb/cs/swt/delphi/featuredefinitions/FeatureListMapping.scala b/src/main/scala/de/upb/cs/swt/delphi/featuredefinitions/FeatureListMapping.scala deleted file mode 100644 index d4c165f..0000000 --- a/src/main/scala/de/upb/cs/swt/delphi/featuredefinitions/FeatureListMapping.scala +++ /dev/null @@ -1,404 +0,0 @@ -package de.upb.cs.swt.delphi.featuredefinitions - -object FeatureListMapping { - - //Returns a list of all defined features - def featureList: List[String] = - (for {(key, des) <- featureMap if des != null} yield key).toList - - //Maps all field names onto descriptions of that field. Null fields cannot be searched in the current version - private val featureMap = Map[String, FeatureDescription]( - "ProjectPackages" -> null, - "⟨SizeOfInheritanceTree⟩" -> null, - "ProjectFields" -> null, - "LibraryMethods" -> null, - "ProjectInstructions" -> null, - "ProjectMethods" -> null, - "LibraryClassFiles" -> null, - "LibraryFields" -> null, - "LibraryPackages" -> null, - "ProjectClassFiles" -> null, - "0 FPC" -> null, - "1-3 FPC" -> null, - "4-10 FPC" -> null, - ">10 FPC" -> null, - "0 MPC" -> null, - "1-3 MPC" -> null, - "4-10 MPC" -> null, - ">10 MPC" -> null, - "1-3 CPP" -> null, - "4-10 CPP" -> null, - ">10 CPP" -> null, - "0 NOC" -> null, - "1-3 NOC" -> null, - "4-10 NOC" -> null, - ">10 NOC" -> null, - "linear methods (McCabe)" -> null, - "2-3 McCabe" -> null, - "4-10 McCabe" -> null, - ">10 McCabe" -> null, - "Designator" -> null, - "Taxonomy" -> null, - "Joiner" -> null, - "Pool" -> null, - "Function Pointer" -> null, - "Function Object" -> null, - "Cobol Like" -> null, - "Stateless" -> null, - "Common State" -> null, - "Immutable" -> null, - "Restricted Creation" -> null, - "Sampler" -> null, - "Box" -> null, - "Compound Box" -> null, - "Canopy" -> null, - "Record" -> null, - "Data Manager" -> null, - "Sink" -> null, - "Outline" -> null, - "Trait" -> null, - "State Machine" -> null, - "Pure Type" -> null, - "Augmented Type" -> null, - "Pseudo Class" -> null, - "Implementor" -> null, - "Overrider" -> null, - "Extender" -> null, - "unused private fields" -> null, - "unused package visible fields" -> null, - "unused protected fields" -> null, - "unused public fields" -> null, - "package visible fields only used by defining type" -> null, - "protected fields only used by defining type" -> null, - "public fields only used by defininig type " -> null, - "Trivial Class.forName Usage" -> null, - "Nontrivial Class.forName Usage" -> null, - "nop (opcode:0)" -> null, - "aconst_null (opcode:1)" -> null, - "iconst_m1 (opcode:2)" -> null, - "iconst_0 (opcode:3)" -> null, - "iconst_1 (opcode:4)" -> null, - "iconst_2 (opcode:5)" -> null, - "iconst_3 (opcode:6)" -> null, - "iconst_4 (opcode:7)" -> null, - "iconst_5 (opcode:8)" -> null, - "lconst_0 (opcode:9)" -> null, - "lconst_1 (opcode:10)" -> null, - "fconst_0 (opcode:11)" -> null, - "fconst_1 (opcode:12)" -> null, - "fconst_2 (opcode:13)" -> null, - "dconst_0 (opcode:14)" -> null, - "dconst_1 (opcode:15)" -> null, - "bipush (opcode:16)" -> null, - "sipush (opcode:17)" -> null, - "ldc (opcode:18)" -> null, - "ldc_w (opcode:19)" -> null, - "ldc2_w (opcode:20)" -> null, - "iload (opcode:21)" -> null, - "lload (opcode:22)" -> null, - "fload (opcode:23)" -> null, - "dload (opcode:24)" -> null, - "aload (opcode:25)" -> null, - "iload_0 (opcode:26)" -> null, - "iload_1 (opcode:27)" -> null, - "iload_2 (opcode:28)" -> null, - "iload_3 (opcode:29)" -> null, - "lload_0 (opcode:30)" -> null, - "lload_1 (opcode:31)" -> null, - "lload_2 (opcode:32)" -> null, - "lload_3 (opcode:33)" -> null, - "fload_0 (opcode:34)" -> null, - "fload_1 (opcode:35)" -> null, - "fload_2 (opcode:36)" -> null, - "fload_3 (opcode:37)" -> null, - "dload_0 (opcode:38)" -> null, - "dload_1 (opcode:39)" -> null, - "dload_2 (opcode:40)" -> null, - "dload_3 (opcode:41)" -> null, - "aload_0 (opcode:42)" -> null, - "aload_1 (opcode:43)" -> null, - "aload_2 (opcode:44)" -> null, - "aload_3 (opcode:45)" -> null, - "iaload (opcode:46)" -> null, - "laload (opcode:47)" -> null, - "faload (opcode:48)" -> null, - "daload (opcode:49)" -> null, - "aaload (opcode:50)" -> null, - "baload (opcode:51)" -> null, - "caload (opcode:52)" -> null, - "saload (opcode:53)" -> null, - "istore (opcode:54)" -> null, - "lstore (opcode:55)" -> null, - "fstore (opcode:56)" -> null, - "dstore (opcode:57)" -> null, - "astore (opcode:58)" -> null, - "istore_0 (opcode:59)" -> null, - "istore_1 (opcode:60)" -> null, - "istore_2 (opcode:61)" -> null, - "istore_3 (opcode:62)" -> null, - "lstore_0 (opcode:63)" -> null, - "lstore_1 (opcode:64)" -> null, - "lstore_2 (opcode:65)" -> null, - "lstore_3 (opcode:66)" -> null, - "fstore_0 (opcode:67)" -> null, - "fstore_1 (opcode:68)" -> null, - "fstore_2 (opcode:69)" -> null, - "fstore_3 (opcode:70)" -> null, - "dstore_0 (opcode:71)" -> null, - "dstore_1 (opcode:72)" -> null, - "dstore_2 (opcode:73)" -> null, - "dstore_3 (opcode:74)" -> null, - "astore_0 (opcode:75)" -> null, - "astore_1 (opcode:76)" -> null, - "astore_2 (opcode:77)" -> null, - "astore_3 (opcode:78)" -> null, - "iastore (opcode:79)" -> null, - "lastore (opcode:80)" -> null, - "fastore (opcode:81)" -> null, - "dastore (opcode:82)" -> null, - "aastore (opcode:83)" -> null, - "bastore (opcode:84)" -> null, - "castore (opcode:85)" -> null, - "sastore (opcode:86)" -> null, - "pop (opcode:87)" -> null, - "pop2 (opcode:88)" -> null, - "dup (opcode:89)" -> null, - "dup_x1 (opcode:90)" -> null, - "dup_x2 (opcode:91)" -> null, - "dup2 (opcode:92)" -> null, - "dup2_x1 (opcode:93)" -> null, - "dup2_x2 (opcode:94)" -> null, - "swap (opcode:95)" -> null, - "iadd (opcode:96)" -> null, - "ladd (opcode:97)" -> null, - "fadd (opcode:98)" -> null, - "dadd (opcode:99)" -> null, - "isub (opcode:100)" -> null, - "lsub (opcode:101)" -> null, - "fsub (opcode:102)" -> null, - "dsub (opcode:103)" -> null, - "imul (opcode:104)" -> null, - "lmul (opcode:105)" -> null, - "fmul (opcode:106)" -> null, - "dmul (opcode:107)" -> null, - "idiv (opcode:108)" -> null, - "ldiv (opcode:109)" -> null, - "fdiv (opcode:110)" -> null, - "ddiv (opcode:111)" -> null, - "irem (opcode:112)" -> null, - "lrem (opcode:113)" -> null, - "frem (opcode:114)" -> null, - "drem (opcode:115)" -> null, - "ineg (opcode:116)" -> null, - "lneg (opcode:117)" -> null, - "fneg (opcode:118)" -> null, - "dneg (opcode:119)" -> null, - "ishl (opcode:120)" -> null, - "lshl (opcode:121)" -> null, - "ishr (opcode:122)" -> null, - "lshr (opcode:123)" -> null, - "iushr (opcode:124)" -> null, - "lushr (opcode:125)" -> null, - "iand (opcode:126)" -> null, - "land (opcode:127)" -> null, - "ior (opcode:128)" -> null, - "lor (opcode:129)" -> null, - "ixor (opcode:130)" -> null, - "lxor (opcode:131)" -> null, - "iinc (opcode:132)" -> null, - "i2l (opcode:133)" -> null, - "i2f (opcode:134)" -> null, - "i2d (opcode:135)" -> null, - "l2i (opcode:136)" -> null, - "l2f (opcode:137)" -> null, - "l2d (opcode:138)" -> null, - "f2i (opcode:139)" -> null, - "f2l (opcode:140)" -> null, - "f2d (opcode:141)" -> null, - "d2i (opcode:142)" -> null, - "d2l (opcode:143)" -> null, - "d2f (opcode:144)" -> null, - "i2b (opcode:145)" -> null, - "i2c (opcode:146)" -> null, - "i2s (opcode:147)" -> null, - "lcmp (opcode:148)" -> null, - "fcmpl (opcode:149)" -> null, - "fcmpg (opcode:150)" -> null, - "dcmpl (opcode:151)" -> null, - "dcmpg (opcode:152)" -> null, - "ifeq (opcode:153)" -> null, - "ifne (opcode:154)" -> null, - "iflt (opcode:155)" -> null, - "ifge (opcode:156)" -> null, - "ifgt (opcode:157)" -> null, - "ifle (opcode:158)" -> null, - "if_icmpeq (opcode:159)" -> null, - "if_icmpne (opcode:160)" -> null, - "if_icmplt (opcode:161)" -> null, - "if_icmpge (opcode:162)" -> null, - "if_icmpgt (opcode:163)" -> null, - "if_icmple (opcode:164)" -> null, - "if_acmpeq (opcode:165)" -> null, - "if_acmpne (opcode:166)" -> null, - "goto (opcode:167)" -> null, - "jsr (opcode:168)" -> null, - "ret (opcode:169)" -> null, - "tableswitch (opcode:170)" -> null, - "lookupswitch (opcode:171)" -> null, - "ireturn (opcode:172)" -> null, - "lreturn (opcode:173)" -> null, - "freturn (opcode:174)" -> null, - "dreturn (opcode:175)" -> null, - "areturn (opcode:176)" -> null, - "return (opcode:177)" -> null, - "getstatic (opcode:178)" -> null, - "putstatic (opcode:179)" -> null, - "getfield (opcode:180)" -> null, - "putfield (opcode:181)" -> null, - "invokevirtual (opcode:182)" -> null, - "invokespecial (opcode:183)" -> null, - "invokestatic (opcode:184)" -> null, - "invokeinterface (opcode:185)" -> null, - "invokedynamic (opcode:186)" -> null, - "new (opcode:187)" -> null, - "newarray (opcode:188)" -> null, - "anewarray (opcode:189)" -> null, - "arraylength (opcode:190)" -> null, - "athrow (opcode:191)" -> null, - "checkcast (opcode:192)" -> null, - "instanceof (opcode:193)" -> null, - "monitorenter (opcode:194)" -> null, - "monitorexit (opcode:195)" -> null, - "wide (opcode:196)" -> null, - "multianewarray (opcode:197)" -> null, - "ifnull (opcode:198)" -> null, - "ifnonnull (opcode:199)" -> null, - "goto_w (opcode:200)" -> null, - "jsr_w (opcode:201)" -> null, - "Self-recursive Data Structure" -> null, - "Mutually-recursive Data Structure 2 Types" -> null, - "Mutually-recursive Data Structure 3 Types" -> null, - "Mutually-recursive Data Structure 4 Types" -> null, - "Mutually-recursive Data Structure more than 4 Types" -> null, - "Never Returns Normally" -> null, - "Method with Infinite Loop" -> null, - "Class File With Source Attribute" -> null, - "Method With Line Number Table" -> null, - "Method With Local Variable Table" -> null, - "Method With Local Variable Type Table" -> null, - "FanOut - Category 1" -> null, - "FanOut - Category 2" -> null, - "FanOut - Category 3" -> null, - "FanOut - Category 4" -> null, - "FanOut - Category 5" -> null, - "FanOut - Category 6" -> null, - "FanIn - Category 1" -> null, - "FanIn - Category 2" -> null, - "FanIn - Category 3" -> null, - "FanIn - Category 4" -> null, - "FanIn - Category 5" -> null, - "FanIn - Category 6" -> null, - "FanIn/FanOut - Category 1" -> null, - "FanIn/FanOut - Category 2" -> null, - "FanIn/FanOut - Category 3" -> null, - "FanIn/FanOut - Category 4" -> null, - "FanIn/FanOut - Category 5" -> null, - "FanIn/FanOut - Category 6" -> null, - "custom ClassLoader implementation" -> null, - "Retrieving the SystemClassLoader" -> null, - "Retrieving some ClassLoader" -> null, - "define new classes/packages" -> null, - "accessing resources" -> null, - "javax.crypto.Cipher getInstance" -> null, - "using SecureRandom" -> null, - "using MessageDigest" -> null, - "using Signature" -> null, - "using Mac" -> null, - "cryptographic key handling" -> null, - "using KeyStore" -> null, - "using Certificates" -> null, - "native methods" -> null, - "synthetic methods" -> null, - "bridge methods" -> null, - "synchronized methods" -> null, - "varargs methods" -> null, - "static initializers" -> null, - "static methods (not including static initializers)" -> null, - "constructors" -> null, - "instance methods" -> null, - "java.lang.Class forName" -> null, - "reflective instance creation" -> null, - "reflective field write" -> null, - "reflective field read" -> null, - "makes fields accessible" -> null, - "makes methods or constructors accessible" -> null, - "makes an AccessibleObject accessible (exact type unknown)" -> null, - "java.lang.reflect.Method Object invoke(Object, Object[])" -> null, - "java.lang.invoke.MethodHandles lookup" -> null, - "java.lang.invoke.MethodHandles publicLookup" -> null, - "method handle invocation" -> null, - "java.lang.reflect.Proxy newProxyInstance" -> null, - "Process" -> null, - "JVM exit" -> null, - "Native Libraries" -> null, - "java.lang.System getSecurityManager" -> null, - "java.lang.System setSecurityManager" -> null, - "Environment" -> null, - "Sound" -> null, - "Network sockets" -> null, - "Object-based Thread Notification" -> null, - "Usage of Thread API" -> null, - "Usage of ThreadGroup API" -> null, - "sun.misc.Unsafe sun.misc.Unsafe getUnsafe()" -> null, - "Unsafe - Alloc" -> null, - "Unsafe - Array" -> null, - "Unsafe - compareAndSwap" -> null, - "Unsafe - Class" -> null, - "Unsafe - Fence" -> null, - "Unsafe - Fetch & Add" -> null, - "Unsafe - Heap" -> null, - "Unsafe - Heap Get" -> null, - "Unsafe - Heap Put" -> null, - "Misc" -> null, - "Unsafe - Monitor" -> null, - "Unsafe - Off-Heap" -> null, - "Unsafe - Offset" -> null, - "Unsafe - Ordered Put" -> null, - "Unsafe - Park" -> null, - "Unsafe - Throw" -> null, - "Unsafe - Volatile Get" -> null, - "Unsafe - Volatile Put" -> null, - "java.sql.DriverManager getConnection" -> null, - "java.sql.Connection rollback" -> null, - "creation and execution of java.sql.Statement" -> null, - "creation and execution of java.sql.PreparedStatement" -> null, - "creation and execution of java.sql.CallableStatement" -> null, - "class file retransformation" -> null, - "instrumenting native methods" -> null, - "appending class loader search" -> null, - "retrieve classes information" -> null, - "(concrete) classes" -> null, - "abstract classes" -> null, - "annotations" -> null, - "enumerations" -> null, - "marker interfaces" -> null, - "simple functional interfaces (single abstract method (SAM) interface)" -> null, - "non-functional interface with default methods (Java >8)" -> null, - "non-functional interface with static methods (Java >8)" -> null, - "(standard) interface" -> null, - "module (Java >9)" -> null, - "Very Small Inheritance Tree" -> null, - "Small Inheritance Tree" -> null, - "Medium Inheritance Tree" -> null, - "Large Inheritance Tree" -> null, - "Very Large Inheritance Tree" -> null, - "Huge Inheritance Tree" -> null, - "Size of the Inheritance Tree Unknown" -> null, - "Class File JDK 1.1 (JDK 1.0.2)" -> null, - "Class File Java 5" -> null, - "Class File Java 6" -> null, - "Class File Java 7" -> null, - "Class File Java 8" -> null, - "Class File Java 9" -> null - ) -} diff --git a/src/main/scala/de/upb/cs/swt/delphi/instancemanagement/Instance.scala b/src/main/scala/de/upb/cs/swt/delphi/instancemanagement/Instance.scala new file mode 100644 index 0000000..db29ab8 --- /dev/null +++ b/src/main/scala/de/upb/cs/swt/delphi/instancemanagement/Instance.scala @@ -0,0 +1,86 @@ +// Copyright (C) 2018 The Delphi Team. +// See the LICENCE file distributed with this work for additional +// information regarding copyright ownership. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package de.upb.cs.swt.delphi.instancemanagement +import akka.http.scaladsl.marshallers.sprayjson.SprayJsonSupport +import spray.json.{DefaultJsonProtocol, DeserializationException, JsString, JsValue, JsonFormat} + +trait JsonSupport extends SprayJsonSupport with DefaultJsonProtocol { + + implicit val componentTypeFormat : JsonFormat[InstanceEnums.ComponentType] = new JsonFormat[InstanceEnums.ComponentType] { + + def write(compType : InstanceEnums.ComponentType) = JsString(compType.toString) + + def read(value: JsValue) : InstanceEnums.ComponentType = value match { + case JsString(s) => s match { + case "Crawler" => InstanceEnums.ComponentType.Crawler + case "WebApi" => InstanceEnums.ComponentType.WebApi + case "WebApp" => InstanceEnums.ComponentType.WebApp + case "DelphiManagement" => InstanceEnums.ComponentType.DelphiManagement + case "ElasticSearch" => InstanceEnums.ComponentType.ElasticSearch + case x => throw DeserializationException(s"Unexpected string value $x for component type.") + } + case y => throw DeserializationException(s"Unexpected type $y while deserializing component type.") + } + } + + implicit val stateFormat : JsonFormat[InstanceEnums.State] = new JsonFormat[InstanceEnums.State] { + + def write(compType : InstanceEnums.State) = JsString(compType.toString) + + def read(value: JsValue) : InstanceEnums.State = value match { + case JsString(s) => s match { + case "Running" => InstanceEnums.InstanceState.Running + case "Stopped" => InstanceEnums.InstanceState.Stopped + case "Failed" => InstanceEnums.InstanceState.Failed + case "Paused" => InstanceEnums.InstanceState.Paused + case "NotReachable" => InstanceEnums.InstanceState.NotReachable + case x => throw DeserializationException(s"Unexpected string value $x for instance state.") + } + case y => throw DeserializationException(s"Unexpected type $y while deserializing instance state.") + } + } + + implicit val instanceFormat : JsonFormat[Instance] = jsonFormat7(Instance) +} + +final case class Instance ( + id: Option[Long], + host: String, + portNumber: Long, + name: String, + componentType: InstanceEnums.ComponentType, + dockerId: Option[String], + instanceState: InstanceEnums.State + ) +object InstanceEnums { + type ComponentType = ComponentType.Value + object ComponentType extends Enumeration { + val Crawler : Value = Value("Crawler") + val ElasticSearch : Value = Value("ElasticSearch") + val WebApi : Value = Value("WebApi") + val WebApp : Value = Value("WebApp") + val DelphiManagement : Value = Value("DelphiManagement") + } + type State = InstanceState.Value + object InstanceState extends Enumeration { + val Running : Value = Value("Running") + val Paused : Value = Value("Paused") + val Stopped : Value = Value("Stopped") + val Failed : Value = Value("Failed") + val NotReachable : Value = Value("NotReachable") + } +} diff --git a/src/main/scala/de/upb/cs/swt/delphi/instancemanagement/InstanceRegistry.scala b/src/main/scala/de/upb/cs/swt/delphi/instancemanagement/InstanceRegistry.scala new file mode 100644 index 0000000..f01d42d --- /dev/null +++ b/src/main/scala/de/upb/cs/swt/delphi/instancemanagement/InstanceRegistry.scala @@ -0,0 +1,326 @@ +// Copyright (C) 2018 The Delphi Team. +// See the LICENCE file distributed with this work for additional +// information regarding copyright ownership. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package de.upb.cs.swt.delphi.instancemanagement + +import java.net.InetAddress + +import akka.actor.ActorSystem +import akka.http.scaladsl.Http +import akka.http.scaladsl.model._ +import akka.http.scaladsl.unmarshalling.Unmarshal +import akka.stream.ActorMaterializer +import akka.util.ByteString +import de.upb.cs.swt.delphi.instancemanagement.InstanceEnums.{ComponentType, InstanceState} +import de.upb.cs.swt.delphi.webapi.{AppLogging, Configuration, Server} + +import scala.concurrent.{Await, ExecutionContext, Future} +import scala.concurrent.duration._ +import scala.util.{Failure, Success, Try} +import spray.json._ + +object InstanceRegistry extends JsonSupport with AppLogging +{ + + implicit val system : ActorSystem = Server.system + implicit val ec : ExecutionContext = system.dispatcher + implicit val materializer : ActorMaterializer = Server.materializer + + lazy val instanceIdFromEnv : Option[Long] = Try[Long](sys.env("INSTANCE_ID").toLong).toOption + + + def handleInstanceStart(configuration: Configuration) : Option[Long] = { + instanceIdFromEnv match { + case Some(id) => + reportStart(configuration) match { + case Success(_) => Some(id) + case Failure(_) => None + } + case None => + register(configuration) match { + case Success(id) => Some(id) + case Failure(_) => None + } + } + } + + def handleInstanceStop(configuration: Configuration) : Try[Unit] = { + if(instanceIdFromEnv.isDefined) { + reportStop(configuration) + } else { + deregister(configuration) + } + } + + def handleInstanceFailure(configuration: Configuration) : Try[Unit] = { + if(instanceIdFromEnv.isDefined) { + reportFailure(configuration) + } else { + deregister(configuration) + } + } + + def reportStart(configuration: Configuration) : Try[Unit] = executeReportOperation(configuration, ReportOperationType.Start) + + def reportStop(configuration: Configuration) : Try[Unit] = { + if(configuration.usingInstanceRegistry) { + executeReportOperation(configuration, ReportOperationType.Stop) + } else { + Failure(new RuntimeException("Cannot report stop, no instance registry available.")) + } + } + + def reportFailure(configuration: Configuration) : Try[Unit] = { + if(configuration.usingInstanceRegistry){ + executeReportOperation(configuration, ReportOperationType.Failure) + } else { + Failure(new RuntimeException("Cannot report failure, no instance registry available.")) + } + } + + private def executeReportOperation(configuration: Configuration, operationType: ReportOperationType.Value) : Try[Unit] = { + instanceIdFromEnv match { + case Some(id) => + val request = HttpRequest( + method = HttpMethods.POST, + configuration.instanceRegistryUri + ReportOperationType.toOperationUriString(operationType, id)) + + Await.result(Http(system).singleRequest(request) map {response => + if(response.status == StatusCodes.OK){ + log.info(s"Successfully reported ${operationType.toString} to Instance Registry.") + Success() + } + else { + log.warning(s"Failed to report ${operationType.toString} to Instance Registry, server returned ${response.status}") + Failure(new RuntimeException(s"Failed to report ${operationType.toString} to Instance Registry, server returned ${response.status}")) + } + + } recover {case ex => + log.warning(s"Failed to report ${operationType.toString} to Instance Registry, exception: $ex") + Failure(new RuntimeException(s"Failed to report ${operationType.toString} to Instance Registry, exception: $ex")) + }, Duration.Inf) + case None => + log.warning(s"Cannot report ${operationType.toString} to Instance Registry, no instance id is present in env var 'INSTANCE_ID'.") + Failure(new RuntimeException(s"Cannot report ${operationType.toString} to Instance Registry, no instance id is present in env var 'INSTANCE_ID'.")) + } + } + def register(configuration: Configuration) :Try[Long] = { + val instance = createInstance(None,configuration.bindPort, configuration.instanceName) + + Await.result(postInstance(instance, configuration.instanceRegistryUri + "/register") map {response => + if(response.status == StatusCodes.OK){ + Await.result(Unmarshal(response.entity).to[String] map { assignedID => + val id = assignedID.toLong + log.info(s"Successfully registered at Instance Registry, got ID $id.") + Success(id) + } recover { case ex => + log.warning(s"Failed to read assigned ID from Instance Registry, exception: $ex") + Failure(ex) + }, Duration.Inf) + } + else { + val statuscode = response.status + log.warning(s"Failed to register at Instance Registry, server returned $statuscode") + Failure(new RuntimeException(s"Failed to register at Instance Registry, server returned $statuscode")) + } + + } recover {case ex => + log.warning(s"Failed to register at Instance Registry, exception: $ex") + Failure(ex) + }, Duration.Inf) + } + + def retrieveElasticSearchInstance(configuration: Configuration) : Try[Instance] = { + if(!configuration.usingInstanceRegistry) { + Failure(new RuntimeException("Cannot get ElasticSearch instance from Instance Registry, no Instance Registry available.")) + } else { + val request = HttpRequest(method = HttpMethods.GET, configuration.instanceRegistryUri + "/matchingInstance?ComponentType=ElasticSearch") + + Await.result(Http(system).singleRequest(request) map {response => + response.status match { + case StatusCodes.OK => + try { + val instanceString : String = Await.result(response.entity.dataBytes.runFold(ByteString(""))(_ ++ _).map(_.utf8String), 5 seconds) + val esInstance = instanceString.parseJson.convertTo[Instance](instanceFormat) + val elasticIP = esInstance.host + log.info(s"Instance Registry assigned ElasticSearch instance at $elasticIP") + Success(esInstance) + } catch { + case px: spray.json.JsonParser.ParsingException => + log.warning(s"Failed to read response from Instance Registry, exception: $px") + Failure(px) + } + case StatusCodes.NotFound => + log.warning(s"No matching instance of type 'ElasticSearch' is present at the instance registry.") + Failure(new RuntimeException(s"Instance Registry did not contain matching instance, server returned ${StatusCodes.NotFound}")) + case _ => + val status = response.status + log.warning(s"Failed to read matching instance from Instance Registry, server returned $status") + Failure(new RuntimeException(s"Failed to read matching instance from Instance Registry, server returned $status")) + } + } recover { case ex => + log.warning(s"Failed to request ElasticSearch instance from Instance Registry, exception: $ex ") + Failure(ex) + }, Duration.Inf) + } + } + + def sendMatchingResult(isElasticSearchReachable : Boolean, configuration: Configuration) : Try[Unit] = { + + if(!configuration.usingInstanceRegistry) { + Failure(new RuntimeException("Cannot post matching result to Instance Registry, no Instance Registry available.")) + } else { + if(configuration.elasticsearchInstance.id.isEmpty) { + Failure(new RuntimeException("Cannot post matching result to Instance Registry, assigned ElasticSearch instance has no ID.")) + } else { + val idToPost = configuration.elasticsearchInstance.id.getOrElse(-1L) + val request = HttpRequest( + method = HttpMethods.POST, + configuration.instanceRegistryUri + s"/matchingResult?Id=$idToPost&MatchingSuccessful=$isElasticSearchReachable") + + Await.result(Http(system).singleRequest(request) map {response => + val status=response.status + if(response.status == StatusCodes.OK){ + log.info(s"Successfully posted matching result to Instance Registry.") + Success() + } + else { + val statuscode = response.status + log.warning(s"Failed to post matching result to Instance Registry, server returned $statuscode") + Failure(new RuntimeException(s"Failed to post matching result to Instance Registry, server returned $statuscode")) + } + + } recover {case ex => + log.warning(s"Failed to post matching result to Instance Registry, exception: $ex") + Failure(new RuntimeException(s"Failed to post matching result tot Instance Registry, exception: $ex")) + }, Duration.Inf) + } + } + + } + + def deregister(configuration: Configuration) : Try[Unit] = { + if(!configuration.usingInstanceRegistry){ + Failure(new RuntimeException("Cannot deregister from Instance Registry, no Instance Registry available.")) + } else { + val id : Long = configuration.assignedID.getOrElse(-1L) + + val request = HttpRequest(method = HttpMethods.POST, configuration.instanceRegistryUri + s"/deregister?Id=$id") + + Await.result(Http(system).singleRequest(request) map {response => + if(response.status == StatusCodes.OK){ + log.info("Successfully deregistered from Instance Registry.") + Success() + } + else { + val statuscode = response.status + log.warning(s"Failed to deregister from Instance Registry, server returned $statuscode") + Failure(new RuntimeException(s"Failed to deregister from Instance Registry, server returned $statuscode")) + } + + } recover {case ex => + log.warning(s"Failed to deregister to Instance Registry, exception: $ex") + Failure(ex) + }, Duration.Inf) + } + } + + def postInstance(instance : Instance, uri: String) () : Future[HttpResponse] = { + try { + val request = HttpRequest(method = HttpMethods.POST, uri = uri, entity = instance.toJson(instanceFormat).toString()) + Http(system).singleRequest(request) + } catch { + case dx: DeserializationException => + log.warning(s"Failed to deregister to Instance Registry, exception: $dx") + Future.failed(dx) + } + } + + + private def createInstance(id: Option[Long], controlPort : Int, name : String) : Instance = + Instance(id, InetAddress.getLocalHost.getHostAddress, + controlPort, name, ComponentType.WebApi, None, InstanceState.Running) + + def reportStart(id: String, configuration: Configuration):Try[ResponseEntity] ={ + val request = HttpRequest(method = HttpMethods.GET, configuration.instanceRegistryUri + "/reportStart") + Await.result(Http(system).singleRequest(request) map {response => + if(response.status == StatusCodes.OK){ + Success(response.entity) + } + else { + val statuscode = response.status + log.warning(s"Failed to perform reportStart, server returned $statuscode") + Failure(new RuntimeException(s"Failed to perform reportStart, server returned $statuscode")) + } + } recover {case ex => + log.warning(s"Failed to perform reportStart, exception: $ex") + Failure(new RuntimeException(s"Failed to perform reportStart, server returned, exception: $ex")) + }, Duration.Inf) + } + + def reportFailure(id: String, configuration: Configuration):Try[ResponseEntity] = { + + val request = HttpRequest(method = HttpMethods.GET, configuration.instanceRegistryUri + "/reportFailure") + Await.result(Http(system).singleRequest(request) map {response => + if(response.status == StatusCodes.OK){ + Success(response.entity) + } + else { + val statuscode = response.status + log.warning(s"Failed to perform reportFailure, server returned $statuscode") + Failure(new RuntimeException(s"Failed to perform reportFailure, server returned $statuscode")) + } + } recover {case ex => + log.warning(s"Failed to perform reportFailure, server returned, exception: $ex") + Failure(new RuntimeException(s"Failed to perform reportFailure, server returned, exception: $ex")) + }, Duration.Inf) + } + + def reportStop(id: String, configuration: Configuration):Try[ResponseEntity] = { + + val request = HttpRequest(method = HttpMethods.GET, configuration.instanceRegistryUri + "/reportStop") + Await.result(Http(system).singleRequest(request) map {response => + if(response.status == StatusCodes.OK){ + Success(response.entity) + } + else { + val statuscode = response.status + log.warning(s"Failed to perform reportStop, server returned $statuscode") + Failure(new RuntimeException(s"Failed to perform reportStop, server returned $statuscode")) + } + } recover {case ex => + log.warning(s"Failed to perform reportStop, server returned, exception: $ex") + Failure(new RuntimeException(s"Failed to perform reportStop, server returned, exception: $ex")) + }, Duration.Inf) + } + + object ReportOperationType extends Enumeration { + val Start : Value = Value("Start") + val Stop : Value = Value("Stop") + val Failure : Value = Value("Failure") + + def toOperationUriString(operation: ReportOperationType.Value, id: Long) : String = { + operation match { + case Start => + s"/reportStart?Id=$id" + case Stop => + s"/reportStop?Id=$id" + case _ => + s"/reportFailure?Id=$id" + } + } + } +} \ No newline at end of file diff --git a/src/main/scala/de/upb/cs/swt/delphi/querylanguage/AST.scala b/src/main/scala/de/upb/cs/swt/delphi/querylanguage/AST.scala deleted file mode 100644 index 57b8abe..0000000 --- a/src/main/scala/de/upb/cs/swt/delphi/querylanguage/AST.scala +++ /dev/null @@ -1,19 +0,0 @@ -package de.upb.cs.swt.delphi.querylanguage - -trait CombinatorialExpr - -case class AndExpr(Left: CombinatorialExpr, Right: CombinatorialExpr) extends CombinatorialExpr -case class OrExpr(Left: CombinatorialExpr, Right: CombinatorialExpr) extends CombinatorialExpr -case class NotExpr(Expr: CombinatorialExpr) extends CombinatorialExpr -case class XorExpr(Left: CombinatorialExpr, Right: CombinatorialExpr) extends CombinatorialExpr - -trait SingularConditionExpr extends CombinatorialExpr - -case class EqualExpr(Left: String, Right: String) extends SingularConditionExpr -case class NotEqualExpr(Left: String, Right: String) extends SingularConditionExpr -case class GreaterThanExpr(Left: String, Right: String) extends SingularConditionExpr -case class GreaterOrEqualExpr(Left: String, Right: String) extends SingularConditionExpr -case class LessThanExpr(Left: String, Right: String) extends SingularConditionExpr -case class LessOrEqualExpr(Left: String, Right: String) extends SingularConditionExpr -case class LikeExpr(Left: String, Right: String) extends SingularConditionExpr -case class TrueExpr(Expr: String) extends SingularConditionExpr diff --git a/src/main/scala/de/upb/cs/swt/delphi/querylanguage/Syntax.scala b/src/main/scala/de/upb/cs/swt/delphi/querylanguage/Syntax.scala deleted file mode 100644 index 1779b56..0000000 --- a/src/main/scala/de/upb/cs/swt/delphi/querylanguage/Syntax.scala +++ /dev/null @@ -1,57 +0,0 @@ -package de.upb.cs.swt.delphi.querylanguage - -import org.parboiled2.{CharPredicate, Parser, ParserInput, Rule1} - -/** - * The syntax definition and parser for the Delphi QL. - * - * @author Lisa Nguyen Quang Do - * @author Ben Hermann - * - */ -class Syntax(val input : ParserInput) extends Parser { - - def QueryRule = rule { - CombinatorialRule ~ EOI - } - - // Combinatorial rules. - def CombinatorialRule : Rule1[CombinatorialExpr] = rule { - OrOrElseRule | NotRule - } - def OrOrElseRule = rule { - AndOrElseRule ~ zeroOrMore("||" ~ AndOrElseRule ~> OrExpr) - } - def AndOrElseRule = rule { - XorOrElseRule ~ zeroOrMore("&&" ~ XorOrElseRule ~> AndExpr) - } - def XorOrElseRule = rule { - Factor ~ zeroOrMore("%%" ~ Factor ~> XorExpr) - } - - // Handling parentheses. - def Factor : Rule1[CombinatorialExpr] = rule { - Parentheses | SingularConditionRule | NotRule - } - def Parentheses = rule { '(' ~ CombinatorialRule ~ ')' } - def NotRule = rule { '!' ~ Factor ~> NotExpr } - - // Singular conditions. - def SingularConditionRule = rule { - EqualRule | NotEqualRule | GreaterThanRule | GreaterOrEqual | - LessThan | LessOrEqual | Like | True - } - def EqualRule = rule { Literal ~ "=" ~ Literal ~> EqualExpr } - def NotEqualRule = rule { Literal ~ "!=" ~ Literal ~> NotEqualExpr } - def GreaterThanRule = rule { Literal ~ ">" ~ Literal ~> GreaterThanExpr } - def GreaterOrEqual = rule { Literal ~ ">=" ~ Literal ~> GreaterOrEqualExpr } - def LessThan = rule { Literal ~ "<" ~ Literal ~> LessThanExpr } - def LessOrEqual = rule { Literal ~ "<=" ~ Literal ~> LessOrEqualExpr } - def Like = rule { Literal ~ "%" ~ Literal ~> LikeExpr } - def True = rule { Literal ~> TrueExpr } - - // Literals - def Literal = rule { capture(oneOrMore(CharPredicate.AlphaNum)) ~> (_.toString) } -} - - diff --git a/src/main/scala/de/upb/cs/swt/delphi/webapi/AppLogging.scala b/src/main/scala/de/upb/cs/swt/delphi/webapi/AppLogging.scala index 673d2f2..514ec42 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/webapi/AppLogging.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/webapi/AppLogging.scala @@ -1,3 +1,19 @@ +// Copyright (C) 2018 The Delphi Team. +// See the LICENCE file distributed with this work for additional +// information regarding copyright ownership. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package de.upb.cs.swt.delphi.webapi import akka.actor.{ActorSystem, ExtendedActorSystem} diff --git a/src/main/scala/de/upb/cs/swt/delphi/webapi/Configuration.scala b/src/main/scala/de/upb/cs/swt/delphi/webapi/Configuration.scala index 53ed334..3874191 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/webapi/Configuration.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/webapi/Configuration.scala @@ -1,20 +1,90 @@ +// Copyright (C) 2018 The Delphi Team. +// See the LICENCE file distributed with this work for additional +// information regarding copyright ownership. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package de.upb.cs.swt.delphi.webapi -import com.sksamuel.elastic4s.{ElasticsearchClientUri, IndexAndType} import com.sksamuel.elastic4s.http.ElasticDsl._ +import com.sksamuel.elastic4s.{ElasticsearchClientUri, Index, IndexAndType} +import de.upb.cs.swt.delphi.instancemanagement.InstanceEnums.{ComponentType, InstanceState} +import de.upb.cs.swt.delphi.instancemanagement.{Instance, InstanceRegistry} + +import scala.util.{Failure, Success, Try} /** * @author Ben Hermann */ -class Configuration( //Server and Elasticsearch configuration - val bindHost: String = "0.0.0.0", - val bindPort: Int = 8080, - val elasticsearchClientUri: ElasticsearchClientUri = ElasticsearchClientUri( - sys.env.getOrElse("DELPHI_ELASTIC_URI", "elasticsearch://localhost:9200")), - val esProjectIndex: IndexAndType = "delphi" / "project", - - //Actor system configuration - val elasticActorPoolSize: Int = 8 +class Configuration( //Server and Elasticsearch configuration + val bindHost: String = "0.0.0.0", + val bindPort: Int = 8080, + val esIndex: String = "delphi", + val esType: String = "project", + + //Actor system configuration + val elasticActorPoolSize: Int = 8 ) { + lazy val esProjectIndex: IndexAndType = esIndex / esType + lazy val elasticsearchClientUri: ElasticsearchClientUri = ElasticsearchClientUri( + elasticsearchInstance.host + ":" + elasticsearchInstance.portNumber) + + lazy val elasticsearchInstance: Instance = InstanceRegistry.retrieveElasticSearchInstance(configuration = this) match { + case Success(instance) => instance + case Failure(_) => Instance( + None, + fallbackElasticSearchHost, + fallbackElasticSearchPort, + "Default ElasticSearch instance", + ComponentType.ElasticSearch, + None, + InstanceState.Running) + } + val defaultElasticSearchPort: Int = 9200 + val defaultElasticSearchHost: String = "elasticsearch://localhost" + val instanceName = "MyWebApiInstance" + val instanceRegistryUri: String = sys.env.getOrElse("DELPHI_IR_URI", "http://localhost:8087") + lazy val usingInstanceRegistry: Boolean = assignedID match { + case Some(_) => true + case None => false + } + lazy val assignedID: Option[Long] = InstanceRegistry.handleInstanceStart(configuration = this) + + lazy val fallbackElasticSearchPort: Int = sys.env.get("DELPHI_ELASTIC_URI") match { + case Some(hostString) => if (hostString.count(c => c == ':') == 3) { + Try(hostString.split(":")(2).toInt) match { + case Success(port) => port + case Failure(_) => defaultElasticSearchPort + } + } else { + defaultElasticSearchPort + } + case None => defaultElasticSearchPort + } + + lazy val fallbackElasticSearchHost: String = sys.env.get("DELPHI_ELASTIC_URI") match { + case Some(hostString) => + if (hostString.count(c => c == ':') == 2) { + hostString.substring(0, hostString.lastIndexOf(":")) + } else { + defaultElasticSearchHost + } + case None => defaultElasticSearchHost + + } + lazy val instanceId: Option[Long] = InstanceRegistry.handleInstanceStart(configuration = this) + } + + diff --git a/src/main/scala/de/upb/cs/swt/delphi/webapi/ElasticActor.scala b/src/main/scala/de/upb/cs/swt/delphi/webapi/ElasticActor.scala deleted file mode 100644 index c4d4dcd..0000000 --- a/src/main/scala/de/upb/cs/swt/delphi/webapi/ElasticActor.scala +++ /dev/null @@ -1,46 +0,0 @@ -package de.upb.cs.swt.delphi.webapi - -import akka.actor.{Actor, ActorLogging, Props} -import com.sksamuel.elastic4s.IndexAndType -import com.sksamuel.elastic4s.http.{ElasticClient, RequestFailure, RequestSuccess} -import com.sksamuel.elastic4s.http.ElasticDsl._ -import de.upb.cs.swt.delphi.webapi.ElasticActorManager.{Enqueue, Retrieve} - -import scala.concurrent.ExecutionContext -import scala.concurrent.duration._ - -class ElasticActor(configuration: Configuration, index: IndexAndType) extends Actor with ActorLogging{ - - implicit val executionContext: ExecutionContext = context.system.dispatchers.lookup("elasticsearch-handling-dispatcher") - val client = ElasticClient(configuration.elasticsearchClientUri) - - override def preStart(): Unit = log.info("Search actor started") - override def postStop(): Unit = log.info("Search actor shut down") - context.setReceiveTimeout(2 seconds) - - override def receive = { - case Enqueue(id) => getSource(id) - case Retrieve(id) => getSource(id) - } - - private def getSource(id: String) = { - log.info("Executing get on entry {}", id) - def queryResponse = client.execute{ - log.info(s"Got retrieve request for $id.") - searchWithType(index) query must ( - matchQuery("name", s"http://repo1.maven.org/maven2/:$id") - ) - }.await - - val source = queryResponse match { - case results: RequestSuccess[_] => results.body.get - case failure: RequestFailure => Option.empty - } - sender().tell(source, context.self) - } -} - -object ElasticActor{ - def props(configuration: Configuration, index: IndexAndType) : Props = Props(new ElasticActor(configuration, index)) - .withMailbox("es-priority-mailbox") -} diff --git a/src/main/scala/de/upb/cs/swt/delphi/webapi/ElasticActorManager.scala b/src/main/scala/de/upb/cs/swt/delphi/webapi/ElasticActorManager.scala deleted file mode 100644 index aa8f6fc..0000000 --- a/src/main/scala/de/upb/cs/swt/delphi/webapi/ElasticActorManager.scala +++ /dev/null @@ -1,44 +0,0 @@ -package de.upb.cs.swt.delphi.webapi - -import akka.actor.{Actor, ActorLogging, Props, Terminated} -import akka.routing.{ActorRefRoutee, RoundRobinRoutingLogic, Router} -import de.upb.cs.swt.delphi.webapi.ElasticActorManager.ElasticMessage - -class ElasticActorManager(configuration: Configuration) extends Actor with ActorLogging{ - - private val index = configuration.esProjectIndex - private var elasticRouter = { - val routees = Vector.fill(configuration.elasticActorPoolSize) { - val r = context.actorOf(ElasticActor.props(configuration, index)) - context watch r - ActorRefRoutee(r) - } - Router(RoundRobinRoutingLogic(), routees) - } - - override def preStart(): Unit = log.info("Actor manager started") - override def postStop(): Unit = log.info("Actor manager shut down") - - override def receive = { - case em: ElasticMessage => { - log.info("Forwarding request {} to ElasticActor", em) - elasticRouter.route(em, sender()) - } - case Terminated(id) => { - elasticRouter.removeRoutee(id) - val r = context.actorOf(ElasticActor.props(configuration, index)) - context watch r - elasticRouter = elasticRouter.addRoutee(r) - } - } -} - -object ElasticActorManager{ - def props(configuration: Configuration) : Props = Props(new ElasticActorManager(configuration)) - .withMailbox("es-priority-mailbox") - - sealed trait ElasticMessage - - final case class Retrieve(id: String) extends ElasticMessage - final case class Enqueue(id: String) extends ElasticMessage -} \ No newline at end of file diff --git a/src/main/scala/de/upb/cs/swt/delphi/webapi/ElasticPriorityMailbox.scala b/src/main/scala/de/upb/cs/swt/delphi/webapi/ElasticPriorityMailbox.scala deleted file mode 100644 index 5600c01..0000000 --- a/src/main/scala/de/upb/cs/swt/delphi/webapi/ElasticPriorityMailbox.scala +++ /dev/null @@ -1,14 +0,0 @@ -package de.upb.cs.swt.delphi.webapi - -import akka.actor.ActorSystem -import akka.dispatch.{PriorityGenerator, UnboundedStablePriorityMailbox} -import de.upb.cs.swt.delphi.webapi.ElasticActorManager.{Enqueue, Retrieve} -import com.typesafe.config.Config - -class ElasticPriorityMailbox (settings: ActorSystem.Settings, config: Config) - extends UnboundedStablePriorityMailbox( - PriorityGenerator{ - case Retrieve(_) => 5 - case Enqueue(_) => 1 - case _ => 2 - }) diff --git a/src/main/scala/de/upb/cs/swt/delphi/webapi/ElasticRequestLimiter.scala b/src/main/scala/de/upb/cs/swt/delphi/webapi/ElasticRequestLimiter.scala deleted file mode 100644 index 4395630..0000000 --- a/src/main/scala/de/upb/cs/swt/delphi/webapi/ElasticRequestLimiter.scala +++ /dev/null @@ -1,82 +0,0 @@ -package de.upb.cs.swt.delphi.webapi - - -import akka.actor.{Actor, ActorLogging, ActorRef, Props} -import akka.actor.Timers -import akka.http.scaladsl.model.RemoteAddress -import de.upb.cs.swt.delphi.webapi.ElasticRequestLimiter._ -import de.upb.cs.swt.delphi.webapi.ElasticActorManager.ElasticMessage - -import scala.concurrent.duration._ -import scala.collection.mutable - -//Limits the number of requests any given IP can make by tracking how many requests an IP has made within a given -// window of time, and timing out any IP that exceeds a threshold by rejecting any further request for a period of time -class ElasticRequestLimiter(configuration: Configuration, nextActor: ActorRef) extends Actor with ActorLogging with Timers { - - private val window = 1 second - private val threshold = 10 - private val timeout = 2 hours - - private var recentIPs: mutable.Map[String, Int] = mutable.Map() - private var blockedIPs: mutable.Set[String] = mutable.Set() - - override def preStart(): Unit = { - log.info("Request limiter started") - timers.startPeriodicTimer(ClearTimer, ClearLogs, window) - } - override def postStop(): Unit = log.info("Request limiter shut down") - - override def receive = { - case Validate(rawIp, message) => { - val ip = rawIp.toOption.map(_.getHostAddress).getOrElse("unknown") - //First, reject IPs marked as blocked - if (blockedIPs.contains(ip)) { - rejectRequest() - } else { - //Check if this IP has made any requests recently - if (recentIPs.contains(ip)) { - //If so, increment their counter and test if they have exceeded the request threshold - recentIPs.update(ip, recentIPs(ip) + 1) - if (recentIPs(ip) > threshold) { - //If the threshold has been exceeded, mark this IP as blocked and reject it, and set up a message to unblock it after a period - blockedIPs += ip - log.info("Blocked IP {} due to exceeding request frequency threshold", ip) - timers.startSingleTimer(ForgiveTimer(ip), Forgive(ip), timeout) - rejectRequest() - } else { - //Else, forward this message - nextActor forward message - } - } else { - //Else, register their request in the map and pass it to the next actor - recentIPs += (ip -> 1) - nextActor forward message - } - } - } - case ClearLogs => - recentIPs.clear() - case Forgive(ip) => { - blockedIPs -= ip - log.info("Forgave IP {} after timeout", ip) - } - } - - //Rejects requests from blocked IPs - private def rejectRequest() = - sender() ! "Sorry, you have exceeded the limit on request frequency for unregistered users.\n" + - "As a result, you have been timed out.\n" + - "Please wait a while or register an account with us to continue using this service." -} - -object ElasticRequestLimiter{ - def props(configuration: Configuration, nextActor: ActorRef) : Props = Props(new ElasticRequestLimiter(configuration, nextActor)) - - final case class Validate(rawIp: RemoteAddress, message: ElasticMessage) - final case object ClearLogs - final case class Forgive(ip: String) - - final case object ClearTimer - final case class ForgiveTimer(ip: String) -} \ No newline at end of file diff --git a/src/main/scala/de/upb/cs/swt/delphi/webapi/FeatureQuery.scala b/src/main/scala/de/upb/cs/swt/delphi/webapi/FeatureQuery.scala new file mode 100644 index 0000000..623e64f --- /dev/null +++ b/src/main/scala/de/upb/cs/swt/delphi/webapi/FeatureQuery.scala @@ -0,0 +1,53 @@ +// Copyright (C) 2018 The Delphi Team. +// See the LICENCE file distributed with this work for additional +// information regarding copyright ownership. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package de.upb.cs.swt.delphi.webapi + +import com.sksamuel.elastic4s.http.ElasticDsl._ +import com.sksamuel.elastic4s.http.{ElasticClient, RequestSuccess} +import org.slf4j.LoggerFactory +import spray.json._ + +class FeatureQuery(configuration: Configuration) { + private val log = LoggerFactory.getLogger(this.getClass) + + lazy val featureList: Iterable[String] = { + val client = ElasticClient(configuration.elasticsearchClientUri) + val mappingRequest = client.execute { + getMapping(configuration.esProjectIndex) + }.await + + mappingRequest match { + case RequestSuccess(_, body, _, mappings) => { + val bodyJson = body.getOrElse("").parseJson.asJsObject + + bodyJson + .fields("delphi").asJsObject + .fields("mappings").asJsObject + .fields("project").asJsObject + .fields("properties").asJsObject + .fields("hermes").asJsObject + .fields("properties").asJsObject + .fields("features").asJsObject + .fields("properties").asJsObject.fields.keys + } + case _ => { + log.warn(s"Could not retrieve current feature list. Error was: $mappingRequest") + List() + } + } + } +} diff --git a/src/main/scala/de/upb/cs/swt/delphi/webapi/JsonSupport.scala b/src/main/scala/de/upb/cs/swt/delphi/webapi/JsonSupport.scala index 4d8c771..082970f 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/webapi/JsonSupport.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/webapi/JsonSupport.scala @@ -1,3 +1,19 @@ +// Copyright (C) 2018 The Delphi Team. +// See the LICENCE file distributed with this work for additional +// information regarding copyright ownership. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package de.upb.cs.swt.delphi.webapi import akka.http.scaladsl.marshallers.sprayjson.SprayJsonSupport diff --git a/src/main/scala/de/upb/cs/swt/delphi/webapi/MavenIdentifier.scala b/src/main/scala/de/upb/cs/swt/delphi/webapi/MavenIdentifier.scala new file mode 100644 index 0000000..81be0c1 --- /dev/null +++ b/src/main/scala/de/upb/cs/swt/delphi/webapi/MavenIdentifier.scala @@ -0,0 +1,62 @@ +// Copyright (C) 2018 The Delphi Team. +// See the LICENCE file distributed with this work for additional +// information regarding copyright ownership. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package de.upb.cs.swt.delphi.webapi + +import java.net.{URI, URLEncoder} +import java.nio.charset.StandardCharsets + +import de.upb.cs.swt.delphi.webapi.artifacts.Identifier + +case class MavenIdentifier(val repository: Option[String], val groupId: String, val artifactId: String, val version: Option[String]) extends Identifier { + + def toUniqueString: String = { + repository.getOrElse(MavenIdentifier.DefaultRepository) + ":" + groupId + ":" + artifactId + ":" + version.getOrElse("") + } + + override val toString: String = groupId + ":" + artifactId + ":" + version + + def toJarLocation : URI = { + constructArtifactBaseUri().resolve(encode(artifactId) + "-" + encode(version.getOrElse("")) + ".jar") + } + + def toPomLocation : URI = { + constructArtifactBaseUri().resolve(encode(artifactId) + "-" + encode(version.getOrElse("")) + ".pom") + } + + private def constructArtifactBaseUri(): URI = + new URI(repository.getOrElse(MavenIdentifier.DefaultRepository)) + .resolve(encode(groupId).replace('.', '/') + "/") + .resolve(encode(artifactId) + "/") + .resolve(encode(version.getOrElse("")) + "/") + + private def encode(input : String) : String = + URLEncoder.encode(input, StandardCharsets.UTF_8.toString()) +} + +object MavenIdentifier { + private val DefaultRepository = "http://repo1.maven.org/maven2/" + + private implicit def wrapOption[A](value : A) : Option[A] = Some(value) + + def apply(s: String): Option[MavenIdentifier] = { + val splitString: Array[String] = s.split(':') + if (splitString.length < 2 || splitString.length > 3) return None + + MavenIdentifier(None, splitString(0), splitString(1), if (splitString.length < 3) None else splitString(2)) + + } +} \ No newline at end of file diff --git a/src/main/scala/de/upb/cs/swt/delphi/webapi/RetrieveQuery.scala b/src/main/scala/de/upb/cs/swt/delphi/webapi/RetrieveQuery.scala new file mode 100644 index 0000000..6c8a942 --- /dev/null +++ b/src/main/scala/de/upb/cs/swt/delphi/webapi/RetrieveQuery.scala @@ -0,0 +1,60 @@ +// Copyright (C) 2018 The Delphi Team. +// See the LICENCE file distributed with this work for additional +// information regarding copyright ownership. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package de.upb.cs.swt.delphi.webapi + +import com.sksamuel.elastic4s.http.ElasticDsl._ +import com.sksamuel.elastic4s.http.get.GetResponse +import com.sksamuel.elastic4s.http.{ElasticClient, RequestSuccess, Response} +import de.upb.cs.swt.delphi.webapi.artifacts.{Artifact, ArtifactTransformer} + +object RetrieveQuery { + + def retrieve(identifier: String)(implicit configuration: Configuration): Option[List[Artifact]] = { + val client = ElasticClient(configuration.elasticsearchClientUri) + + val parsedIdentifier = MavenIdentifier(identifier) + + parsedIdentifier match { + case None => None + case Some(m) => { + + val response: Response[GetResponse] = client.execute { + get(configuration.esProjectIndex.index, configuration.esProjectIndex.`type`, m.toUniqueString) + }.await + + response match { + case RequestSuccess(_, body, _, results: GetResponse) => { + results.found match { + case false => None + case _ => { + Some(List(ArtifactTransformer.transformResult(results.id, results.sourceAsMap))) + } + } + } + case _ => None + } + } + } + } +} + + + + + + + diff --git a/src/main/scala/de/upb/cs/swt/delphi/webapi/Server.scala b/src/main/scala/de/upb/cs/swt/delphi/webapi/Server.scala index 02e72ee..d59dd29 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/webapi/Server.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/webapi/Server.scala @@ -1,33 +1,61 @@ +// Copyright (C) 2018 The Delphi Team. +// See the LICENCE file distributed with this work for additional +// information regarding copyright ownership. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package de.upb.cs.swt.delphi.webapi import java.util.concurrent.TimeUnit import akka.actor.ActorSystem -import akka.http.scaladsl.server.HttpApp -import akka.pattern.ask +import akka.http.scaladsl.model.{HttpResponse, StatusCodes} +import akka.http.scaladsl.server.{HttpApp, Route} +import akka.stream.ActorMaterializer import akka.util.Timeout -import de.upb.cs.swt.delphi.featuredefinitions.FeatureListMapping -import de.upb.cs.swt.delphi.webapi.ElasticActorManager.{Enqueue, Retrieve} -import de.upb.cs.swt.delphi.webapi.ElasticRequestLimiter.Validate +import de.upb.cs.swt.delphi.instancemanagement.InstanceRegistry +import de.upb.cs.swt.delphi.webapi.artifacts.ArtifactJson._ +import de.upb.cs.swt.delphi.webapi.search.QueryRequestJson._ +import de.upb.cs.swt.delphi.webapi.search.{QueryRequest, SearchQuery} import spray.json._ +import scala.concurrent.ExecutionContext +import scala.util.{Failure, Success} + /** * Web server configuration for Delphi web API. */ object Server extends HttpApp with JsonSupport with AppLogging { - private val configuration = new Configuration() - private val system = ActorSystem("delphi-webapi") - private val actorManager = system.actorOf(ElasticActorManager.props(configuration)) - private val requestLimiter = system.actorOf(ElasticRequestLimiter.props(configuration, actorManager)) - implicit val timeout = Timeout(5, TimeUnit.SECONDS) + implicit val system = ActorSystem("delphi-webapi") + implicit val materializer = ActorMaterializer() + + private implicit val configuration = new Configuration() + private implicit val timeout = Timeout(5, TimeUnit.SECONDS) - override def routes = - path("version") { version } ~ - path("features") { features } ~ - pathPrefix("search" / Remaining) { query => search(query) } ~ - pathPrefix("retrieve" / Remaining) { identifier => retrieve(identifier) } ~ - pathPrefix("enqueue" / Remaining) { identifier => enqueue(identifier) } + + override def routes: Route = + path("version") { + version + } ~ + path("features") { + features + } ~ + path("statistics") { + statistics + } ~ + pathPrefix("search" ) { search } ~ + pathPrefix("retrieve" / Remaining) { identifier => retrieve(identifier) } private def version = { @@ -38,50 +66,80 @@ object Server extends HttpApp with JsonSupport with AppLogging { } } + private val featureExtractor = new FeatureQuery(configuration) + private def features = { get { - complete { - FeatureListMapping.featureList.toJson + parameter('pretty.?) { (pretty) => + complete( + //TODO: Introduce failure concept for feature extractor + prettyPrint(pretty, featureExtractor.featureList.toJson) + ) } } } - def retrieve(identifier: String) = { + private def statistics = { get { - pass { //TODO: Require authentication here - complete( - (actorManager ? Retrieve(identifier)).mapTo[String] - ) - } ~ extractClientIP{ ip => - complete( - (requestLimiter ? Validate(ip, Retrieve(identifier))).mapTo[String] - ) + parameter('pretty.?) { (pretty) => + complete { + val result = new StatisticsQuery(configuration).retrieveStandardStatistics + result match { + case Some(stats) => { + import StatisticsJson._ + prettyPrint(pretty, stats.toJson) + } + case _ => HttpResponse(StatusCodes.InternalServerError) + } + } } } } - def enqueue(identifier: String) = { + private def retrieve(identifier: String): Route = { get { - pass { //TODO: Require authorization here + parameter('pretty.?) { (pretty) => complete( - (actorManager ? Enqueue(identifier)).mapTo[String] + RetrieveQuery.retrieve(identifier) match { + case Some(result) => prettyPrint(pretty, result.toJson) + case None => HttpResponse(StatusCodes.NotFound) + } ) } } } - def search(query: String) = { - get { - complete { - query + def search: Route = { + post { + parameter('pretty.?) { (pretty) => + entity(as[QueryRequest]) { input => + log.info(s"Received search query: ${input.query}") + complete( + new SearchQuery(configuration, featureExtractor).search(input) match { + case Success(result) => prettyPrint(pretty, result.toJson) + case Failure(e) => e.getMessage + } + ) + } } } } def main(args: Array[String]): Unit = { - val configuration = new Configuration() - Server.startServer(configuration.bindHost, configuration.bindPort) - system.terminate() + sys.addShutdownHook({ + log.warning("Received shutdown signal.") + InstanceRegistry.handleInstanceStop(configuration) + }) + + StartupCheck.check(configuration) + Server.startServer(configuration.bindHost, configuration.bindPort, system) + + implicit val ec: ExecutionContext = system.dispatcher + val terminationFuture = system.terminate() + + terminationFuture.onComplete { + sys.exit(0) + } } diff --git a/src/main/scala/de/upb/cs/swt/delphi/webapi/StartupCheck.scala b/src/main/scala/de/upb/cs/swt/delphi/webapi/StartupCheck.scala new file mode 100644 index 0000000..637e762 --- /dev/null +++ b/src/main/scala/de/upb/cs/swt/delphi/webapi/StartupCheck.scala @@ -0,0 +1,49 @@ +// Copyright (C) 2018 The Delphi Team. +// See the LICENCE file distributed with this work for additional +// information regarding copyright ownership. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package de.upb.cs.swt.delphi.webapi + +import akka.actor.ActorSystem +import com.sksamuel.elastic4s.http.ElasticClient +import com.sksamuel.elastic4s.http.ElasticDsl._ +import de.upb.cs.swt.delphi.instancemanagement.InstanceRegistry +import scala.concurrent.duration.Duration +import scala.concurrent.{Await, ExecutionContext} +import scala.util.{Failure, Success, Try} + +object StartupCheck extends AppLogging { + def check(configuration: Configuration)(implicit system: ActorSystem): Try[Configuration] = { + log.warning("Performing Instance Registry checks") + implicit val ec : ExecutionContext = system.dispatcher + lazy val client = ElasticClient(configuration.elasticsearchClientUri) + + val f = (client.execute { + nodeInfo() + } map { i => { + InstanceRegistry.sendMatchingResult(isElasticSearchReachable = true, configuration) + Success(configuration) + } + } recover { case e => + InstanceRegistry.sendMatchingResult(isElasticSearchReachable = false, configuration) + Failure(e) + + }).andThen { + case _ => client.close() + } + + Await.result(f, Duration.Inf) + } +} diff --git a/src/main/scala/de/upb/cs/swt/delphi/webapi/StatisticsQuery.scala b/src/main/scala/de/upb/cs/swt/delphi/webapi/StatisticsQuery.scala new file mode 100644 index 0000000..d913369 --- /dev/null +++ b/src/main/scala/de/upb/cs/swt/delphi/webapi/StatisticsQuery.scala @@ -0,0 +1,88 @@ +// Copyright (C) 2018 The Delphi Team. +// See the LICENCE file distributed with this work for additional +// information regarding copyright ownership. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package de.upb.cs.swt.delphi.webapi + +import com.sksamuel.elastic4s.http.ElasticDsl._ +import com.sksamuel.elastic4s.http.{ElasticClient, RequestSuccess} +import org.slf4j.LoggerFactory +import spray.json.DefaultJsonProtocol + +class StatisticsQuery(configuration: Configuration) { + private val log = LoggerFactory.getLogger(this.getClass) + + def retrieveStandardStatistics = { + val client = ElasticClient(configuration.elasticsearchClientUri) + + val fullIndexSize = searchWithType(configuration.esProjectIndex) size 0 + val hermesEnabledProjects = searchWithType(configuration.esProjectIndex) query bool { + must { + existsQuery("hermes") + } + } size (0) + + + client.execute { + multi ( + fullIndexSize, + hermesEnabledProjects + ) + }.await match { + // TODO: These matchers are non exhaustive + case RequestSuccess(_, _, _, results) => { + assert(results.size == 2) + val totalOpt = results.items(0).response match { + case Right(s) => { + Some(s.hits.total) + } + case _ => { + println(s"Received some other response: ${results.items(0).response}") + None + } + } + val hermesTotalOpt = results.items(1).response match { + case Right(s) => { + Some(s.hits.total) + } + case _ => { + println(s"Received some other response: ${results.items(1).response}") + None + } + } + totalOpt match { + case Some(total) => { + hermesTotalOpt match { + case Some(hermesTotal) => Some(Statistics(total, hermesTotal)) + case _ => None + } + } + case _ => None + } + + } + case result => { + log.warn(s"Request failed: $result") + None + } + } + } +} + +case class Statistics(total : Long, hermesEnabled : Long) + +object StatisticsJson extends DefaultJsonProtocol { + implicit val statisticsFormat = jsonFormat2(Statistics) +} diff --git a/src/main/scala/de/upb/cs/swt/delphi/webapi/artifacts/Artifact.scala b/src/main/scala/de/upb/cs/swt/delphi/webapi/artifacts/Artifact.scala new file mode 100644 index 0000000..265409b --- /dev/null +++ b/src/main/scala/de/upb/cs/swt/delphi/webapi/artifacts/Artifact.scala @@ -0,0 +1,19 @@ +// Copyright (C) 2018 The Delphi Team. +// See the LICENCE file distributed with this work for additional +// information regarding copyright ownership. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package de.upb.cs.swt.delphi.webapi.artifacts + +case class Artifact(id: String, metadata: ArtifactMetadata, metricResults: Map[String, Int]) diff --git a/src/main/scala/de/upb/cs/swt/delphi/webapi/artifacts/ArtifactJson.scala b/src/main/scala/de/upb/cs/swt/delphi/webapi/artifacts/ArtifactJson.scala new file mode 100644 index 0000000..617815c --- /dev/null +++ b/src/main/scala/de/upb/cs/swt/delphi/webapi/artifacts/ArtifactJson.scala @@ -0,0 +1,46 @@ +// Copyright (C) 2018 The Delphi Team. +// See the LICENCE file distributed with this work for additional +// information regarding copyright ownership. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package de.upb.cs.swt.delphi.webapi.artifacts + +import org.joda.time.DateTime +import org.joda.time.format.{DateTimeFormatter, ISODateTimeFormat} +import spray.json.{DefaultJsonProtocol, DeserializationException, JsString, JsValue, RootJsonFormat} + +object ArtifactJson extends DefaultJsonProtocol { + + implicit object DateJsonFormat extends RootJsonFormat[DateTime] { + + private val parserISO: DateTimeFormatter = ISODateTimeFormat.dateTime() + + override def write(obj: DateTime) = JsString(parserISO.print(obj)) + + override def read(json: JsValue): DateTime = json match { + case JsString(s) => parserISO.parseDateTime(s) + case _ => throw new DeserializationException("Error info you want here ...") + } + } + + implicit val artifactMetadataFormat = jsonFormat5(ArtifactMetadata) + implicit val artifactFormat = jsonFormat3(Artifact) + + def prettyPrint(pretty: Option[_], value: JsValue): String = { + pretty.isDefined match { + case true => value.sortedPrint + case false => value.compactPrint + } + } +} diff --git a/src/main/scala/de/upb/cs/swt/delphi/webapi/artifacts/ArtifactMetadata.scala b/src/main/scala/de/upb/cs/swt/delphi/webapi/artifacts/ArtifactMetadata.scala new file mode 100644 index 0000000..f35dbfb --- /dev/null +++ b/src/main/scala/de/upb/cs/swt/delphi/webapi/artifacts/ArtifactMetadata.scala @@ -0,0 +1,21 @@ +// Copyright (C) 2018 The Delphi Team. +// See the LICENCE file distributed with this work for additional +// information regarding copyright ownership. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package de.upb.cs.swt.delphi.webapi.artifacts + +import org.joda.time.DateTime + +case class ArtifactMetadata(source: String, discovered: DateTime, groupId: String, artifactId: String, version: String) diff --git a/src/main/scala/de/upb/cs/swt/delphi/webapi/artifacts/ArtifactTransformer.scala b/src/main/scala/de/upb/cs/swt/delphi/webapi/artifacts/ArtifactTransformer.scala new file mode 100644 index 0000000..a34fd12 --- /dev/null +++ b/src/main/scala/de/upb/cs/swt/delphi/webapi/artifacts/ArtifactTransformer.scala @@ -0,0 +1,56 @@ +// Copyright (C) 2018 The Delphi Team. +// See the LICENCE file distributed with this work for additional +// information regarding copyright ownership. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package de.upb.cs.swt.delphi.webapi.artifacts + +import com.sksamuel.elastic4s.http.search.{SearchHit, SearchHits} +import org.joda.time.format.ISODateTimeFormat + +object ArtifactTransformer { + + private def getHermesResults(sourceMap: Map[String, AnyRef]): Map[String, Int] = { + if (!sourceMap.contains("hermes")) return Map() + if (!sourceMap("hermes").isInstanceOf[Map[String, AnyRef]]) return Map() + if (!sourceMap("hermes").asInstanceOf[Map[String, AnyRef]].contains("features")) return Map() + + val hermesMap = sourceMap("hermes").asInstanceOf[Map[String, AnyRef]] + + hermesMap("features").asInstanceOf[Map[String, Int]] + } + + private def getMetadata(sourceMap: Map[String, AnyRef]): ArtifactMetadata = { + val identifier = sourceMap("identifier").asInstanceOf[Map[String, String]] + ArtifactMetadata(sourceMap("source").asInstanceOf[String], + ISODateTimeFormat.dateTime().parseDateTime(sourceMap("discovered").asInstanceOf[String]), + identifier("groupId"), identifier("artifactId"), identifier("version")) + } + + def transformResult(id : String, sourceMap : Map[String, AnyRef]) :Artifact = { + Artifact(id, getMetadata(sourceMap), getHermesResults(sourceMap)) + } + + + private def transformResult(h: SearchHit): Artifact = { + transformResult(h.id, h.sourceAsMap) + } + + + def transformResults(hits: SearchHits): Array[Artifact] = { + hits.hits.map(h => transformResult(h)) + } + + val baseFields = Seq("source", "discovered", "identifier.groupId", "identifier.artifactId", "identifier.version") +} diff --git a/src/main/scala/de/upb/cs/swt/delphi/webapi/artifacts/Identifier.scala b/src/main/scala/de/upb/cs/swt/delphi/webapi/artifacts/Identifier.scala new file mode 100644 index 0000000..ed419b3 --- /dev/null +++ b/src/main/scala/de/upb/cs/swt/delphi/webapi/artifacts/Identifier.scala @@ -0,0 +1,22 @@ +// Copyright (C) 2018 The Delphi Team. +// See the LICENCE file distributed with this work for additional +// information regarding copyright ownership. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package de.upb.cs.swt.delphi.webapi.artifacts + +/** + * Represents an identifier for a software artifact + */ +trait Identifier diff --git a/src/main/scala/de/upb/cs/swt/delphi/webapi/search/QueryRequest.scala b/src/main/scala/de/upb/cs/swt/delphi/webapi/search/QueryRequest.scala new file mode 100644 index 0000000..3479577 --- /dev/null +++ b/src/main/scala/de/upb/cs/swt/delphi/webapi/search/QueryRequest.scala @@ -0,0 +1,3 @@ +package de.upb.cs.swt.delphi.webapi.search + +case class QueryRequest (query : String, limit : Option[Int] = Some(50)) diff --git a/src/main/scala/de/upb/cs/swt/delphi/webapi/search/QueryRequestJson.scala b/src/main/scala/de/upb/cs/swt/delphi/webapi/search/QueryRequestJson.scala new file mode 100644 index 0000000..6ae239e --- /dev/null +++ b/src/main/scala/de/upb/cs/swt/delphi/webapi/search/QueryRequestJson.scala @@ -0,0 +1,7 @@ +package de.upb.cs.swt.delphi.webapi.search + +import spray.json.DefaultJsonProtocol + +object QueryRequestJson extends DefaultJsonProtocol { + implicit val queryRequestFormat = jsonFormat2(QueryRequest) +} diff --git a/src/main/scala/de/upb/cs/swt/delphi/webapi/search/SearchQuery.scala b/src/main/scala/de/upb/cs/swt/delphi/webapi/search/SearchQuery.scala new file mode 100644 index 0000000..80c11b1 --- /dev/null +++ b/src/main/scala/de/upb/cs/swt/delphi/webapi/search/SearchQuery.scala @@ -0,0 +1,124 @@ +// Copyright (C) 2018 The Delphi Team. +// See the LICENCE file distributed with this work for additional +// information regarding copyright ownership. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package de.upb.cs.swt.delphi.webapi.search + +import com.sksamuel.elastic4s.http.ElasticDsl._ +import com.sksamuel.elastic4s.http.search.SearchHits +import com.sksamuel.elastic4s.http.{ElasticClient, RequestSuccess} +import com.sksamuel.elastic4s.searches.queries.{NoopQuery, Query} +import de.upb.cs.swt.delphi.webapi.{Configuration, FeatureQuery} +import de.upb.cs.swt.delphi.webapi.artifacts.ArtifactTransformer +import de.upb.cs.swt.delphi.webapi.search.querylanguage._ + +import scala.util.{Failure, Success, Try} + +class SearchQuery(configuration: Configuration, featureExtractor: FeatureQuery) { + private val client = ElasticClient(configuration.elasticsearchClientUri) + + private def checkAndExecuteParsedQuery(ast: CombinatorialExpr, limit : Int): Try[SearchHits] = { + val fields = collectFieldNames(ast) + if (fields.diff(featureExtractor.featureList.toSeq).size > 0) return Failure(new IllegalArgumentException("Unknown field name used.")) + + val query = searchWithType(configuration.esProjectIndex) + .query(translate(ast)) + .sourceInclude(ArtifactTransformer.baseFields ++ fields.intersect(featureExtractor.featureList.toSeq).map(i => addPrefix(i))) + .limit(limit) + + val response = client.execute { + query + }.await + + response match { + case RequestSuccess(_, body, _, result) => Success(result.hits) + case r => Failure(new IllegalArgumentException(r.toString)) + } + } + + private def addPrefix(fieldName: String): String = s"hermes.features.$fieldName" + + private def translate(node: CombinatorialExpr): Query = { + node match { + case AndExpr(left, right) => bool { + must( + translate(left), + translate(right) + ) + } + case OrExpr(left, right) => bool { + should( + translate(left), + translate(right) + ) + } + case NotExpr(expr) => bool { + not(translate(expr)) + } + case XorExpr(left, right) => bool { + should( + must( + translate(left), + not(translate(right)) + ), + must( + not(translate(right)), + translate(left) + ) + ) + } + case EqualExpr(field, value) => matchQuery(addPrefix(field.fieldName), value) + case NotEqualExpr(field, value) => bool(not(matchQuery(addPrefix(field.fieldName), value))) + case GreaterThanExpr(field, value) => rangeQuery(addPrefix(field.fieldName)).gt(value.toLong) + case GreaterOrEqualExpr(field, value) => rangeQuery(addPrefix(field.fieldName)).gte(value.toLong) + case LessThanExpr(field, value) => rangeQuery(addPrefix(field.fieldName)).lt(value.toLong) + case LessOrEqualExpr(field, value) => rangeQuery(addPrefix(field.fieldName)).lte(value.toLong) + case LikeExpr(field, value) => prefixQuery(addPrefix(field.fieldName), value) + case _ => NoopQuery + } + } + + private def collectFieldNames(node: CombinatorialExpr): Seq[String] = { + node match { + case AndExpr(left, right) => collectFieldNames(left) ++ collectFieldNames(right) + case OrExpr(left, right) => collectFieldNames(left) ++ collectFieldNames(right) + case NotExpr(expr) => collectFieldNames(expr) + case XorExpr(left, right) => collectFieldNames(left) ++ collectFieldNames(right) + case EqualExpr(field, _) => Seq(field.fieldName) + case NotEqualExpr(field, _) => Seq(field.fieldName) + case GreaterThanExpr(field, _) => Seq(field.fieldName) + case GreaterOrEqualExpr(field, _) => Seq(field.fieldName) + case LessThanExpr(field, _) => Seq(field.fieldName) + case LessOrEqualExpr(field, _) => Seq(field.fieldName) + case LikeExpr(field, _) => Seq(field.fieldName) + case IsTrueExpr(field) => Seq(field.fieldName) + case FieldReference(name) => Seq(name) + case _ => Seq() + } + } + + def search(query: QueryRequest) = { + val parserResult = new Syntax(query.query).QueryRule.run() + parserResult match { + case Failure(e) => Failure(e) + case Success(ast) => { + checkAndExecuteParsedQuery(ast, query.limit.getOrElse(50)) match { + case Failure(e) => Failure(e) + case Success(hits) => Success(ArtifactTransformer.transformResults(hits)) + } + } + } + } +} diff --git a/src/main/scala/de/upb/cs/swt/delphi/webapi/search/querylanguage/AST.scala b/src/main/scala/de/upb/cs/swt/delphi/webapi/search/querylanguage/AST.scala new file mode 100644 index 0000000..ccece70 --- /dev/null +++ b/src/main/scala/de/upb/cs/swt/delphi/webapi/search/querylanguage/AST.scala @@ -0,0 +1,36 @@ +// Copyright (C) 2018 The Delphi Team. +// See the LICENCE file distributed with this work for additional +// information regarding copyright ownership. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package de.upb.cs.swt.delphi.webapi.search.querylanguage + +trait CombinatorialExpr + +case class AndExpr(left: CombinatorialExpr, right: CombinatorialExpr) extends CombinatorialExpr +case class OrExpr(left: CombinatorialExpr, right: CombinatorialExpr) extends CombinatorialExpr +case class NotExpr(expr: CombinatorialExpr) extends CombinatorialExpr +case class XorExpr(left: CombinatorialExpr, right: CombinatorialExpr) extends CombinatorialExpr + +trait SingularConditionExpr extends CombinatorialExpr + +case class EqualExpr(left: FieldReference, right: String) extends SingularConditionExpr +case class NotEqualExpr(left: FieldReference, right: String) extends SingularConditionExpr +case class GreaterThanExpr(left: FieldReference, right: String) extends SingularConditionExpr +case class GreaterOrEqualExpr(left: FieldReference, right: String) extends SingularConditionExpr +case class LessThanExpr(left: FieldReference, right: String) extends SingularConditionExpr +case class LessOrEqualExpr(left: FieldReference, right: String) extends SingularConditionExpr +case class LikeExpr(left: FieldReference, right: String) extends SingularConditionExpr +case class IsTrueExpr(fieldName: FieldReference) extends SingularConditionExpr +case class FieldReference(fieldName: String) extends CombinatorialExpr \ No newline at end of file diff --git a/src/main/scala/de/upb/cs/swt/delphi/webapi/search/querylanguage/Syntax.scala b/src/main/scala/de/upb/cs/swt/delphi/webapi/search/querylanguage/Syntax.scala new file mode 100644 index 0000000..2df2cb4 --- /dev/null +++ b/src/main/scala/de/upb/cs/swt/delphi/webapi/search/querylanguage/Syntax.scala @@ -0,0 +1,74 @@ +// Copyright (C) 2018 The Delphi Team. +// See the LICENCE file distributed with this work for additional +// information regarding copyright ownership. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package de.upb.cs.swt.delphi.webapi.search.querylanguage + +import org.parboiled2.{CharPredicate, Parser, ParserInput, Rule1} + +/** + * The syntax definition and parser for the Delphi QL. + * + * @author Lisa Nguyen Quang Do + * @author Ben Hermann + * + */ +class Syntax(val input : ParserInput) extends Parser { + + def QueryRule = rule { + CombinatorialRule ~ EOI + } + + // Combinatorial rules. + def CombinatorialRule : Rule1[CombinatorialExpr] = rule { + OrOrElseRule | NotRule + } + def OrOrElseRule = rule { + AndOrElseRule ~ zeroOrMore("||" ~ AndOrElseRule ~> OrExpr) + } + def AndOrElseRule = rule { + XorOrElseRule ~ zeroOrMore("&&" ~ XorOrElseRule ~> AndExpr) + } + def XorOrElseRule = rule { + Factor ~ zeroOrMore("%%" ~ Factor ~> XorExpr) + } + + // Handling parentheses. + def Factor : Rule1[CombinatorialExpr] = rule { + Parentheses | SingularConditionRule | NotRule + } + def Parentheses = rule { '(' ~ CombinatorialRule ~ ')' } + def NotRule = rule { '!' ~ Factor ~> NotExpr } + + // Singular conditions. + def SingularConditionRule = rule { + EqualRule | NotEqualRule | GreaterThanRule | GreaterOrEqual | + LessThan | LessOrEqual | Like | IsTrue + } + def EqualRule = rule { FieldReferenceRule ~ "=" ~ Literal ~> EqualExpr } + def NotEqualRule = rule { FieldReferenceRule ~ "!=" ~ Literal ~> NotEqualExpr } + def GreaterThanRule = rule { FieldReferenceRule ~ ">" ~ Literal ~> GreaterThanExpr } + def GreaterOrEqual = rule { FieldReferenceRule ~ ">=" ~ Literal ~> GreaterOrEqualExpr } + def LessThan = rule { FieldReferenceRule ~ "<" ~ Literal ~> LessThanExpr } + def LessOrEqual = rule { FieldReferenceRule ~ "<=" ~ Literal ~> LessOrEqualExpr } + def Like = rule { FieldReferenceRule ~ "%" ~ Literal ~> LikeExpr } + def IsTrue = rule { FieldReferenceRule ~> IsTrueExpr } + + // Literals + def FieldReferenceRule = rule { "[" ~ capture(oneOrMore(CharPredicate.AlphaNum ++ '-' ++ ' ' ++ '_' ++ '(' ++ ':' ++')')) ~ "]" ~> FieldReference } + def Literal = rule { capture(oneOrMore(CharPredicate.AlphaNum)) ~> (_.toString) } +} + + diff --git a/src/test/scala/de/upb/cs/swt/delphi/querylanguage/SyntaxTest.scala b/src/test/scala/de/upb/cs/swt/delphi/querylanguage/SyntaxTest.scala deleted file mode 100644 index 336d420..0000000 --- a/src/test/scala/de/upb/cs/swt/delphi/querylanguage/SyntaxTest.scala +++ /dev/null @@ -1,152 +0,0 @@ -package de.upb.cs.swt.delphi.querylanguage - -import org.scalatest.{FlatSpec, Matchers} -import scala.util.{Failure, Success} - -/** - * Tests for the DelphiQL syntax. - * - * @author Lisa Nguyen Quang Do - */ -class SyntaxTest extends FlatSpec with Matchers { - - "Syntax.singularConditionWithOperator" should "be valid" in { - val parseResult = new Syntax("Filter1=abc").QueryRule.run() - parseResult shouldBe a [Success[_]] - parseResult match { - case Success(ast) => { - ast.toString shouldEqual "EqualExpr(Filter1,abc)" - } - } - } - - "Syntax.singularConditionNoOperator" should "be valid" in { - val parseResult = new Syntax("Filter1").QueryRule.run() - parseResult shouldBe a [Success[_]] - parseResult match { - case Success(ast) => { - ast.toString shouldEqual "TrueExpr(Filter1)" - } - } - } - - "Syntax.singularConditionTypo" should "be valid" in { - val parseResult = new Syntax("Filter1=<3").QueryRule.run() - parseResult shouldBe a [Failure[_]] - } - - "Syntax.singularConditionOddCharacters" should "be valid" in { - val parseResult = new Syntax("Filter1%Filter2%'}.:").QueryRule.run() - parseResult shouldBe a [Failure[_]] - } - - "Syntax.combinatoryConditionSimple" should "be valid" in { - val parseResult = new Syntax("Filter1&&Filter2=3").QueryRule.run() - parseResult shouldBe a [Success[_]] - parseResult match { - case Success(ast) => { - ast.toString shouldEqual "AndExpr(TrueExpr(Filter1),EqualExpr(Filter2,3))" - } - } - } - - "Syntax.combinatoryConditionParentheses" should "be valid" in { - val parseResult = new Syntax("Filter1||(Filter2&&(Filter3<3||Filter4>0))").QueryRule.run() - parseResult shouldBe a [Success[_]] - parseResult match { - case Success(ast) => { - ast.toString shouldEqual "OrExpr(TrueExpr(Filter1),AndExpr(TrueExpr(Filter2)," + - "OrExpr(LessThanExpr(Filter3,3),GreaterThanExpr(Filter4,0))))" - } - } - } - - "Syntax.combinatoryConditionParenthesesComplex" should "be valid" in { - val parseResult = new Syntax("Filter1&&((Filter2<3||Filter2>0)%%(Filter4&&Filter5))").QueryRule.run() - parseResult shouldBe a [Success[_]] - parseResult match { - case Success(ast) => { - ast.toString shouldEqual "AndExpr(TrueExpr(Filter1),XorExpr(OrExpr(LessThanExpr(Filter2,3)," + - "GreaterThanExpr(Filter2,0)),AndExpr(TrueExpr(Filter4),TrueExpr(Filter5))))" - } - } - } - - "Syntax.combinatoryConditionNonMatchingParentheses" should "be valid" in { - val parseResult = new Syntax("Filter1&&(Filter2<3||Filter2>0%%(Filter4)").QueryRule.run() - parseResult shouldBe a [Failure[_]] - } - - "Syntax.combinatoryConditionTypo" should "be valid" in { - val parseResult = new Syntax("Filter1&Filter2<3)").QueryRule.run() - parseResult shouldBe a [Failure[_]] - } - - "Syntax.combinatoryConditionLeftToRightPriority" should "be valid" in { - val parseResult = new Syntax("Filter1&&Filter2&&Filter3").QueryRule.run() - parseResult shouldBe a [Success[_]] - parseResult match { - case Success(ast) => { - ast.toString shouldEqual "AndExpr(AndExpr(TrueExpr(Filter1)," + - "TrueExpr(Filter2)),TrueExpr(Filter3))" - } - } - } - - "Syntax.combinatoryConditionOperatorPriorities" should "be valid" in { - val parseResult = new Syntax("Filter1||Filter2%%!Filter3&&Filter4").QueryRule.run() - parseResult shouldBe a [Success[_]] - parseResult match { - case Success(ast) => { - ast.toString shouldEqual "OrExpr(TrueExpr(Filter1),AndExpr(XorExpr(" + - "TrueExpr(Filter2),NotExpr(TrueExpr(Filter3))),TrueExpr(Filter4)))" - } - } - } - - "Syntax.combinatoryConditionOperatorPrioritiesParentheses" should "be valid" in { - val parseResult = new Syntax("(Filter1||Filter2)&&!Filter3%%!(Filter4&&Filter5)").QueryRule.run() - parseResult shouldBe a [Success[_]] - parseResult match { - case Success(ast) => { - ast.toString shouldEqual "AndExpr(OrExpr(TrueExpr(Filter1),TrueExpr(Filter2))," + - "XorExpr(NotExpr(TrueExpr(Filter3)),NotExpr(AndExpr(TrueExpr(Filter4)," + - "TrueExpr(Filter5)))))" - } - } - } - - "Syntax.notConditionSimple" should "be valid" in { - val parseResult = new Syntax("!Filter1&&!(Filter2)").QueryRule.run() - parseResult shouldBe a [Success[_]] - parseResult match { - case Success(ast) => { - ast.toString shouldEqual "AndExpr(NotExpr(TrueExpr(Filter1))," + - "NotExpr(TrueExpr(Filter2)))" - } - } - } - - "Syntax.notConditionSimpleParentheses" should "be valid" in { - val parseResult = new Syntax("!(Filter1&&!Filter2)").QueryRule.run() - parseResult shouldBe a [Success[_]] - parseResult match { - case Success(ast) => { - ast.toString shouldEqual "NotExpr(AndExpr(TrueExpr(Filter1)," + - "NotExpr(TrueExpr(Filter2))))" - } - } - } - - "Syntax.notConditionComplex" should "be valid" in { - val parseResult = new Syntax("!!(Filter1)&&!(Filter2<=0||!(Filter3&&!Filter4%abc))").QueryRule.run() - parseResult shouldBe a [Success[_]] - parseResult match { - case Success(ast) => { - ast.toString shouldEqual "AndExpr(NotExpr(NotExpr(TrueExpr(Filter1)))," + - "NotExpr(OrExpr(LessOrEqualExpr(Filter2,0),NotExpr(AndExpr(TrueExpr(Filter3)," + - "NotExpr(LikeExpr(Filter4,abc)))))))" - } - } - } -} \ No newline at end of file diff --git a/src/test/scala/de/upb/cs/swt/delphi/webapi/MavenIdentifierApplyTest.scala b/src/test/scala/de/upb/cs/swt/delphi/webapi/MavenIdentifierApplyTest.scala new file mode 100644 index 0000000..ae55a7c --- /dev/null +++ b/src/test/scala/de/upb/cs/swt/delphi/webapi/MavenIdentifierApplyTest.scala @@ -0,0 +1,61 @@ +// Copyright (C) 2018 The Delphi Team. +// See the LICENCE file distributed with this work for additional +// information regarding copyright ownership. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package de.upb.cs.swt.delphi.webapi + +import org.scalatest.{FlatSpec, Matchers} + +class MavenIdentifierApplyTest extends FlatSpec with Matchers { + "Valid identifiers with version" should "convert nicely" in { + val fullIdentifier = MavenIdentifier("log4j:log4j-test:1.4.2") + fullIdentifier match { + case Some(m) => { + m.repository.isDefined shouldBe false + m.groupId shouldBe "log4j" + m.artifactId shouldBe "log4j-test" + m.version.isDefined shouldBe true + m.version.getOrElse("") shouldBe "1.4.2" + } + case _ => fail("Identifier could not be properly parsed.") + } + } + "Valid identifiers without version" should "convert nicely" in { + val partial = MavenIdentifier("log4j:log4j-test") + partial match { + case Some(m) => { + m.repository.isDefined shouldBe false + m.groupId shouldBe "log4j" + m.artifactId shouldBe "log4j-test" + m.version.isDefined shouldBe false + } + case _ => fail("Identifier could not be properly parsed.") + } + } + + "Invalid identifiers" should "fail" in { + val invalid = MavenIdentifier("log4j") + invalid.isDefined shouldBe false + + val invalid2 = MavenIdentifier("log4j:::") + invalid2.isDefined shouldBe false + + val invalid3 = MavenIdentifier("log4j::") + invalid3.isDefined shouldBe false + + val invalid4 = MavenIdentifier("") + invalid4.isDefined shouldBe false + } +} diff --git a/src/test/scala/de/upb/cs/swt/delphi/webapi/search/querylanguage/SyntaxTest.scala b/src/test/scala/de/upb/cs/swt/delphi/webapi/search/querylanguage/SyntaxTest.scala new file mode 100644 index 0000000..94a79d0 --- /dev/null +++ b/src/test/scala/de/upb/cs/swt/delphi/webapi/search/querylanguage/SyntaxTest.scala @@ -0,0 +1,176 @@ +// Copyright (C) 2018 The Delphi Team. +// See the LICENCE file distributed with this work for additional +// information regarding copyright ownership. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package de.upb.cs.swt.delphi.webapi.search.querylanguage + +import org.scalatest.{FlatSpec, Matchers} +import scala.util.{Failure, Success} + +/** + * Tests for the DelphiQL syntax. + * + * @author Lisa Nguyen Quang Do + */ +class SyntaxTest extends FlatSpec with Matchers { + + "Syntax.singularConditionWithOperator" should "be valid" in { + val parseResult = new Syntax("[Filter1]=abc").QueryRule.run() + parseResult shouldBe a [Success[_]] + parseResult match { + case Success(ast) => { + ast.toString shouldEqual "EqualExpr(FieldReference(Filter1),abc)" + } + } + } + + + "Syntax.singularConditionNoOperator" should "be valid" in { + val parseResult = new Syntax("[Filter1]").QueryRule.run() + parseResult shouldBe a [Success[_]] + parseResult match { + case Success(ast) => { + ast.toString shouldEqual "IsTrueExpr(FieldReference(Filter1))" + } + } + } + + + "Syntax.singularConditionTypo" should "be valid" in { + val parseResult = new Syntax("[Filter1]=<3").QueryRule.run() + parseResult shouldBe a [Failure[_]] + } + + "Syntax.singularConditionOddCharacters" should "be valid" in { + val parseResult = new Syntax("[Filter1]%[Filter2]%'}.:").QueryRule.run() + parseResult shouldBe a [Failure[_]] + } + + + "Syntax.combinatoryConditionSimple" should "be valid" in { + val parseResult = new Syntax("[Filter1]&&[Filter2]=3").QueryRule.run() + parseResult shouldBe a [Success[_]] + parseResult match { + case Success(ast) => { + ast.toString shouldEqual "AndExpr(IsTrueExpr(FieldReference(Filter1)),EqualExpr(FieldReference(Filter2),3))" + } + } + } + + + "Syntax.combinatoryConditionParentheses" should "be valid" in { + val parseResult = new Syntax("[Filter1]||([Filter2]&&([Filter3]<3||[Filter4]>0))").QueryRule.run() + parseResult shouldBe a [Success[_]] + parseResult match { + case Success(ast) => { + ast.toString shouldEqual "OrExpr(IsTrueExpr(FieldReference(Filter1)),AndExpr(IsTrueExpr(FieldReference(Filter2))," + + "OrExpr(LessThanExpr(FieldReference(Filter3),3),GreaterThanExpr(FieldReference(Filter4),0))))" + } + } + } + + + "Syntax.combinatoryConditionParenthesesComplex" should "be valid" in { + val parseResult = new Syntax("[Filter1]&&(([Filter2]<3||[Filter2]>0)%%([Filter4]&&[Filter5]))").QueryRule.run() + parseResult shouldBe a [Success[_]] + parseResult match { + case Success(ast) => { + ast.toString shouldEqual "AndExpr(IsTrueExpr(FieldReference(Filter1)),XorExpr(OrExpr(LessThanExpr(FieldReference(Filter2),3)," + + "GreaterThanExpr(FieldReference(Filter2),0)),AndExpr(IsTrueExpr(FieldReference(Filter4)),IsTrueExpr(FieldReference(Filter5)))))" + } + } + } + + + "Syntax.combinatoryConditionNonMatchingParentheses" should "be valid" in { + val parseResult = new Syntax("[Filter1]&&([Filter2]<3||[Filter2]>0%%([Filter4])").QueryRule.run() + parseResult shouldBe a [Failure[_]] + } + + "Syntax.combinatoryConditionTypo" should "be valid" in { + val parseResult = new Syntax("[Filter1]&[Filter2]<3)").QueryRule.run() + parseResult shouldBe a [Failure[_]] + } + + "Syntax.combinatoryConditionLeftToRightPriority" should "be valid" in { + val parseResult = new Syntax("[Filter1]&&[Filter2]&&[Filter3]").QueryRule.run() + parseResult shouldBe a [Success[_]] + parseResult match { + case Success(ast) => { + ast.toString shouldEqual "AndExpr(AndExpr(IsTrueExpr(FieldReference(Filter1))," + + "IsTrueExpr(FieldReference(Filter2))),IsTrueExpr(FieldReference(Filter3)))" + } + } + } + + "Syntax.combinatoryConditionOperatorPriorities" should "be valid" in { + val parseResult = new Syntax("[Filter1]||[Filter2]%%![Filter3]&&[Filter4]").QueryRule.run() + parseResult shouldBe a [Success[_]] + parseResult match { + case Success(ast) => { + ast.toString shouldEqual "OrExpr(IsTrueExpr(FieldReference(Filter1)),AndExpr(XorExpr(" + + "IsTrueExpr(FieldReference(Filter2)),NotExpr(IsTrueExpr(FieldReference(Filter3)))),IsTrueExpr(FieldReference(Filter4))))" + } + } + } + + + "Syntax.combinatoryConditionOperatorPrioritiesParentheses" should "be valid" in { + val parseResult = new Syntax("([Filter1]||[Filter2])&&![Filter3]%%!([Filter4]&&[Filter5])").QueryRule.run() + parseResult shouldBe a [Success[_]] + parseResult match { + case Success(ast) => { + ast.toString shouldEqual "AndExpr(OrExpr(IsTrueExpr(FieldReference(Filter1)),IsTrueExpr(FieldReference(Filter2)))," + + "XorExpr(NotExpr(IsTrueExpr(FieldReference(Filter3))),NotExpr(AndExpr(IsTrueExpr(FieldReference(Filter4))," + + "IsTrueExpr(FieldReference(Filter5))))))" + } + } + } + + + "Syntax.notConditionSimple" should "be valid" in { + val parseResult = new Syntax("![Filter1]&&!([Filter2])").QueryRule.run() + parseResult shouldBe a [Success[_]] + parseResult match { + case Success(ast) => { + ast.toString shouldEqual "AndExpr(NotExpr(IsTrueExpr(FieldReference(Filter1)))," + + "NotExpr(IsTrueExpr(FieldReference(Filter2))))" + } + } + } + + "Syntax.notConditionSimpleParentheses" should "be valid" in { + val parseResult = new Syntax("!([Filter1]&&![Filter2])").QueryRule.run() + parseResult shouldBe a [Success[_]] + parseResult match { + case Success(ast) => { + ast.toString shouldEqual "NotExpr(AndExpr(IsTrueExpr(FieldReference(Filter1))," + + "NotExpr(IsTrueExpr(FieldReference(Filter2)))))" + } + } + } + + "Syntax.notConditionComplex" should "be valid" in { + val parseResult = new Syntax("!!([Filter1])&&!([Filter2]<=0||!([Filter3]&&![Filter4]%abc))").QueryRule.run() + parseResult shouldBe a [Success[_]] + parseResult match { + case Success(ast) => { + ast.toString shouldEqual "AndExpr(NotExpr(NotExpr(IsTrueExpr(FieldReference(Filter1))))," + + "NotExpr(OrExpr(LessOrEqualExpr(FieldReference(Filter2),0),NotExpr(AndExpr(IsTrueExpr(FieldReference(Filter3))," + + "NotExpr(LikeExpr(FieldReference(Filter4),abc)))))))" + } + } + } +} \ No newline at end of file