Skip to content

Commit 654e612

Browse files
almackenalmacken
committed
Improved code quality
1 parent 9665292 commit 654e612

File tree

7 files changed

+202
-193
lines changed

7 files changed

+202
-193
lines changed

‎build.sbt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@ libraryDependencies ++= Seq(
4949
"com.typesafe.akka" %% "akka-stream" % akkaVersion,
5050
"com.typesafe.akka" %% "akka-stream-testkit" % akkaVersion % Test,
5151
"com.typesafe.akka" %% "akka-slf4j" % akkaVersion,
52-
"com.typesafe.akka" %% "akka-http" % "10.1.3"
52+
"com.typesafe.akka" %% "akka-http" % "10.1.4"
5353
)
5454

5555
libraryDependencies += "ch.qos.logback" % "logback-classic" % "1.2.3" % Runtime

‎src/main/scala/de/upb/cs/swt/delphi/crawler/processing/CallGraphStream.scala‎

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ import org.opalj.ai.analyses.cg.UnresolvedMethodCall
2020

2121
import scala.concurrent.duration._
2222
import scala.concurrent.{ExecutionContext, Future}
23+
import scala.util.Try
2324

2425
/*
2526
* This component takes a Maven identifier, determines what methods it calls in each of it's dependencies, and
@@ -81,7 +82,7 @@ class CallGraphStream(configuration: Configuration) extends Actor with ActorLogg
8182
val fileGen = b.add(fileGenFlow)
8283
val edgeGen = b.add(edgeSetFlow)
8384

84-
fileGen.filter(t => (t._3.artifactId != null)) ~> dependencyConverter.filter{case (ix, jf, i) =>
85+
fileGen ~> dependencyConverter.filter{case (ix, jf, i) =>
8586
if(ix.isEmpty) { log.info(i.toString + " not mapped, incomplete POM file."); false }
8687
else { log.info(i.toString + " mapped, valid POM file."); true }} ~> edgeGen.in
8788

@@ -107,13 +108,13 @@ class CallGraphStream(configuration: Configuration) extends Actor with ActorLogg
107108
})
108109

109110
def fetchFiles(mavenIdentifier: MavenIdentifier): (PomFile, JarFile, MavenIdentifier) = {
110-
try {
111-
val downloader = new MavenDownloader(mavenIdentifier)
112-
(downloader.downloadPom(), downloader.downloadJar(), mavenIdentifier)
113-
} catch {
114-
case e: FileNotFoundException => {
111+
val downloader = new MavenDownloader(mavenIdentifier)
112+
def files = Try((downloader.downloadPom(), downloader.downloadJar(), mavenIdentifier))
113+
files match {
114+
case util.Success(f) => f
115+
case util.Failure(e: FileNotFoundException) => {
115116
log.info("{} not mapped, missing POM file", mavenIdentifier.toString)
116-
(PomFile(null), JarFile(null, null), MavenIdentifier(null, null, null, null)) //There might be a more elegant way of doing this
117+
throw e
117118
}
118119
}
119120
}

‎src/main/scala/de/upb/cs/swt/delphi/crawler/processing/ElasticEdgeSearchActor.scala‎

Lines changed: 52 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,8 @@ import de.upb.cs.swt.delphi.crawler.discovery.maven.MavenIdentifier
88
import de.upb.cs.swt.delphi.crawler.processing.CallGraphStream.{MappedEdge, unresMCtoStr}
99
import org.opalj.ai.analyses.cg.UnresolvedMethodCall
1010

11+
import scala.util.Try
12+
1113
/*
1214
* This class matches unresolved method calls to dependencies using the local Elasticsearch server.
1315
*
@@ -17,19 +19,17 @@ import org.opalj.ai.analyses.cg.UnresolvedMethodCall
1719

1820
class ElasticEdgeSearchActor(client: ElasticClient) extends Actor with ActorLogging{
1921

20-
val maxBatchSize = 150
22+
val maxBatchSize = 100
2123

2224
override def receive: Receive = {
2325
case (mx: Set[UnresolvedMethodCall], ix: Set[MavenIdentifier]) => {
24-
try {
25-
segmentFun(searchMethods, maxBatchSize)(mx, ix.toList) match {
26-
case (unmapped, mapped) =>
27-
sender() ! (unmapped, ix, mapped)
28-
}
29-
} catch {
30-
case e: Exception =>
26+
Try(segmentFun(searchMethods, maxBatchSize)(mx, ix.toList)) match {
27+
case util.Success((unmapped, mapped)) =>
28+
sender() ! (unmapped, ix, mapped)
29+
case util.Failure(e) => {
3130
log.warning("Elastic mapper threw exception " + e)
32-
akka.actor.Status.Failure(e)
31+
sender() ! akka.actor.Status.Failure(e)
32+
}
3333
}
3434
}
3535
}
@@ -39,52 +39,61 @@ class ElasticEdgeSearchActor(client: ElasticClient) extends Actor with ActorLogg
3939
private def segmentFun(fx: ((Set[UnresolvedMethodCall], List[MavenIdentifier]) => (Set[UnresolvedMethodCall], Set[MappedEdge])), batch: Int)
4040
(mx: Set[UnresolvedMethodCall], ix: List[MavenIdentifier]): (Set[UnresolvedMethodCall], Set[MappedEdge]) = {
4141
if (mx.size > batch) {
42-
val segmentedMx = mx.splitAt(batch)
43-
val segmentResults = fx(segmentedMx._1, ix)
44-
val remainingResults = segmentFun(fx, batch)(segmentedMx._2, ix)
45-
(segmentResults._1 ++ remainingResults._1, segmentResults._2 ++ remainingResults._2)
42+
mx.splitAt(batch) match { case (currSeg, restSeg) =>
43+
val segmentResults = fx (currSeg, ix)
44+
val remainingResults = segmentFun (fx, batch) (restSeg, ix)
45+
(segmentResults, remainingResults) match { case ((seg1, seg2), (rem1, rem2)) =>
46+
(seg1 ++ rem1, seg2 ++ rem2)
47+
}
48+
}
4649
} else {
4750
fx(mx, ix)
4851
}
4952
}
5053

51-
private def searchMethods(calls: Set[UnresolvedMethodCall], ids: List[MavenIdentifier]): (Set[UnresolvedMethodCall], Set[MappedEdge]) = {
52-
53-
def genSearchDef(call: UnresolvedMethodCall, id: MavenIdentifier) = {
54-
search("delphi").query {
55-
nestedQuery("calls",
56-
boolQuery().must(
57-
termQuery("calls.name", id.toString),
58-
termQuery("calls.methods", unresMCtoStr(call))
59-
)
54+
def genSearchDef(call: UnresolvedMethodCall, id: MavenIdentifier) = {
55+
search("delphi").query {
56+
nestedQuery("calls",
57+
boolQuery().must(
58+
termQuery("calls.name", id.toString),
59+
termQuery("calls.methods", unresMCtoStr(call))
6060
)
61-
}
61+
)
6262
}
63+
}
6364

64-
def searchEsDb(calls: List[UnresolvedMethodCall], id: MavenIdentifier): (UnresolvedMethodCall) => Boolean = {
65-
val resp: Response[MultiSearchResponse] = client.execute{
66-
multi(
67-
for(call <- calls) yield genSearchDef(call, id)
68-
)
69-
}.await
65+
def searchEsDb(calls: List[UnresolvedMethodCall], id: MavenIdentifier): (UnresolvedMethodCall) => Boolean = {
66+
val resp: Response[MultiSearchResponse] = client.execute{
67+
multi(
68+
for(call <- calls) yield genSearchDef(call, id)
69+
)
70+
}.await
7071

71-
val result = resp.result.items
72-
val hitIndices = result.filter(_.response.isRight).filter(_.response.right.get.totalHits > 0).map(_.index)
73-
val hits = for (i <- hitIndices) yield calls(i)
72+
val result = resp.result.items
73+
val hitIndices = result.filter(_.response.isRight).filter(_.response.right.getOrElse(throw new Exception).totalHits > 0).map(_.index)
74+
val hits = for (i <- hitIndices) yield calls(i)
7475

75-
hits.contains
76-
}
76+
hits.contains
77+
}
7778

78-
if (ids.isEmpty) (calls, Set[MappedEdge]())
79-
else if (calls.isEmpty) (Set[UnresolvedMethodCall](), Set[MappedEdge]())
80-
else {
81-
val id = ids.head
82-
def exists = searchEsDb(calls.toList, id)
83-
val splitSet = calls.partition(exists)
84-
val mappedMethods: Set[MappedEdge] = splitSet._1.map(m => MappedEdge(id, unresMCtoStr(m)))
79+
private def searchMethods(calls: Set[UnresolvedMethodCall], ids: List[MavenIdentifier]): (Set[UnresolvedMethodCall], Set[MappedEdge]) = {
80+
if (calls.isEmpty) (Set[UnresolvedMethodCall](), Set[MappedEdge]())
81+
82+
ids.headOption match {
83+
case None => (calls, Set[MappedEdge]())
84+
case Some(id) => {
85+
val id = ids.head
86+
def exists = searchEsDb(calls.toList, id)
87+
calls.partition(exists) match {
88+
case (hits, misses) => {
89+
val mappedMethods: Set[MappedEdge] = hits.map(m => MappedEdge(id, unresMCtoStr(m)))
8590

86-
searchMethods(splitSet._2, ids.tail) match { case (unresolved, mapped) =>
87-
(unresolved, mapped ++ mappedMethods)
91+
searchMethods(misses, ids.tail) match {
92+
case (unresolved, mapped) =>
93+
(unresolved, mapped ++ mappedMethods)
94+
}
95+
}
96+
}
8897
}
8998
}
9099
}

‎src/main/scala/de/upb/cs/swt/delphi/crawler/processing/MavenDependencyActor.scala‎

Lines changed: 56 additions & 59 deletions
Original file line numberDiff line numberDiff line change
@@ -3,13 +3,14 @@ package de.upb.cs.swt.delphi.crawler.processing
33
import akka.actor.{Actor, ActorLogging, Props}
44
import de.upb.cs.swt.delphi.crawler.Configuration
55
import de.upb.cs.swt.delphi.crawler.discovery.maven.MavenIdentifier
6-
import de.upb.cs.swt.delphi.crawler.preprocessing.{JarFile, MavenDownloader, PomFile}
6+
import de.upb.cs.swt.delphi.crawler.preprocessing.{MavenDownloader, PomFile}
77
import org.apache.maven.artifact.repository.metadata.io.xpp3.MetadataXpp3Reader
88
import org.apache.maven.artifact.versioning.ComparableVersion
9-
import org.apache.maven.model.Dependency
9+
import org.apache.maven.model.{Dependency, Model}
1010
import org.apache.maven.model.io.xpp3.MavenXpp3Reader
1111

1212
import scala.collection.JavaConverters._
13+
import scala.util.Try
1314

1415
/*
1516
* This class extracts the dependencies for a project from a POM file.
@@ -26,79 +27,75 @@ class MavenDependencyActor(configuration: Configuration) extends Actor with Acto
2627

2728
override def receive: Receive = {
2829
case pf: PomFile =>
29-
try {
30-
sender() ! getDependencies(pf)
31-
} catch {
32-
case e: Exception =>{
30+
Try(getDependencies(pf)) match {
31+
case util.Success(dx) => sender() ! dx
32+
case util.Failure(e) =>{
3333
log.warning("Dependency mapper threw exception " + e)
3434
sender() ! akka.actor.Status.Failure(e)
3535
}
3636
}
3737
}
3838

39-
def getDependencies(pomFile: PomFile): Set[MavenIdentifier] = {
40-
val pomObj = pomReader.read(pomFile.is)
41-
pomFile.is.close()
42-
43-
def resolveProperty(str: String) = {
44-
if(str == null || !str.startsWith("$")) {
45-
str
46-
} else {
47-
val extractedVar = str.drop(2).dropRight(1)
48-
val splitVar = extractedVar.split("\\.", 2)
49-
if(splitVar(0) == "project" || splitVar(0) == "pom"){
50-
val evaluatedVar = splitVar(1) match {
51-
case "groupId" => pomObj.getGroupId
52-
case "artifactId" => pomObj.getArtifactId
53-
case "version" => pomObj.getVersion
54-
case _ => null
55-
}
56-
evaluatedVar
57-
} else {
58-
val evaluatedVar = pomObj.getProperties.getProperty(extractedVar)
59-
evaluatedVar
39+
def resolveProperty(str: String, pomObj: Model): Option[String] = {
40+
if (str == null || !str.startsWith("$")){
41+
Option(str)
42+
} else {
43+
val extractedVar = str.drop(2).dropRight(1)
44+
val splitVar = extractedVar.split("\\.", 2)
45+
if (splitVar(0) == "project" || splitVar(0) == "pom") {
46+
val evaluatedVar = splitVar(1) match {
47+
case "groupId" => Some(pomObj.getGroupId)
48+
case "artifactId" => Some(pomObj.getArtifactId)
49+
case "version" => Some(pomObj.getVersion)
50+
case _ => None
6051
}
52+
evaluatedVar
53+
} else {
54+
val evaluatedVar = pomObj.getProperties.getProperty(extractedVar)
55+
Option(evaluatedVar)
6156
}
6257
}
58+
}
6359

64-
def resolveIdentifier(d: Dependency): MavenIdentifier = {
65-
val groupId = resolveProperty(d.getGroupId)
66-
val artifactId = resolveProperty(d.getArtifactId)
60+
def resolveIdentifier(d: Dependency, pomObj: Model): Try[MavenIdentifier] = {
61+
Try {
62+
val groupId = resolveProperty(d.getGroupId, pomObj).getOrElse(throw new Exception)
63+
val artifactId = resolveProperty(d.getArtifactId, pomObj).getOrElse(throw new Exception)
64+
val versionSpec = resolveProperty(d.getVersion, pomObj).getOrElse(throw new Exception)
6765

68-
if (groupId == null || artifactId == null) {
69-
MavenIdentifier(null, null, null, null)
70-
} else {
71-
val versionSpec = resolveProperty(d.getVersion)
72-
val tempId = MavenIdentifier(configuration.mavenRepoBase.toString, groupId, artifactId, versionSpec)
66+
val tempId = MavenIdentifier(configuration.mavenRepoBase.toString, groupId, artifactId, versionSpec)
7367

74-
try {
75-
val downloader = new MavenDownloader(tempId)
76-
val metaFile = downloader.downloadMeta
77-
val metaObj = metaReader.read(metaFile.is)
78-
metaFile.is.close
79-
val versionList = metaObj.getVersioning.getVersions.asScala
80-
if (versionList.contains(versionSpec) || versionSpec == null) {
81-
tempId
82-
} else {
83-
val versionComp = new ComparableVersion(versionSpec)
84-
val versionNum = versionList.indexWhere(v => new ComparableVersion(v).compareTo(versionComp) >= 0)
85-
val version = if (versionNum == -1) versionList.last else if (versionNum == 0) versionList.head else versionList(versionNum - 1)
86-
MavenIdentifier(configuration.mavenRepoBase.toString, groupId, artifactId, version)
87-
}
88-
} catch {
89-
case e: java.io.FileNotFoundException => {
90-
log.warning("Dependency {}:{} does not exist", groupId, artifactId)
91-
MavenIdentifier(null, null, null, null)
92-
}
68+
val downloader = new MavenDownloader(tempId)
69+
val metaFile = downloader.downloadMeta()
70+
val metaObj = metaReader.read(metaFile.is)
71+
metaFile.is.close()
72+
val versionList = metaObj.getVersioning.getVersions.asScala
73+
if (versionList.contains(versionSpec) || versionSpec == null) {
74+
tempId
75+
} else {
76+
val versionComp = new ComparableVersion(versionSpec)
77+
val versionNum = versionList.indexWhere(v => new ComparableVersion(v).compareTo(versionComp) >= 0)
78+
val version = versionNum match {
79+
case -1 => versionList.lastOption
80+
case 0 => versionList.headOption
81+
case _ => Option(versionList(versionNum - 1))
82+
}
83+
version match {
84+
case Some(v) =>
85+
MavenIdentifier(configuration.mavenRepoBase.toString, groupId, artifactId, v)
86+
case None =>
87+
throw new Exception
9388
}
9489
}
9590
}
91+
}
92+
93+
def getDependencies(pomFile: PomFile): Set[MavenIdentifier] = {
94+
val pomObj = pomReader.read(pomFile.is)
95+
pomFile.is.close()
9696

97-
val pomSet = pomObj.getDependencies()
98-
.asScala.toSet[Dependency].map(resolveIdentifier).filter(
99-
m => !(m.version == null || m.groupId == null || m.artifactId == null)
100-
)
101-
pomSet
97+
val pomSet = pomObj.getDependencies.asScala.toSet[Dependency].map(resolveIdentifier(_, pomObj))
98+
for (util.Success(id) <- pomSet) yield id
10299
}
103100
}
104101

0 commit comments

Comments
 (0)