Added an approach to differ tests produced by Fuzzer and SymbolicExecutor #581 (#583)

amandelpie · web-flow · commit 71611c3cd231 · 2022-07-26T11:28:52.000Z
* Handled the bad case with the empty paths

* Added the ability to differ tests produced by fuzzing and symbolic engine

* Fixed a bug
diff --git a/utbot-framework-api/src/main/kotlin/org/utbot/framework/plugin/api/Api.kt b/utbot-framework-api/src/main/kotlin/org/utbot/framework/plugin/api/Api.kt
@@ -139,6 +139,8 @@ sealed class UtResult
  * - static fields changed during execution;
  * - required instrumentation details (such as randoms, time, static methods).
  * - coverage information (instructions) if this execution was obtained from the concrete execution.
+ * - the engine type that created this execution.
+ * - comments, method names and display names created by utbot-summary module.
  */
 data class UtExecution(
     val stateBefore: EnvironmentModels,
@@ -148,6 +150,7 @@ data class UtExecution(
     val path: MutableList<Step>,
     val fullPath: List<Step>,
     val coverage: Coverage? = null,
+    val createdBy: UtExecutionCreator? = null,
     var summary: List<DocStatement>? = null,
     var testMethodName: String? = null,
     var displayName: String? = null,
@@ -1172,21 +1175,28 @@ private fun StringBuilder.appendOptional(name: String, value: Map<*, *>) {
 }
 
 /**
- * Entity that represents cluster information that should appear in the comment
+ * Enum that represents different type of engines that produce tests.
+ */
+enum class UtExecutionCreator {
+    FUZZER, SYMBOLIC_ENGINE
+}
+
+/**
+ * Entity that represents cluster information that should appear in the comment.
  */
 data class UtClusterInfo(
     val header: String? = null,
     val content: String? = null
 )
 
 /**
- * Entity that represents cluster of executions
+ * Entity that represents cluster of executions.
  */
 data class UtExecutionCluster(val clusterInfo: UtClusterInfo, val executions: List<UtExecution>)
 
 
 /**
- * Entity that represents various types of statements in comments
+ * Entity that represents various types of statements in comments.
  */
 sealed class DocStatement
 
diff --git a/utbot-framework/src/main/kotlin/org/utbot/engine/UtBotSymbolicEngine.kt b/utbot-framework/src/main/kotlin/org/utbot/engine/UtBotSymbolicEngine.kt
@@ -68,6 +68,7 @@ import org.utbot.framework.plugin.api.UtAssembleModel
 import org.utbot.framework.plugin.api.UtConcreteExecutionFailure
 import org.utbot.framework.plugin.api.UtError
 import org.utbot.framework.plugin.api.UtExecution
+import org.utbot.framework.plugin.api.UtExecutionCreator
 import org.utbot.framework.plugin.api.UtInstrumentation
 import org.utbot.framework.plugin.api.UtMethod
 import org.utbot.framework.plugin.api.UtNullModel
@@ -304,7 +305,8 @@ class UtBotSymbolicEngine(
                                 instrumentation,
                                 mutableListOf(),
                                 listOf(),
-                                concreteExecutionResult.coverage
+                                concreteExecutionResult.coverage,
+                                UtExecutionCreator.SYMBOLIC_ENGINE
                             )
                             emit(concreteUtExecution)
 
@@ -487,6 +489,7 @@ class UtBotSymbolicEngine(
                         path = mutableListOf(),
                         fullPath = emptyList(),
                         coverage = concreteExecutionResult.coverage,
+                        createdBy = UtExecutionCreator.FUZZER,
                         testMethodName = testMethodName?.testName,
                         displayName = testMethodName?.takeIf { hasMethodUnderTestParametersToFuzz }?.displayName
                     )
@@ -511,7 +514,8 @@ class UtBotSymbolicEngine(
             result = UtConcreteExecutionFailure(e),
             instrumentation = emptyList(),
             path = mutableListOf(),
-            fullPath = listOf()
+            fullPath = listOf(),
+            createdBy = UtExecutionCreator.SYMBOLIC_ENGINE,
         )
 
         emit(failedConcreteExecution)
@@ -546,12 +550,13 @@ class UtBotSymbolicEngine(
         require(stateBefore.parameters.size == stateAfter.parameters.size)
 
         val symbolicUtExecution = UtExecution(
-            stateBefore,
-            stateAfter,
-            symbolicExecutionResult,
-            instrumentation,
-            entryMethodPath(state),
-            state.fullPath()
+            stateBefore = stateBefore,
+            stateAfter = stateAfter,
+            result = symbolicExecutionResult,
+            instrumentation = instrumentation,
+            path = entryMethodPath(state),
+            fullPath = state.fullPath(),
+            createdBy = UtExecutionCreator.SYMBOLIC_ENGINE,
         )
 
         globalGraph.traversed(state)
diff --git a/utbot-framework/src/main/kotlin/org/utbot/external/api/UtBotJavaApi.kt b/utbot-framework/src/main/kotlin/org/utbot/external/api/UtBotJavaApi.kt
@@ -18,6 +18,7 @@ import org.utbot.framework.plugin.api.MockFramework
 import org.utbot.framework.plugin.api.MockStrategyApi
 import org.utbot.framework.plugin.api.TestCaseGenerator
 import org.utbot.framework.plugin.api.UtExecution
+import org.utbot.framework.plugin.api.UtExecutionCreator
 import org.utbot.framework.plugin.api.UtMethod
 import org.utbot.framework.plugin.api.UtPrimitiveModel
 import org.utbot.framework.plugin.api.UtMethodTestSet
@@ -240,12 +241,12 @@ object UtBotJavaApi {
         }
 
         val utExecution = UtExecution(
-            testInfo.initialState,
-            testInfo.initialState, // it seems ok for concrete execution
-            utExecutionResult,
-            emptyList(),
-            mutableListOf(),
-            listOf()
+            stateBefore = testInfo.initialState,
+            stateAfter = testInfo.initialState, // it seems ok for concrete execution
+            result = utExecutionResult,
+            instrumentation = emptyList(),
+            path = mutableListOf(),
+            fullPath = listOf()
         )
 
         val utMethod = UtMethod(methodCallable, containingClass.kotlin)
diff --git a/utbot-summary/src/main/kotlin/org/utbot/summary/Summarization.kt b/utbot-summary/src/main/kotlin/org/utbot/summary/Summarization.kt
@@ -5,6 +5,7 @@ import org.utbot.framework.UtSettings
 import org.utbot.framework.plugin.api.UtClusterInfo
 import org.utbot.framework.plugin.api.UtExecution
 import org.utbot.framework.plugin.api.UtExecutionCluster
+import org.utbot.framework.plugin.api.UtExecutionCreator
 import org.utbot.framework.plugin.api.UtMethodTestSet
 import org.utbot.instrumentation.instrumentation.instrumenter.Instrumenter
 import org.utbot.summary.SummarySentenceConstants.NEW_LINE
@@ -75,29 +76,50 @@ class Summarization(val sourceFile: File?, val invokeDescriptions: List<InvokeDe
         val updatedExecutions = mutableListOf<UtExecution>()
         val clustersToReturn = mutableListOf<UtExecutionCluster>()
 
-        // TODO: Now it excludes tests generated by Fuzzer, handle it properly, related to the https://github.com/UnitTestBot/UTBotJava/issues/428
-        val executionsProducedByFuzzer = getExecutionsWithEmptyPath(testSet)
+        // handles tests produced by fuzzing
+        val executionsProducedByFuzzer = testSet.executions.filter { it.createdBy == UtExecutionCreator.FUZZER }
 
         if (executionsProducedByFuzzer.isNotEmpty()) {
             executionsProducedByFuzzer.forEach {
                 logger.info {
-                    "The path for test ${it.testMethodName} " +
+                    "Test is created by Fuzzing. The path for test ${it.testMethodName} " +
                             "for method ${testSet.method.clazz.qualifiedName} is empty and summaries could not be generated."
                 }
             }
 
             clustersToReturn.add(
                 UtExecutionCluster(
-                    UtClusterInfo(),
+                    UtClusterInfo(), // TODO: add something https://github.com/UnitTestBot/UTBotJava/issues/430
                     executionsProducedByFuzzer
                 )
             )
         }
 
+        // handles tests produced by symbolic engine, but with empty paths
+        val executionsWithEmptyPaths = getExecutionsCreatedBySymbolicEngineWithEmptyPath(testSet)
+
+        if (executionsWithEmptyPaths.isNotEmpty()) {
+            executionsWithEmptyPaths.forEach {
+                logger.info {
+                    "Test is created by Symbolic Engine. The path for test ${it.testMethodName} " +
+                            "for method ${testSet.method.clazz.qualifiedName} is empty and summaries could not be generated."
+                }
+            }
+
+            clustersToReturn.add(
+                UtExecutionCluster(
+                    UtClusterInfo(), // TODO: https://github.com/UnitTestBot/UTBotJava/issues/430
+                    executionsWithEmptyPaths
+                )
+            )
+        }
+
+        val testSetForAnalysis = prepareTestSetForByteCodeAnalysis(testSet)
+
         // analyze
         if (jimpleBody != null && sootToAST != null) {
             val methodUnderTest = jimpleBody.method
-            val clusteredTags = tagGenerator.testSetToTags(testSet)
+            val clusteredTags = tagGenerator.testSetToTags(testSetForAnalysis)
             jimpleBodyAnalysis.traceStructuralAnalysis(jimpleBody, clusteredTags, methodUnderTest, invokeDescriptions)
             val numberOfSuccessfulClusters = clusteredTags.filter { it.isSuccessful }.size
             for (clusterTraceTags in clusteredTags) {
@@ -108,21 +130,22 @@ class Summarization(val sourceFile: File?, val invokeDescriptions: List<InvokeDe
                     && clusterTraceTags.traceTags.size > 1 // add if there is more than 1 execution
                 ) {
                     SimpleClusterCommentBuilder(clusterTraceTags.commonStepsTraceTag, sootToAST)
-                            .buildString(methodUnderTest)
-                            .takeIf { it.isNotBlank() }
-                            ?.let {
-                                buildString {
-                                    append("${NEW_LINE}Common steps:")
-                                    append("$NEW_LINE$it")
-                                }
+                        .buildString(methodUnderTest)
+                        .takeIf { it.isNotBlank() }
+                        ?.let {
+                            buildString {
+                                append("${NEW_LINE}Common steps:")
+                                append("$NEW_LINE$it")
                             }
+                        }
                 } else {
                     null
                 }
 
                 for (traceTags in clusterTraceTags.traceTags) {
                     if (GENERATE_COMMENTS) {
-                        traceTags.execution.summary = SimpleCommentBuilder(traceTags, sootToAST).buildDocStmts(methodUnderTest)
+                        traceTags.execution.summary =
+                            SimpleCommentBuilder(traceTags, sootToAST).buildDocStmts(methodUnderTest)
                     }
 
                     if (GENERATE_DISPLAY_NAMES || GENERATE_NAMES) {
@@ -164,8 +187,21 @@ class Summarization(val sourceFile: File?, val invokeDescriptions: List<InvokeDe
         return listOf(UtExecutionCluster(UtClusterInfo(), testSet.executions))
     }
 
-    private fun getExecutionsWithEmptyPath(testSet: UtMethodTestSet) =
-        testSet.executions.filter { it.path.isEmpty() }
+    private fun prepareTestSetForByteCodeAnalysis(testSet: UtMethodTestSet): UtMethodTestSet {
+        val executions =
+            testSet.executions.filterNot { it.createdBy == UtExecutionCreator.FUZZER || (it.createdBy == UtExecutionCreator.SYMBOLIC_ENGINE && it.path.isEmpty()) }
+
+        return UtMethodTestSet(
+            method = testSet.method,
+            executions = executions,
+            jimpleBody = testSet.jimpleBody,
+            errors = testSet.errors,
+            clustersInfo = testSet.clustersInfo
+        )
+    }
+
+    private fun getExecutionsCreatedBySymbolicEngineWithEmptyPath(testSet: UtMethodTestSet) =
+        testSet.executions.filter { it.createdBy == UtExecutionCreator.SYMBOLIC_ENGINE && it.path.isEmpty() }
 
     /*
     * asts of invokes also included
diff --git a/utbot-summary/src/main/kotlin/org/utbot/summary/TagGenerator.kt b/utbot-summary/src/main/kotlin/org/utbot/summary/TagGenerator.kt
@@ -95,7 +95,7 @@ private fun generateExecutionTags(executions: List<UtExecution>, splitSteps: Spl
  * @return clustered executions
  */
 private fun toClusterExecutions(testSet: UtMethodTestSet): List<ExecutionCluster> {
-    val methodExecutions = testSet.executions.filter { it.path.isNotEmpty() } // TODO: Now it excludes tests generated by Fuzzer, handle it properly, related to the https://github.com/UnitTestBot/UTBotJava/issues/428
+    val methodExecutions = testSet.executions
     val clusters = mutableListOf<ExecutionCluster>()
     val commentPostfix = "for method ${testSet.method.displayName}"
 
