From 5259d5b0b63e59baa8d22bd39930b617ead64ab3 Mon Sep 17 00:00:00 2001 From: Ian Botsford <83236726+ianbotsf@users.noreply.github.com> Date: Fri, 28 Mar 2025 22:17:31 +0000 Subject: [PATCH 1/3] chore: reintegrate E2E smoke tests into the top-level Gradle build; modularize smoke test runners to improve testability --- .../kotlin/codegen/core/RuntimeTypes.kt | 1 + .../smithy/kotlin/codegen/lang/KotlinTypes.kt | 1 + .../smoketests/SmokeTestsRunnerGenerator.kt | 86 +++++++++------- .../SmokeTestsRunnerGeneratorTest.kt | 99 ++++++++++++------- runtime/runtime-core/api/runtime-core.api | 1 + .../runtime/smoketests/SmokeTestsFunctions.kt | 14 +++ 6 files changed, 131 insertions(+), 71 deletions(-) diff --git a/codegen/smithy-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/core/RuntimeTypes.kt b/codegen/smithy-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/core/RuntimeTypes.kt index b6341e0443..3d59c8d1e9 100644 --- a/codegen/smithy-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/core/RuntimeTypes.kt +++ b/codegen/smithy-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/core/RuntimeTypes.kt @@ -115,6 +115,7 @@ object RuntimeTypes { } object SmokeTests : RuntimeTypePackage(KotlinDependency.CORE, "smoketests") { + val DefaultPrinter = symbol("DefaultPrinter") val exitProcess = symbol("exitProcess") val printExceptionStackTrace = symbol("printExceptionStackTrace") val SmokeTestsException = symbol("SmokeTestsException") diff --git a/codegen/smithy-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/lang/KotlinTypes.kt b/codegen/smithy-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/lang/KotlinTypes.kt index 242ba4d0da..6d2384584e 100644 --- a/codegen/smithy-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/lang/KotlinTypes.kt +++ b/codegen/smithy-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/lang/KotlinTypes.kt @@ -109,6 +109,7 @@ object KotlinTypes { } object Text : RuntimeTypePackage(KotlinDependency.KOTLIN_STDLIB, "text") { + val Appendable = stdlibSymbol("Appendable") val encodeToByteArray = stdlibSymbol("encodeToByteArray") } diff --git a/codegen/smithy-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/rendering/smoketests/SmokeTestsRunnerGenerator.kt b/codegen/smithy-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/rendering/smoketests/SmokeTestsRunnerGenerator.kt index df8eeda53e..98bb4d1370 100644 --- a/codegen/smithy-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/rendering/smoketests/SmokeTestsRunnerGenerator.kt +++ b/codegen/smithy-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/rendering/smoketests/SmokeTestsRunnerGenerator.kt @@ -4,6 +4,7 @@ import software.amazon.smithy.codegen.core.Symbol import software.amazon.smithy.kotlin.codegen.core.* import software.amazon.smithy.kotlin.codegen.integration.SectionId import software.amazon.smithy.kotlin.codegen.integration.SectionKey +import software.amazon.smithy.kotlin.codegen.lang.KotlinTypes import software.amazon.smithy.kotlin.codegen.model.getTrait import software.amazon.smithy.kotlin.codegen.model.hasTrait import software.amazon.smithy.kotlin.codegen.rendering.ShapeValueGenerator @@ -17,8 +18,8 @@ import software.amazon.smithy.kotlin.codegen.rendering.util.format import software.amazon.smithy.kotlin.codegen.utils.dq import software.amazon.smithy.kotlin.codegen.utils.toCamelCase import software.amazon.smithy.kotlin.codegen.utils.topDownOperations -import software.amazon.smithy.model.node.* -import software.amazon.smithy.model.shapes.* +import software.amazon.smithy.model.node.Node +import software.amazon.smithy.model.shapes.OperationShape import software.amazon.smithy.smoketests.traits.SmokeTestCase import software.amazon.smithy.smoketests.traits.SmokeTestsTrait import kotlin.jvm.optionals.getOrNull @@ -61,25 +62,47 @@ class SmokeTestsRunnerGenerator( ) { internal fun render() { writer.declareSection(SmokeTestSectionIds.SmokeTestsFile) { - writer.write("private var exitCode = 0") + write("") + + withBlock("public suspend fun main() {", "}") { + write("val success = SmokeTestRunner().runAllTests()") + withBlock("if (!success) {", "}") { + write("#T(1)", RuntimeTypes.Core.SmokeTests.exitProcess) + } + } + write("") + renderRunnerClass() + } + } + + private fun renderRunnerClass() { + writer.withBlock( + "public class SmokeTestRunner(private val platform: #1T = #1T.System, private val printer: #2T = #3T) {", + "}", + RuntimeTypes.Core.Utils.PlatformProvider, + KotlinTypes.Text.Appendable, + RuntimeTypes.Core.SmokeTests.DefaultPrinter, + ) { renderEnvironmentVariables() - writer.declareSection(SmokeTestSectionIds.AdditionalEnvironmentVariables) - writer.write("") - writer.withBlock("public suspend fun main() {", "}") { - renderFunctionCalls() - write("#T(exitCode)", RuntimeTypes.Core.SmokeTests.exitProcess) + declareSection(SmokeTestSectionIds.AdditionalEnvironmentVariables) + write("") + + withBlock("public suspend fun runAllTests(): Boolean =", "") { + withBlock("listOf Boolean>(", ")") { + renderFunctionReferences() + } + indent() + write(".map { it() }") + write(".none { !it }") + dedent() } - writer.write("") renderFunctions() } } private fun renderEnvironmentVariables() { // Skip tags - writer.writeInline( - "private val skipTags = #T.System.getenv(", - RuntimeTypes.Core.Utils.PlatformProvider, - ) + writer.writeInline("private val skipTags = platform.getenv(") writer.declareSection(SmokeTestSectionIds.SkipTags) { writer.writeInline("#S", SKIP_TAGS) } @@ -89,10 +112,7 @@ class SmokeTestsRunnerGenerator( ) // Service filter - writer.writeInline( - "private val serviceFilter = #T.System.getenv(", - RuntimeTypes.Core.Utils.PlatformProvider, - ) + writer.writeInline("private val serviceFilter = platform.getenv(") writer.declareSection(SmokeTestSectionIds.ServiceFilter) { writer.writeInline("#S", SERVICE_FILTER) } @@ -102,10 +122,10 @@ class SmokeTestsRunnerGenerator( ) } - private fun renderFunctionCalls() { + private fun renderFunctionReferences() { operations.forEach { operation -> operation.getTrait()?.testCases?.forEach { testCase -> - writer.write("${testCase.functionName}()") + writer.write("::${testCase.functionName},") } } } @@ -120,7 +140,7 @@ class SmokeTestsRunnerGenerator( } private fun renderFunction(operation: OperationShape, testCase: SmokeTestCase) { - writer.withBlock("private suspend fun ${testCase.functionName}() {", "}") { + writer.withBlock("private suspend fun ${testCase.functionName}(): Boolean {", "}") { write("val tags = setOf(${testCase.tags.joinToString(",") { it.dq()} })") writer.withBlock("if ((serviceFilter.isNotEmpty() && #S !in serviceFilter) || tags.any { it in skipTags }) {", "}", sdkId) { printTestResult( @@ -131,10 +151,10 @@ class SmokeTestsRunnerGenerator( "ok", "# skip", ) - writer.write("return") + writer.write("return true") } write("") - withInlineBlock("try {", "} ") { + withInlineBlock("return try {", "} ") { renderTestCase(operation, testCase) } withBlock("catch (exception: Exception) {", "}") { @@ -149,6 +169,8 @@ class SmokeTestsRunnerGenerator( closeAndOpenBlock("}.#T { client ->", RuntimeTypes.Core.IO.use) renderOperation(operation, testCase) } + writer.write("") + writer.write("error(#S)", "Unexpectedly completed smoke test operation without throwing exception") } private fun renderClientConfig(testCase: SmokeTestCase) { @@ -212,9 +234,11 @@ class SmokeTestsRunnerGenerator( ) writer.withBlock("if (!success) {", "}") { - write("#T(exception)", RuntimeTypes.Core.SmokeTests.printExceptionStackTrace) - write("exitCode = 1") + write("printer.appendLine(exception.stackTraceToString().prependIndent(#S))", "#") } + + writer.write("") + writer.write("success") } // Helpers @@ -241,7 +265,7 @@ class SmokeTestsRunnerGenerator( val expectation = if (errorExpected) "error expected from service" else "no error expected from service" val status = statusOverride ?: "\$status" val testResult = "$status $service $testCase - $expectation $directive" - writer.write("println(#S)", testResult) + writer.write("printer.appendLine(#S)", testResult) } /** @@ -250,18 +274,6 @@ class SmokeTestsRunnerGenerator( private val SmokeTestCase.functionName: String get() = this.id.toCamelCase() - /** - * Get the operation parameters for a [SmokeTestCase] - */ - private val SmokeTestCase.operationParameters: Map - get() = this.params.get().members - - /** - * Checks if there are operation parameters for a [SmokeTestCase] - */ - private val SmokeTestCase.hasOperationParameters: Boolean - get() = this.params.isPresent - /** * Check if a [SmokeTestCase] is expecting a specific error */ diff --git a/codegen/smithy-kotlin-codegen/src/test/kotlin/software/amazon/smithy/kotlin/codegen/rendering/smoketests/SmokeTestsRunnerGeneratorTest.kt b/codegen/smithy-kotlin-codegen/src/test/kotlin/software/amazon/smithy/kotlin/codegen/rendering/smoketests/SmokeTestsRunnerGeneratorTest.kt index 50291a482f..c07a3bd604 100644 --- a/codegen/smithy-kotlin-codegen/src/test/kotlin/software/amazon/smithy/kotlin/codegen/rendering/smoketests/SmokeTestsRunnerGeneratorTest.kt +++ b/codegen/smithy-kotlin-codegen/src/test/kotlin/software/amazon/smithy/kotlin/codegen/rendering/smoketests/SmokeTestsRunnerGeneratorTest.kt @@ -74,10 +74,9 @@ class SmokeTestsRunnerGeneratorTest { fun variablesTest() { generatedCode.shouldContainOnlyOnceWithDiff( """ - private var exitCode = 0 - private val skipTags = PlatformProvider.System.getenv("SMOKE_TEST_SKIP_TAGS")?.let { it.split(",").map { it.trim() }.toSet() } ?: emptySet() - private val serviceFilter = PlatformProvider.System.getenv("SMOKE_TEST_SERVICE_IDS")?.let { it.split(",").map { it.trim() }.toSet() } - """.trimIndent(), + private val skipTags = platform.getenv("SMOKE_TEST_SKIP_TAGS")?.let { it.split(",").map { it.trim() }.toSet() } ?: emptySet() + private val serviceFilter = platform.getenv("SMOKE_TEST_SERVICE_IDS")?.let { it.split(",").map { it.trim() }.toSet() } + """.formatForTest(), ) } @@ -86,27 +85,50 @@ class SmokeTestsRunnerGeneratorTest { generatedCode.shouldContainOnlyOnceWithDiff( """ public suspend fun main() { - successTest() - invalidMessageErrorTest() - failureTest() - exitProcess(exitCode) + val success = SmokeTestRunner().runAllTests() + if (!success) { + exitProcess(1) + } } """.trimIndent(), ) } + @Test + fun runnerClassTest() { + generatedCode.shouldContainOnlyOnceWithDiff( + "public class SmokeTestRunner(private val platform: PlatformProvider = PlatformProvider.System, private val printer: Appendable = DefaultPrinter) {", + ) + } + + @Test + fun runAllTestsTest() { + generatedCode.shouldContainOnlyOnceWithDiff( + """ + public suspend fun runAllTests(): Boolean = + listOf Boolean>( + ::successTest, + ::invalidMessageErrorTest, + ::failureTest, + ) + .map { it() } + .none { !it } + """.formatForTest() + ) + } + @Test fun successTest() { generatedCode.shouldContainOnlyOnceWithDiff( """ - private suspend fun successTest() { + private suspend fun successTest(): Boolean { val tags = setOf("success") if ((serviceFilter.isNotEmpty() && "Test" !in serviceFilter) || tags.any { it in skipTags }) { - println("ok Test SuccessTest - no error expected from service # skip") - return + printer.appendLine("ok Test SuccessTest - no error expected from service # skip") + return true } - - try { + + return try { TestClient { interceptors.add(SmokeTestsInterceptor()) region = "eu-central-1" @@ -118,18 +140,21 @@ class SmokeTestsRunnerGeneratorTest { } ) } - + + error("Unexpectedly completed smoke test operation without throwing exception") + } catch (exception: Exception) { val success: Boolean = exception is SmokeTestsSuccessException val status: String = if (success) "ok" else "not ok" - println("${'$'}status Test SuccessTest - no error expected from service ") + printer.appendLine("${'$'}status Test SuccessTest - no error expected from service ") if (!success) { - printExceptionStackTrace(exception) - exitCode = 1 + printer.appendLine(exception.stackTraceToString().prependIndent("#")) } + + success } } - """.trimIndent(), + """.formatForTest(), ) } @@ -137,14 +162,14 @@ class SmokeTestsRunnerGeneratorTest { fun invalidMessageErrorTest() { generatedCode.shouldContainOnlyOnceWithDiff( """ - private suspend fun invalidMessageErrorTest() { + private suspend fun invalidMessageErrorTest(): Boolean { val tags = setOf() if ((serviceFilter.isNotEmpty() && "Test" !in serviceFilter) || tags.any { it in skipTags }) { - println("ok Test InvalidMessageErrorTest - error expected from service # skip") - return + printer.appendLine("ok Test InvalidMessageErrorTest - error expected from service # skip") + return true } - try { + return try { TestClient { }.use { client -> client.testOperation( @@ -154,17 +179,20 @@ class SmokeTestsRunnerGeneratorTest { ) } + error("Unexpectedly completed smoke test operation without throwing exception") + } catch (exception: Exception) { val success: Boolean = exception is InvalidMessageError val status: String = if (success) "ok" else "not ok" - println("${'$'}status Test InvalidMessageErrorTest - error expected from service ") + printer.appendLine("${'$'}status Test InvalidMessageErrorTest - error expected from service ") if (!success) { - printExceptionStackTrace(exception) - exitCode = 1 + printer.appendLine(exception.stackTraceToString().prependIndent("#")) } + + success } } - """.trimIndent(), + """.formatForTest(), ) } @@ -172,14 +200,14 @@ class SmokeTestsRunnerGeneratorTest { fun failureTest() { generatedCode.shouldContainOnlyOnceWithDiff( """ - private suspend fun failureTest() { + private suspend fun failureTest(): Boolean { val tags = setOf() if ((serviceFilter.isNotEmpty() && "Test" !in serviceFilter) || tags.any { it in skipTags }) { - println("ok Test FailureTest - error expected from service # skip") - return + printer.appendLine("ok Test FailureTest - error expected from service # skip") + return true } - try { + return try { TestClient { interceptors.add(SmokeTestsInterceptor()) }.use { client -> @@ -190,17 +218,20 @@ class SmokeTestsRunnerGeneratorTest { ) } + error("Unexpectedly completed smoke test operation without throwing exception") + } catch (exception: Exception) { val success: Boolean = exception is SmokeTestsFailureException val status: String = if (success) "ok" else "not ok" - println("${'$'}status Test FailureTest - error expected from service ") + printer.appendLine("${'$'}status Test FailureTest - error expected from service ") if (!success) { - printExceptionStackTrace(exception) - exitCode = 1 + printer.appendLine(exception.stackTraceToString().prependIndent("#")) } + + success } } - """.trimIndent(), + """.formatForTest(), ) } diff --git a/runtime/runtime-core/api/runtime-core.api b/runtime/runtime-core/api/runtime-core.api index 3c8fbfa2bd..21b70b9b5c 100644 --- a/runtime/runtime-core/api/runtime-core.api +++ b/runtime/runtime-core/api/runtime-core.api @@ -2093,6 +2093,7 @@ public final class aws/smithy/kotlin/runtime/smoketests/SmokeTestsFunctionsJVMKt } public final class aws/smithy/kotlin/runtime/smoketests/SmokeTestsFunctionsKt { + public static final fun getDefaultPrinter ()Ljava/lang/Appendable; public static final fun printExceptionStackTrace (Ljava/lang/Exception;)V } diff --git a/runtime/runtime-core/common/src/aws/smithy/kotlin/runtime/smoketests/SmokeTestsFunctions.kt b/runtime/runtime-core/common/src/aws/smithy/kotlin/runtime/smoketests/SmokeTestsFunctions.kt index 682817193d..3da3bb8c23 100644 --- a/runtime/runtime-core/common/src/aws/smithy/kotlin/runtime/smoketests/SmokeTestsFunctions.kt +++ b/runtime/runtime-core/common/src/aws/smithy/kotlin/runtime/smoketests/SmokeTestsFunctions.kt @@ -15,7 +15,21 @@ public expect fun exitProcess(status: Int): Nothing * # at executors.JavaRunnerExecutor$Companion.main(JavaRunnerExecutor.kt:27) * # at executors.JavaRunnerExecutor.main(JavaRunnerExecutor.kt) */ +@Deprecated( + message = "No longer used, target for removal in 1.5", + replaceWith = ReplaceWith("println(exception.stackTraceToString().prependIndent(\"#\"))"), + level = DeprecationLevel.WARNING, +) public fun printExceptionStackTrace(exception: Exception): Unit = println(exception.stackTraceToString().split("\n").joinToString("\n") { "#$it" }) public class SmokeTestsException(message: String) : Exception(message) + +/** + * An [Appendable] which can be used for printing test results to the console + */ +public val DefaultPrinter: Appendable = object : Appendable { + override fun append(c: Char) = this.also { print(c) } + override fun append(csq: CharSequence?) = this.also { print(csq) } + override fun append(csq: CharSequence?, start: Int, end: Int) = this.also { print(csq?.subSequence(start, end)) } +} From ade7c3a78a2ceca33dd4608bd7b9a352ef3a11ee Mon Sep 17 00:00:00 2001 From: Ian Botsford <83236726+ianbotsf@users.noreply.github.com> Date: Tue, 1 Apr 2025 23:01:44 +0000 Subject: [PATCH 2/3] lint --- .../rendering/smoketests/SmokeTestsRunnerGeneratorTest.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codegen/smithy-kotlin-codegen/src/test/kotlin/software/amazon/smithy/kotlin/codegen/rendering/smoketests/SmokeTestsRunnerGeneratorTest.kt b/codegen/smithy-kotlin-codegen/src/test/kotlin/software/amazon/smithy/kotlin/codegen/rendering/smoketests/SmokeTestsRunnerGeneratorTest.kt index c07a3bd604..cf21f8c66d 100644 --- a/codegen/smithy-kotlin-codegen/src/test/kotlin/software/amazon/smithy/kotlin/codegen/rendering/smoketests/SmokeTestsRunnerGeneratorTest.kt +++ b/codegen/smithy-kotlin-codegen/src/test/kotlin/software/amazon/smithy/kotlin/codegen/rendering/smoketests/SmokeTestsRunnerGeneratorTest.kt @@ -113,7 +113,7 @@ class SmokeTestsRunnerGeneratorTest { ) .map { it() } .none { !it } - """.formatForTest() + """.formatForTest(), ) } From 4337c2355f16179e672cf8bc0911594711b1fdde Mon Sep 17 00:00:00 2001 From: Ian Botsford <83236726+ianbotsf@users.noreply.github.com> Date: Wed, 2 Apr 2025 22:05:12 +0000 Subject: [PATCH 3/3] simplifying boolean logic --- .../rendering/smoketests/SmokeTestsRunnerGenerator.kt | 4 ++-- .../rendering/smoketests/SmokeTestsRunnerGeneratorTest.kt | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/codegen/smithy-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/rendering/smoketests/SmokeTestsRunnerGenerator.kt b/codegen/smithy-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/rendering/smoketests/SmokeTestsRunnerGenerator.kt index 98bb4d1370..a3124d7bff 100644 --- a/codegen/smithy-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/rendering/smoketests/SmokeTestsRunnerGenerator.kt +++ b/codegen/smithy-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/rendering/smoketests/SmokeTestsRunnerGenerator.kt @@ -93,7 +93,7 @@ class SmokeTestsRunnerGenerator( } indent() write(".map { it() }") - write(".none { !it }") + write(".all { it }") dedent() } renderFunctions() @@ -234,7 +234,7 @@ class SmokeTestsRunnerGenerator( ) writer.withBlock("if (!success) {", "}") { - write("printer.appendLine(exception.stackTraceToString().prependIndent(#S))", "#") + write("printer.appendLine(exception.stackTraceToString().prependIndent(#S))", "# ") } writer.write("") diff --git a/codegen/smithy-kotlin-codegen/src/test/kotlin/software/amazon/smithy/kotlin/codegen/rendering/smoketests/SmokeTestsRunnerGeneratorTest.kt b/codegen/smithy-kotlin-codegen/src/test/kotlin/software/amazon/smithy/kotlin/codegen/rendering/smoketests/SmokeTestsRunnerGeneratorTest.kt index cf21f8c66d..c9d4b535c1 100644 --- a/codegen/smithy-kotlin-codegen/src/test/kotlin/software/amazon/smithy/kotlin/codegen/rendering/smoketests/SmokeTestsRunnerGeneratorTest.kt +++ b/codegen/smithy-kotlin-codegen/src/test/kotlin/software/amazon/smithy/kotlin/codegen/rendering/smoketests/SmokeTestsRunnerGeneratorTest.kt @@ -112,7 +112,7 @@ class SmokeTestsRunnerGeneratorTest { ::failureTest, ) .map { it() } - .none { !it } + .all { it } """.formatForTest(), ) } @@ -148,7 +148,7 @@ class SmokeTestsRunnerGeneratorTest { val status: String = if (success) "ok" else "not ok" printer.appendLine("${'$'}status Test SuccessTest - no error expected from service ") if (!success) { - printer.appendLine(exception.stackTraceToString().prependIndent("#")) + printer.appendLine(exception.stackTraceToString().prependIndent("# ")) } success @@ -186,7 +186,7 @@ class SmokeTestsRunnerGeneratorTest { val status: String = if (success) "ok" else "not ok" printer.appendLine("${'$'}status Test InvalidMessageErrorTest - error expected from service ") if (!success) { - printer.appendLine(exception.stackTraceToString().prependIndent("#")) + printer.appendLine(exception.stackTraceToString().prependIndent("# ")) } success @@ -225,7 +225,7 @@ class SmokeTestsRunnerGeneratorTest { val status: String = if (success) "ok" else "not ok" printer.appendLine("${'$'}status Test FailureTest - error expected from service ") if (!success) { - printer.appendLine(exception.stackTraceToString().prependIndent("#")) + printer.appendLine(exception.stackTraceToString().prependIndent("# ")) } success