Skip to content

chore: reintegrate E2E smoke tests into the top-level Gradle build; modularize smoke test runners to improve testability #1262

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Apr 2, 2025
Merged

chore: reintegrate E2E smoke tests into the top-level Gradle build; modularize smoke test runners to improve testability #1262

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
5259d5b
chore: reintegrate E2E smoke tests into the top-level Gradle build; m…
ianbotsf Mar 28, 2025
ade7c3a
lint
ianbotsf Apr 1, 2025
4337c23
simplifying boolean logic
ianbotsf Apr 2, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions ...kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/core/RuntimeTypes.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -115,6 +115,7 @@ object RuntimeTypes {
}

object SmokeTests : RuntimeTypePackage(KotlinDependency.CORE, "smoketests") {
val DefaultPrinter = symbol("DefaultPrinter")
val exitProcess = symbol("exitProcess")
val printExceptionStackTrace = symbol("printExceptionStackTrace")
val SmokeTestsException = symbol("SmokeTestsException")
Expand Down
1 change: 1 addition & 0 deletions ...-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/lang/KotlinTypes.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -109,6 +109,7 @@ object KotlinTypes {
}

object Text : RuntimeTypePackage(KotlinDependency.KOTLIN_STDLIB, "text") {
val Appendable = stdlibSymbol("Appendable")
val encodeToByteArray = stdlibSymbol("encodeToByteArray")
}

Expand Down
86 changes: 49 additions & 37 deletions ...n/software/amazon/smithy/kotlin/codegen/rendering/smoketests/SmokeTestsRunnerGenerator.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import software.amazon.smithy.codegen.core.Symbol
import software.amazon.smithy.kotlin.codegen.core.*
import software.amazon.smithy.kotlin.codegen.integration.SectionId
import software.amazon.smithy.kotlin.codegen.integration.SectionKey
import software.amazon.smithy.kotlin.codegen.lang.KotlinTypes
import software.amazon.smithy.kotlin.codegen.model.getTrait
import software.amazon.smithy.kotlin.codegen.model.hasTrait
import software.amazon.smithy.kotlin.codegen.rendering.ShapeValueGenerator
Expand All @@ -17,8 +18,8 @@ import software.amazon.smithy.kotlin.codegen.rendering.util.format
import software.amazon.smithy.kotlin.codegen.utils.dq
import software.amazon.smithy.kotlin.codegen.utils.toCamelCase
import software.amazon.smithy.kotlin.codegen.utils.topDownOperations
import software.amazon.smithy.model.node.*
import software.amazon.smithy.model.shapes.*
import software.amazon.smithy.model.node.Node
import software.amazon.smithy.model.shapes.OperationShape
import software.amazon.smithy.smoketests.traits.SmokeTestCase
import software.amazon.smithy.smoketests.traits.SmokeTestsTrait
import kotlin.jvm.optionals.getOrNull
Expand Down Expand Up @@ -61,25 +62,47 @@ class SmokeTestsRunnerGenerator(
) {
internal fun render() {
writer.declareSection(SmokeTestSectionIds.SmokeTestsFile) {
writer.write("private var exitCode = 0")
write("")

withBlock("public suspend fun main() {", "}") {
write("val success = SmokeTestRunner().runAllTests()")
withBlock("if (!success) {", "}") {
write("#T(1)", RuntimeTypes.Core.SmokeTests.exitProcess)
}
}
write("")
renderRunnerClass()
}
}

private fun renderRunnerClass() {
writer.withBlock(
"public class SmokeTestRunner(private val platform: #1T = #1T.System, private val printer: #2T = #3T) {",
"}",
RuntimeTypes.Core.Utils.PlatformProvider,
KotlinTypes.Text.Appendable,
RuntimeTypes.Core.SmokeTests.DefaultPrinter,
) {
renderEnvironmentVariables()
writer.declareSection(SmokeTestSectionIds.AdditionalEnvironmentVariables)
writer.write("")
writer.withBlock("public suspend fun main() {", "}") {
renderFunctionCalls()
write("#T(exitCode)", RuntimeTypes.Core.SmokeTests.exitProcess)
declareSection(SmokeTestSectionIds.AdditionalEnvironmentVariables)
write("")

withBlock("public suspend fun runAllTests(): Boolean =", "") {
withBlock("listOf<suspend () -> Boolean>(", ")") {
renderFunctionReferences()
}
indent()
write(".map { it() }")
write(".all { it }")
dedent()
}
writer.write("")
renderFunctions()
}
}

private fun renderEnvironmentVariables() {
// Skip tags
writer.writeInline(
"private val skipTags = #T.System.getenv(",
RuntimeTypes.Core.Utils.PlatformProvider,
)
writer.writeInline("private val skipTags = platform.getenv(")
writer.declareSection(SmokeTestSectionIds.SkipTags) {
writer.writeInline("#S", SKIP_TAGS)
}
Expand All @@ -89,10 +112,7 @@ class SmokeTestsRunnerGenerator(
)

// Service filter
writer.writeInline(
"private val serviceFilter = #T.System.getenv(",
RuntimeTypes.Core.Utils.PlatformProvider,
)
writer.writeInline("private val serviceFilter = platform.getenv(")
writer.declareSection(SmokeTestSectionIds.ServiceFilter) {
writer.writeInline("#S", SERVICE_FILTER)
}
Expand All @@ -102,10 +122,10 @@ class SmokeTestsRunnerGenerator(
)
}

private fun renderFunctionCalls() {
private fun renderFunctionReferences() {
operations.forEach { operation ->
operation.getTrait<SmokeTestsTrait>()?.testCases?.forEach { testCase ->
writer.write("${testCase.functionName}()")
writer.write("::${testCase.functionName},")
}
}
}
Expand All @@ -120,7 +140,7 @@ class SmokeTestsRunnerGenerator(
}

private fun renderFunction(operation: OperationShape, testCase: SmokeTestCase) {
writer.withBlock("private suspend fun ${testCase.functionName}() {", "}") {
writer.withBlock("private suspend fun ${testCase.functionName}(): Boolean {", "}") {
write("val tags = setOf<String>(${testCase.tags.joinToString(",") { it.dq()} })")
writer.withBlock("if ((serviceFilter.isNotEmpty() && #S !in serviceFilter) || tags.any { it in skipTags }) {", "}", sdkId) {
printTestResult(
Expand All @@ -131,10 +151,10 @@ class SmokeTestsRunnerGenerator(
"ok",
"# skip",
)
writer.write("return")
writer.write("return true")
}
write("")
withInlineBlock("try {", "} ") {
withInlineBlock("return try {", "} ") {
renderTestCase(operation, testCase)
}
withBlock("catch (exception: Exception) {", "}") {
Expand All @@ -149,6 +169,8 @@ class SmokeTestsRunnerGenerator(
closeAndOpenBlock("}.#T { client ->", RuntimeTypes.Core.IO.use)
renderOperation(operation, testCase)
}
writer.write("")
writer.write("error(#S)", "Unexpectedly completed smoke test operation without throwing exception")
}

private fun renderClientConfig(testCase: SmokeTestCase) {
Expand Down Expand Up @@ -212,9 +234,11 @@ class SmokeTestsRunnerGenerator(
)

writer.withBlock("if (!success) {", "}") {
write("#T(exception)", RuntimeTypes.Core.SmokeTests.printExceptionStackTrace)
write("exitCode = 1")
write("printer.appendLine(exception.stackTraceToString().prependIndent(#S))", "# ")
}

writer.write("")
writer.write("success")
}

// Helpers
Expand All @@ -241,7 +265,7 @@ class SmokeTestsRunnerGenerator(
val expectation = if (errorExpected) "error expected from service" else "no error expected from service"
val status = statusOverride ?: "\$status"
val testResult = "$status $service $testCase - $expectation $directive"
writer.write("println(#S)", testResult)
writer.write("printer.appendLine(#S)", testResult)
}

/**
Expand All @@ -250,18 +274,6 @@ class SmokeTestsRunnerGenerator(
private val SmokeTestCase.functionName: String
get() = this.id.toCamelCase()

/**
* Get the operation parameters for a [SmokeTestCase]
*/
private val SmokeTestCase.operationParameters: Map<StringNode, Node>
get() = this.params.get().members

/**
* Checks if there are operation parameters for a [SmokeTestCase]
*/
private val SmokeTestCase.hasOperationParameters: Boolean
get() = this.params.isPresent

/**
* Check if a [SmokeTestCase] is expecting a specific error
*/
Expand Down
99 changes: 65 additions & 34 deletions ...ftware/amazon/smithy/kotlin/codegen/rendering/smoketests/SmokeTestsRunnerGeneratorTest.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,10 +74,9 @@ class SmokeTestsRunnerGeneratorTest {
fun variablesTest() {
generatedCode.shouldContainOnlyOnceWithDiff(
"""
private var exitCode = 0
private val skipTags = PlatformProvider.System.getenv("SMOKE_TEST_SKIP_TAGS")?.let { it.split(",").map { it.trim() }.toSet() } ?: emptySet()
private val serviceFilter = PlatformProvider.System.getenv("SMOKE_TEST_SERVICE_IDS")?.let { it.split(",").map { it.trim() }.toSet() }
""".trimIndent(),
private val skipTags = platform.getenv("SMOKE_TEST_SKIP_TAGS")?.let { it.split(",").map { it.trim() }.toSet() } ?: emptySet()
private val serviceFilter = platform.getenv("SMOKE_TEST_SERVICE_IDS")?.let { it.split(",").map { it.trim() }.toSet() }
""".formatForTest(),
)
}

Expand All @@ -86,27 +85,50 @@ class SmokeTestsRunnerGeneratorTest {
generatedCode.shouldContainOnlyOnceWithDiff(
"""
public suspend fun main() {
successTest()
invalidMessageErrorTest()
failureTest()
exitProcess(exitCode)
val success = SmokeTestRunner().runAllTests()
if (!success) {
exitProcess(1)
}
}
""".trimIndent(),
)
}

@Test
fun runnerClassTest() {
generatedCode.shouldContainOnlyOnceWithDiff(
"public class SmokeTestRunner(private val platform: PlatformProvider = PlatformProvider.System, private val printer: Appendable = DefaultPrinter) {",
)
}

@Test
fun runAllTestsTest() {
generatedCode.shouldContainOnlyOnceWithDiff(
"""
public suspend fun runAllTests(): Boolean =
listOf<suspend () -> Boolean>(
::successTest,
::invalidMessageErrorTest,
::failureTest,
)
.map { it() }
.all { it }
""".formatForTest(),
)
}

@Test
fun successTest() {
generatedCode.shouldContainOnlyOnceWithDiff(
"""
private suspend fun successTest() {
private suspend fun successTest(): Boolean {
val tags = setOf<String>("success")
if ((serviceFilter.isNotEmpty() && "Test" !in serviceFilter) || tags.any { it in skipTags }) {
println("ok Test SuccessTest - no error expected from service # skip")
return
printer.appendLine("ok Test SuccessTest - no error expected from service # skip")
return true
}

try {
return try {
TestClient {
interceptors.add(SmokeTestsInterceptor())
region = "eu-central-1"
Expand All @@ -118,33 +140,36 @@ class SmokeTestsRunnerGeneratorTest {
}
)
}


error("Unexpectedly completed smoke test operation without throwing exception")

} catch (exception: Exception) {
val success: Boolean = exception is SmokeTestsSuccessException
val status: String = if (success) "ok" else "not ok"
println("${'$'}status Test SuccessTest - no error expected from service ")
printer.appendLine("${'$'}status Test SuccessTest - no error expected from service ")
if (!success) {
printExceptionStackTrace(exception)
exitCode = 1
printer.appendLine(exception.stackTraceToString().prependIndent("# "))
}

success
}
}
""".trimIndent(),
""".formatForTest(),
)
}

@Test
fun invalidMessageErrorTest() {
generatedCode.shouldContainOnlyOnceWithDiff(
"""
private suspend fun invalidMessageErrorTest() {
private suspend fun invalidMessageErrorTest(): Boolean {
val tags = setOf<String>()
if ((serviceFilter.isNotEmpty() && "Test" !in serviceFilter) || tags.any { it in skipTags }) {
println("ok Test InvalidMessageErrorTest - error expected from service # skip")
return
printer.appendLine("ok Test InvalidMessageErrorTest - error expected from service # skip")
return true
}

try {
return try {
TestClient {
}.use { client ->
client.testOperation(
Expand All @@ -154,32 +179,35 @@ class SmokeTestsRunnerGeneratorTest {
)
}

error("Unexpectedly completed smoke test operation without throwing exception")

} catch (exception: Exception) {
val success: Boolean = exception is InvalidMessageError
val status: String = if (success) "ok" else "not ok"
println("${'$'}status Test InvalidMessageErrorTest - error expected from service ")
printer.appendLine("${'$'}status Test InvalidMessageErrorTest - error expected from service ")
if (!success) {
printExceptionStackTrace(exception)
exitCode = 1
printer.appendLine(exception.stackTraceToString().prependIndent("# "))
}

success
}
}
""".trimIndent(),
""".formatForTest(),
)
}

@Test
fun failureTest() {
generatedCode.shouldContainOnlyOnceWithDiff(
"""
private suspend fun failureTest() {
private suspend fun failureTest(): Boolean {
val tags = setOf<String>()
if ((serviceFilter.isNotEmpty() && "Test" !in serviceFilter) || tags.any { it in skipTags }) {
println("ok Test FailureTest - error expected from service # skip")
return
printer.appendLine("ok Test FailureTest - error expected from service # skip")
return true
}

try {
return try {
TestClient {
interceptors.add(SmokeTestsInterceptor())
}.use { client ->
Expand All @@ -190,17 +218,20 @@ class SmokeTestsRunnerGeneratorTest {
)
}

error("Unexpectedly completed smoke test operation without throwing exception")

} catch (exception: Exception) {
val success: Boolean = exception is SmokeTestsFailureException
val status: String = if (success) "ok" else "not ok"
println("${'$'}status Test FailureTest - error expected from service ")
printer.appendLine("${'$'}status Test FailureTest - error expected from service ")
if (!success) {
printExceptionStackTrace(exception)
exitCode = 1
printer.appendLine(exception.stackTraceToString().prependIndent("# "))
}

success
}
}
""".trimIndent(),
""".formatForTest(),
)
}

Expand Down
1 change: 1 addition & 0 deletions runtime/runtime-core/api/runtime-core.api
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2093,6 +2093,7 @@ public final class aws/smithy/kotlin/runtime/smoketests/SmokeTestsFunctionsJVMKt
}

public final class aws/smithy/kotlin/runtime/smoketests/SmokeTestsFunctionsKt {
public static final fun getDefaultPrinter ()Ljava/lang/Appendable;
public static final fun printExceptionStackTrace (Ljava/lang/Exception;)V
}

Expand Down
Loading
Loading