From ebd585b7e1f546cd9e54c09b93b07ef5d96f3f3a Mon Sep 17 00:00:00 2001 From: tom doron Date: Thu, 17 Mar 2022 21:54:50 -0700 Subject: [PATCH 01/18] packaging plugin motivation: add an easy wasy for lambda users to package their lambda and upload it to AWS changes: * add SwiftPM plugin with the verb "archive" * use docker to build and package the lambda(s) --- Examples/Echo/Package.swift | 16 +- Package.swift | 8 +- Package@swift-5.2.swift | 55 ++++ Package@swift-5.3.swift | 1 + Package@swift-5.4.swift | 1 + Package@swift-5.5.swift | 1 + Plugins/AWSLambdaPackager/Plugin.swift | 344 +++++++++++++++++++++++++ scripts/soundness.sh | 2 +- 8 files changed, 420 insertions(+), 8 deletions(-) create mode 100644 Package@swift-5.2.swift create mode 120000 Package@swift-5.3.swift create mode 120000 Package@swift-5.4.swift create mode 120000 Package@swift-5.5.swift create mode 100644 Plugins/AWSLambdaPackager/Plugin.swift diff --git a/Examples/Echo/Package.swift b/Examples/Echo/Package.swift index caae8f03..3040a7af 100644 --- a/Examples/Echo/Package.swift +++ b/Examples/Echo/Package.swift @@ -1,7 +1,16 @@ // swift-tools-version:5.5 +import Foundation import PackageDescription +// this is the dependency on the swift-aws-lambda-runtime library +var dependencies = [Package.Dependency]() +if FileManager.default.fileExists(atPath: "../../Package.swift") { + dependencies.append(Package.Dependency.package(name: "swift-aws-lambda-runtime", path: "../..")) +} else { + dependencies.append(Package.Dependency.package(url: "https://github.com/swift-server/swift-aws-lambda-runtime.git", branch: "main")) +} + let package = Package( name: "swift-aws-lambda-runtime-example", platforms: [ @@ -10,12 +19,7 @@ let package = Package( products: [ .executable(name: "MyLambda", targets: ["MyLambda"]), ], - dependencies: [ - // this is the dependency on the swift-aws-lambda-runtime library - // in real-world projects this would say - // .package(url: "https://github.com/swift-server/swift-aws-lambda-runtime.git", from: "1.0.0") - .package(name: "swift-aws-lambda-runtime", path: "../.."), - ], + dependencies: dependencies, targets: [ .executableTarget( name: "MyLambda", diff --git a/Package.swift b/Package.swift index 90ace14b..2af93087 100644 --- a/Package.swift +++ b/Package.swift @@ -1,4 +1,4 @@ -// swift-tools-version:5.4 +// swift-tools-version:5.6 import PackageDescription @@ -9,6 +9,8 @@ let package = Package( .library(name: "AWSLambdaRuntime", targets: ["AWSLambdaRuntime"]), // this has all the main functionality for lambda and it does not link Foundation .library(name: "AWSLambdaRuntimeCore", targets: ["AWSLambdaRuntimeCore"]), + // plugin to package the lambda, preparing an archive that can be uploaded to AWS. requires docker. + .plugin(name: "AWSLambdaPackager", targets: ["AWSLambdaPackager"]), // for testing only .library(name: "AWSLambdaTesting", targets: ["AWSLambdaTesting"]), ], @@ -31,6 +33,10 @@ let package = Package( .product(name: "NIOConcurrencyHelpers", package: "swift-nio"), .product(name: "NIOPosix", package: "swift-nio"), ]), + .plugin( + name: "AWSLambdaPackager", + capability: .command(intent: .custom(verb: "archive", description: "Archive Lambda binary and prepare it for uploading to AWS")) + ), .testTarget(name: "AWSLambdaRuntimeCoreTests", dependencies: [ .byName(name: "AWSLambdaRuntimeCore"), .product(name: "NIOTestUtils", package: "swift-nio"), diff --git a/Package@swift-5.2.swift b/Package@swift-5.2.swift new file mode 100644 index 00000000..ca0db60e --- /dev/null +++ b/Package@swift-5.2.swift @@ -0,0 +1,55 @@ +// swift-tools-version:5.2 + +import PackageDescription + +let package = Package( + name: "swift-aws-lambda-runtime", + products: [ + // this library exports `AWSLambdaRuntimeCore` and adds Foundation convenience methods + .library(name: "AWSLambdaRuntime", targets: ["AWSLambdaRuntime"]), + // this has all the main functionality for lambda and it does not link Foundation + .library(name: "AWSLambdaRuntimeCore", targets: ["AWSLambdaRuntimeCore"]), + // for testing only + .library(name: "AWSLambdaTesting", targets: ["AWSLambdaTesting"]), + ], + dependencies: [ + .package(url: "https://github.com/apple/swift-nio.git", .upToNextMajor(from: "2.33.0")), + .package(url: "https://github.com/apple/swift-log.git", .upToNextMajor(from: "1.4.2")), + .package(url: "https://github.com/swift-server/swift-backtrace.git", .upToNextMajor(from: "1.2.3")), + ], + targets: [ + .target(name: "AWSLambdaRuntime", dependencies: [ + .byName(name: "AWSLambdaRuntimeCore"), + .product(name: "NIOCore", package: "swift-nio"), + .product(name: "NIOFoundationCompat", package: "swift-nio"), + ]), + .target(name: "AWSLambdaRuntimeCore", dependencies: [ + .product(name: "Logging", package: "swift-log"), + .product(name: "Backtrace", package: "swift-backtrace"), + .product(name: "NIOHTTP1", package: "swift-nio"), + .product(name: "NIOCore", package: "swift-nio"), + .product(name: "NIOConcurrencyHelpers", package: "swift-nio"), + .product(name: "NIOPosix", package: "swift-nio"), + ]), + .testTarget(name: "AWSLambdaRuntimeCoreTests", dependencies: [ + .byName(name: "AWSLambdaRuntimeCore"), + .product(name: "NIOTestUtils", package: "swift-nio"), + .product(name: "NIOFoundationCompat", package: "swift-nio"), + ]), + .testTarget(name: "AWSLambdaRuntimeTests", dependencies: [ + .byName(name: "AWSLambdaRuntimeCore"), + .byName(name: "AWSLambdaRuntime"), + ]), + // testing helper + .target(name: "AWSLambdaTesting", dependencies: [ + .byName(name: "AWSLambdaRuntime"), + .product(name: "NIO", package: "swift-nio"), + ]), + .testTarget(name: "AWSLambdaTestingTests", dependencies: ["AWSLambdaTesting"]), + // for perf testing + .target(name: "MockServer", dependencies: [ + .product(name: "NIOHTTP1", package: "swift-nio"), + .product(name: "NIO", package: "swift-nio"), + ]), + ] +) diff --git a/Package@swift-5.3.swift b/Package@swift-5.3.swift new file mode 120000 index 00000000..08f3d808 --- /dev/null +++ b/Package@swift-5.3.swift @@ -0,0 +1 @@ +Package@swift-5.2.swift \ No newline at end of file diff --git a/Package@swift-5.4.swift b/Package@swift-5.4.swift new file mode 120000 index 00000000..08f3d808 --- /dev/null +++ b/Package@swift-5.4.swift @@ -0,0 +1 @@ +Package@swift-5.2.swift \ No newline at end of file diff --git a/Package@swift-5.5.swift b/Package@swift-5.5.swift new file mode 120000 index 00000000..08f3d808 --- /dev/null +++ b/Package@swift-5.5.swift @@ -0,0 +1 @@ +Package@swift-5.2.swift \ No newline at end of file diff --git a/Plugins/AWSLambdaPackager/Plugin.swift b/Plugins/AWSLambdaPackager/Plugin.swift new file mode 100644 index 00000000..b5636f39 --- /dev/null +++ b/Plugins/AWSLambdaPackager/Plugin.swift @@ -0,0 +1,344 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the SwiftAWSLambdaRuntime open source project +// +// Copyright (c) 2022 Apple Inc. and the SwiftAWSLambdaRuntime project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of SwiftAWSLambdaRuntime project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +import Dispatch +import Foundation +import PackagePlugin + +#if canImport(Glibc) +import Glibc +#endif + +@main +struct AWSLambdaPackager: CommandPlugin { + func performCommand(context: PackagePlugin.PluginContext, arguments: [String]) async throws { + let configuration = Configuration(context: context, arguments: arguments) + guard !configuration.products.isEmpty else { + throw Errors.unknownProduct("no appropriate products found to package") + } + + #if os(macOS) + let builtProducts = try self.buildInDocker( + packageIdentity: context.package.id, + packageDirectory: context.package.directory, + products: configuration.products, + toolsProvider: { name in try context.tool(named: name).path }, + outputDirectory: configuration.outputDirectory, + baseImage: configuration.baseImage, + buildConfiguration: configuration.buildConfiguration, + verboseLogging: configuration.verboseLogging + ) + #elseif os(Linux) + let builtProducts = try self.build( + products: configuration.products, + buildConfiguration: configuration.buildConfiguration, + verboseLogging: configuration.verboseLogging + ) + #else + throw Errors.unsupportedPlatform("only macOS and Linux are supported") + #endif + + let archives = try self.package( + products: builtProducts, + toolsProvider: { name in try context.tool(named: name).path }, + outputDirectory: configuration.outputDirectory, + verboseLogging: configuration.verboseLogging + ) + + if !archives.isEmpty { + print("\(archives.count) archives created:") + for (product, archivePath) in archives { + print(" * \(product.name) at \(archivePath.string)") + } + } else { + print("no archives created") + } + } + + private func buildInDocker( + packageIdentity: Package.ID, + packageDirectory: Path, + products: [Product], + toolsProvider: (String) throws -> Path, + outputDirectory: Path, + baseImage: String, + buildConfiguration: PackageManager.BuildConfiguration, + verboseLogging: Bool + ) throws -> [LambdaProduct: Path] { + let dockerToolPath = try toolsProvider("docker") + + /* + if verboseLogging { + print("-------------------------------------------------------------------------") + print("preparing docker build image") + print("-------------------------------------------------------------------------") + } + let packageDockerFilePath = packageDirectory.appending("Dockerfile") + let tempDockerFilePath = outputDirectory.appending("Dockerfile") + try FileManager.default.createDirectory(atPath: tempDockerFilePath.removingLastComponent().string, withIntermediateDirectories: true) + if FileManager.default.fileExists(atPath: packageDockerFilePath.string) { + try FileManager.default.copyItem(atPath: packageDockerFilePath.string, toPath: tempDockerFilePath.string) + } else { + FileManager.default.createFile(atPath: tempDockerFilePath.string, contents: "FROM \(baseImage)".data(using: .utf8)) + } + try self.execute( + executable: dockerToolPath, + arguments: ["build", "-f", tempDockerFilePath.string, packageDirectory.string , "-t", "\(builderImageName)"], + verboseLogging: verboseLogging + ) + */ + + let builderImageName = baseImage + + var builtProducts = [LambdaProduct: Path]() + for product in products { + if verboseLogging { + print("-------------------------------------------------------------------------") + print("building \"\(product.name)\" in docker") + print("-------------------------------------------------------------------------") + } + // + let buildCommand = "swift build --product \(product.name) -c \(buildConfiguration.rawValue) --static-swift-stdlib" + try self.execute( + executable: dockerToolPath, + arguments: ["run", "--rm", "-v", "\(packageDirectory.string):/workspace", "-w", "/workspace", builderImageName, "bash", "-cl", buildCommand], + verboseLogging: verboseLogging + ) + #warning("this knows too much about the underlying implementation") + builtProducts[.init(product)] = packageDirectory.appending([".build", buildConfiguration.rawValue, product.name]) + } + return builtProducts + } + + private func build( + products: [Product], + buildConfiguration: PackageManager.BuildConfiguration, + verboseLogging: Bool + ) throws -> [LambdaProduct: Path] { + var results = [LambdaProduct: Path]() + for product in products { + if verboseLogging { + print("-------------------------------------------------------------------------") + print("building \"\(product.name)\"") + print("-------------------------------------------------------------------------") + } + var parameters = PackageManager.BuildParameters() + parameters.configuration = buildConfiguration + parameters.otherSwiftcFlags = ["-static-stdlib"] + parameters.logging = verboseLogging ? .verbose : .concise + + let result = try packageManager.build( + .product(product.name), + parameters: parameters + ) + guard result.builtArtifacts.count <= 1 else { + throw Errors.unknownExecutable("too many executable artifacts found for \(product.name)") + } + guard let artifact = result.builtArtifacts.first else { + throw Errors.unknownExecutable("no executable artifacts found for \(product.name)") + } + results[.init(product)] = artifact.path + } + return results + } + + private func package( + products: [LambdaProduct: Path], + toolsProvider: (String) throws -> Path, + outputDirectory: Path, + verboseLogging: Bool + ) throws -> [LambdaProduct: Path] { + let zipToolPath = try toolsProvider("zip") + + var archives = [LambdaProduct: Path]() + for (product, artifactPath) in products { + if verboseLogging { + print("-------------------------------------------------------------------------") + print("archiving \"\(product.name)\"") + print("-------------------------------------------------------------------------") + } + + // prep zipfile location + let zipfilePath = outputDirectory.appending(product.name, "\(product.name).zip") + if FileManager.default.fileExists(atPath: zipfilePath.string) { + try FileManager.default.removeItem(atPath: zipfilePath.string) + } + try FileManager.default.createDirectory(atPath: zipfilePath.removingLastComponent().string, withIntermediateDirectories: true) + + #if os(macOS) || os(Linux) + let arguments = ["--junk-paths", zipfilePath.string, artifactPath.string] + #else + throw Error.unsupportedPlatform("can't or don't know how to create a zipfile on this platform") + #endif + + // run the zip tool + try self.execute( + executable: zipToolPath, + arguments: arguments, + verboseLogging: verboseLogging + ) + + archives[product] = zipfilePath + + /* + + target=".build/lambda/$executable" + rm -rf "$target" + mkdir -p "$target" + cp ".build/release/$executable" "$target/" + # add the target deps based on ldd + ldd ".build/release/$executable" | grep swift | awk '{print $3}' | xargs cp -Lv -t "$target" + cd "$target" + ln -s "$executable" "bootstrap" + zip --symlinks lambda.zip * + + */ + // docker run --rm -v "$workspace":/workspace -w /workspace/Examples/Deployment builder \ + // bash -cl "./scripts/package.sh $executable" + } + return archives + } + + @discardableResult + private func execute( + executable: Path, + arguments: [String], + customWorkingDirectory: Path? = .none, + verboseLogging: Bool + ) throws -> String { + if verboseLogging { + print("\(executable.string) \(arguments.joined(separator: " "))") + } + + let sync = DispatchGroup() + var output = "" + let outputLock = NSLock() + let outputHandler = { (fileHandle: FileHandle) in + sync.enter() + defer { sync.leave() } + if !fileHandle.availableData.isEmpty, let _output = String(data: fileHandle.availableData, encoding: .utf8)?.trimmingCharacters(in: CharacterSet(["\n"])) { + if verboseLogging { + print(_output) + fflush(stdout) + } + outputLock.lock() + output += _output + outputLock.unlock() + } + } + + let stdoutPipe = Pipe() + stdoutPipe.fileHandleForReading.readabilityHandler = outputHandler + let stderrPipe = Pipe() + stderrPipe.fileHandleForReading.readabilityHandler = outputHandler + + let process = Process() + process.standardOutput = stdoutPipe + process.standardError = stderrPipe + process.executableURL = URL(fileURLWithPath: executable.string) + process.arguments = arguments + if let workingDirectory = customWorkingDirectory { + process.currentDirectoryURL = URL(fileURLWithPath: workingDirectory.string) + } + process.terminationHandler = { _ in + // Read and pass on any remaining free-form text output from the plugin. + stderrPipe.fileHandleForReading.readabilityHandler?(stderrPipe.fileHandleForReading) + // Read and pass on any remaining messages from the plugin. + stdoutPipe.fileHandleForReading.readabilityHandler?(stdoutPipe.fileHandleForReading) + } + + try process.run() + process.waitUntilExit() + + // wait for output to be full processed + sync.wait() + + if process.terminationStatus != 0 { + throw Errors.processFailed(process.terminationStatus) + } + + return output + } +} + +private struct Configuration { + public let outputDirectory: Path + public let products: [Product] + public let buildConfiguration: PackageManager.BuildConfiguration + public let staticallyLinkRuntime: Bool + public let verboseLogging: Bool + public let baseImage: String + public let version: String + public let applicationRoot: String + + public init(context: PluginContext, arguments: [String]) { + self.outputDirectory = context.pluginWorkDirectory.appending(subpath: "\(AWSLambdaPackager.self)") // FIXME: read argument + self.products = context.package.products.filter { $0 is ExecutableProduct } // FIXME: read argument, filter is ugly + self.buildConfiguration = .release // FIXME: read argument + #if os(Linux) + self.staticallyLinkRuntime = true // FIXME: read argument + #else + self.staticallyLinkRuntime = false // FIXME: read argument, warn if set to true + #endif + self.verboseLogging = true // FIXME: read argument + let swiftVersion = "5.6" // FIXME: read dynamically current version + self.baseImage = "swift:\(swiftVersion)-amazonlinux2" // FIXME: read argument + self.version = "1.0.0" // FIXME: where can we get this from? argument? + self.applicationRoot = "/app" // FIXME: read argument + } +} + +private enum Errors: Error { + case unsupportedPlatform(String) + case unknownProduct(String) + case unknownExecutable(String) + case buildError(String) + case failedWritingDockerfile + case processFailed(Int32) + case invalidProcessOutput +} + +extension PackageManager.BuildResult { + // find the executable produced by the build + func executableArtifact(for product: Product) throws -> PackageManager.BuildResult.BuiltArtifact? { + let executables = self.builtArtifacts.filter { $0.kind == .executable && $0.path.lastComponent == product.name } + guard !executables.isEmpty else { + return nil + } + guard executables.count == 1, let executable = executables.first else { + return nil + } + return executable + } +} + +struct LambdaProduct: Hashable { + let underlying: Product + + init(_ underlying: Product) { + self.underlying = underlying + } + + var name: String { + self.underlying.name + } + + func hash(into hasher: inout Hasher) { + self.underlying.id.hash(into: &hasher) + } + + static func == (lhs: Self, rhs: Self) -> Bool { + lhs.underlying.id == rhs.underlying.id + } +} diff --git a/scripts/soundness.sh b/scripts/soundness.sh index d9145903..0f13f34f 100755 --- a/scripts/soundness.sh +++ b/scripts/soundness.sh @@ -61,7 +61,7 @@ for language in swift-or-c bash dtrace; do matching_files=( -name '*' ) case "$language" in swift-or-c) - exceptions=( -name Package.swift ) + exceptions=( -name Package.swift -o -name Package@*.swift ) matching_files=( -name '*.swift' -o -name '*.c' -o -name '*.h' ) cat > "$tmp" <<"EOF" //===----------------------------------------------------------------------===// From 95134c0fe7a6c12f25c738a411847df77386d940 Mon Sep 17 00:00:00 2001 From: tom doron Date: Fri, 18 Mar 2022 10:26:33 -0700 Subject: [PATCH 02/18] check for amazonlinux, zip correctly, drop 5.2 and 5.3 --- Package.swift | 4 +- Package@swift-5.2.swift | 55 ---------- Package@swift-5.3.swift | 1 - Package@swift-5.4.swift | 56 +++++++++- Package@swift-5.5.swift | 2 +- Plugins/AWSLambdaPackager/Plugin.swift | 140 +++++++++++-------------- 6 files changed, 122 insertions(+), 136 deletions(-) delete mode 100644 Package@swift-5.2.swift delete mode 120000 Package@swift-5.3.swift mode change 120000 => 100644 Package@swift-5.4.swift diff --git a/Package.swift b/Package.swift index 2af93087..3a1dd937 100644 --- a/Package.swift +++ b/Package.swift @@ -9,7 +9,7 @@ let package = Package( .library(name: "AWSLambdaRuntime", targets: ["AWSLambdaRuntime"]), // this has all the main functionality for lambda and it does not link Foundation .library(name: "AWSLambdaRuntimeCore", targets: ["AWSLambdaRuntimeCore"]), - // plugin to package the lambda, preparing an archive that can be uploaded to AWS. requires docker. + // plugin to package the lambda, creating an archive that can be uploaded to AWS .plugin(name: "AWSLambdaPackager", targets: ["AWSLambdaPackager"]), // for testing only .library(name: "AWSLambdaTesting", targets: ["AWSLambdaTesting"]), @@ -35,7 +35,7 @@ let package = Package( ]), .plugin( name: "AWSLambdaPackager", - capability: .command(intent: .custom(verb: "archive", description: "Archive Lambda binary and prepare it for uploading to AWS")) + capability: .command(intent: .custom(verb: "archive", description: "Archive the Lambda binary and prepare it for uploading to AWS. Requires docker on macOS.")) ), .testTarget(name: "AWSLambdaRuntimeCoreTests", dependencies: [ .byName(name: "AWSLambdaRuntimeCore"), diff --git a/Package@swift-5.2.swift b/Package@swift-5.2.swift deleted file mode 100644 index ca0db60e..00000000 --- a/Package@swift-5.2.swift +++ /dev/null @@ -1,55 +0,0 @@ -// swift-tools-version:5.2 - -import PackageDescription - -let package = Package( - name: "swift-aws-lambda-runtime", - products: [ - // this library exports `AWSLambdaRuntimeCore` and adds Foundation convenience methods - .library(name: "AWSLambdaRuntime", targets: ["AWSLambdaRuntime"]), - // this has all the main functionality for lambda and it does not link Foundation - .library(name: "AWSLambdaRuntimeCore", targets: ["AWSLambdaRuntimeCore"]), - // for testing only - .library(name: "AWSLambdaTesting", targets: ["AWSLambdaTesting"]), - ], - dependencies: [ - .package(url: "https://github.com/apple/swift-nio.git", .upToNextMajor(from: "2.33.0")), - .package(url: "https://github.com/apple/swift-log.git", .upToNextMajor(from: "1.4.2")), - .package(url: "https://github.com/swift-server/swift-backtrace.git", .upToNextMajor(from: "1.2.3")), - ], - targets: [ - .target(name: "AWSLambdaRuntime", dependencies: [ - .byName(name: "AWSLambdaRuntimeCore"), - .product(name: "NIOCore", package: "swift-nio"), - .product(name: "NIOFoundationCompat", package: "swift-nio"), - ]), - .target(name: "AWSLambdaRuntimeCore", dependencies: [ - .product(name: "Logging", package: "swift-log"), - .product(name: "Backtrace", package: "swift-backtrace"), - .product(name: "NIOHTTP1", package: "swift-nio"), - .product(name: "NIOCore", package: "swift-nio"), - .product(name: "NIOConcurrencyHelpers", package: "swift-nio"), - .product(name: "NIOPosix", package: "swift-nio"), - ]), - .testTarget(name: "AWSLambdaRuntimeCoreTests", dependencies: [ - .byName(name: "AWSLambdaRuntimeCore"), - .product(name: "NIOTestUtils", package: "swift-nio"), - .product(name: "NIOFoundationCompat", package: "swift-nio"), - ]), - .testTarget(name: "AWSLambdaRuntimeTests", dependencies: [ - .byName(name: "AWSLambdaRuntimeCore"), - .byName(name: "AWSLambdaRuntime"), - ]), - // testing helper - .target(name: "AWSLambdaTesting", dependencies: [ - .byName(name: "AWSLambdaRuntime"), - .product(name: "NIO", package: "swift-nio"), - ]), - .testTarget(name: "AWSLambdaTestingTests", dependencies: ["AWSLambdaTesting"]), - // for perf testing - .target(name: "MockServer", dependencies: [ - .product(name: "NIOHTTP1", package: "swift-nio"), - .product(name: "NIO", package: "swift-nio"), - ]), - ] -) diff --git a/Package@swift-5.3.swift b/Package@swift-5.3.swift deleted file mode 120000 index 08f3d808..00000000 --- a/Package@swift-5.3.swift +++ /dev/null @@ -1 +0,0 @@ -Package@swift-5.2.swift \ No newline at end of file diff --git a/Package@swift-5.4.swift b/Package@swift-5.4.swift deleted file mode 120000 index 08f3d808..00000000 --- a/Package@swift-5.4.swift +++ /dev/null @@ -1 +0,0 @@ -Package@swift-5.2.swift \ No newline at end of file diff --git a/Package@swift-5.4.swift b/Package@swift-5.4.swift new file mode 100644 index 00000000..90ace14b --- /dev/null +++ b/Package@swift-5.4.swift @@ -0,0 +1,55 @@ +// swift-tools-version:5.4 + +import PackageDescription + +let package = Package( + name: "swift-aws-lambda-runtime", + products: [ + // this library exports `AWSLambdaRuntimeCore` and adds Foundation convenience methods + .library(name: "AWSLambdaRuntime", targets: ["AWSLambdaRuntime"]), + // this has all the main functionality for lambda and it does not link Foundation + .library(name: "AWSLambdaRuntimeCore", targets: ["AWSLambdaRuntimeCore"]), + // for testing only + .library(name: "AWSLambdaTesting", targets: ["AWSLambdaTesting"]), + ], + dependencies: [ + .package(url: "https://github.com/apple/swift-nio.git", .upToNextMajor(from: "2.33.0")), + .package(url: "https://github.com/apple/swift-log.git", .upToNextMajor(from: "1.4.2")), + .package(url: "https://github.com/swift-server/swift-backtrace.git", .upToNextMajor(from: "1.2.3")), + ], + targets: [ + .target(name: "AWSLambdaRuntime", dependencies: [ + .byName(name: "AWSLambdaRuntimeCore"), + .product(name: "NIOCore", package: "swift-nio"), + .product(name: "NIOFoundationCompat", package: "swift-nio"), + ]), + .target(name: "AWSLambdaRuntimeCore", dependencies: [ + .product(name: "Logging", package: "swift-log"), + .product(name: "Backtrace", package: "swift-backtrace"), + .product(name: "NIOHTTP1", package: "swift-nio"), + .product(name: "NIOCore", package: "swift-nio"), + .product(name: "NIOConcurrencyHelpers", package: "swift-nio"), + .product(name: "NIOPosix", package: "swift-nio"), + ]), + .testTarget(name: "AWSLambdaRuntimeCoreTests", dependencies: [ + .byName(name: "AWSLambdaRuntimeCore"), + .product(name: "NIOTestUtils", package: "swift-nio"), + .product(name: "NIOFoundationCompat", package: "swift-nio"), + ]), + .testTarget(name: "AWSLambdaRuntimeTests", dependencies: [ + .byName(name: "AWSLambdaRuntimeCore"), + .byName(name: "AWSLambdaRuntime"), + ]), + // testing helper + .target(name: "AWSLambdaTesting", dependencies: [ + .byName(name: "AWSLambdaRuntime"), + .product(name: "NIO", package: "swift-nio"), + ]), + .testTarget(name: "AWSLambdaTestingTests", dependencies: ["AWSLambdaTesting"]), + // for perf testing + .target(name: "MockServer", dependencies: [ + .product(name: "NIOHTTP1", package: "swift-nio"), + .product(name: "NIO", package: "swift-nio"), + ]), + ] +) diff --git a/Package@swift-5.5.swift b/Package@swift-5.5.swift index 08f3d808..bd1da396 120000 --- a/Package@swift-5.5.swift +++ b/Package@swift-5.5.swift @@ -1 +1 @@ -Package@swift-5.2.swift \ No newline at end of file +Package@swift-5.4.swift \ No newline at end of file diff --git a/Plugins/AWSLambdaPackager/Plugin.swift b/Plugins/AWSLambdaPackager/Plugin.swift index b5636f39..0f0cdae5 100644 --- a/Plugins/AWSLambdaPackager/Plugin.swift +++ b/Plugins/AWSLambdaPackager/Plugin.swift @@ -28,27 +28,29 @@ struct AWSLambdaPackager: CommandPlugin { throw Errors.unknownProduct("no appropriate products found to package") } - #if os(macOS) - let builtProducts = try self.buildInDocker( - packageIdentity: context.package.id, - packageDirectory: context.package.directory, - products: configuration.products, - toolsProvider: { name in try context.tool(named: name).path }, - outputDirectory: configuration.outputDirectory, - baseImage: configuration.baseImage, - buildConfiguration: configuration.buildConfiguration, - verboseLogging: configuration.verboseLogging - ) - #elseif os(Linux) - let builtProducts = try self.build( - products: configuration.products, - buildConfiguration: configuration.buildConfiguration, - verboseLogging: configuration.verboseLogging - ) - #else - throw Errors.unsupportedPlatform("only macOS and Linux are supported") - #endif + let builtProducts: [LambdaProduct: Path] + if self.isAmazonLinux2() { + // build directly on the machine + builtProducts = try self.build( + products: configuration.products, + buildConfiguration: configuration.buildConfiguration, + verboseLogging: configuration.verboseLogging + ) + } else { + // build with docker + builtProducts = try self.buildInDocker( + packageIdentity: context.package.id, + packageDirectory: context.package.directory, + products: configuration.products, + toolsProvider: { name in try context.tool(named: name).path }, + outputDirectory: configuration.outputDirectory, + baseImage: configuration.baseImage, + buildConfiguration: configuration.buildConfiguration, + verboseLogging: configuration.verboseLogging + ) + } + // create the archive let archives = try self.package( products: builtProducts, toolsProvider: { name in try context.tool(named: name).path }, @@ -56,13 +58,9 @@ struct AWSLambdaPackager: CommandPlugin { verboseLogging: configuration.verboseLogging ) - if !archives.isEmpty { - print("\(archives.count) archives created:") - for (product, archivePath) in archives { - print(" * \(product.name) at \(archivePath.string)") - } - } else { - print("no archives created") + print("\(archives.count > 0 ? archives.count.description : "no") archive\(archives.count != 1 ? "s" : "") created") + for (product, archivePath) in archives { + print(" * \(product.name) at \(archivePath.string)") } } @@ -115,7 +113,7 @@ struct AWSLambdaPackager: CommandPlugin { arguments: ["run", "--rm", "-v", "\(packageDirectory.string):/workspace", "-w", "/workspace", builderImageName, "bash", "-cl", buildCommand], verboseLogging: verboseLogging ) - #warning("this knows too much about the underlying implementation") + // TODO: this knows too much about the underlying implementation builtProducts[.init(product)] = packageDirectory.appending([".build", buildConfiguration.rawValue, product.name]) } return builtProducts @@ -142,10 +140,7 @@ struct AWSLambdaPackager: CommandPlugin { .product(product.name), parameters: parameters ) - guard result.builtArtifacts.count <= 1 else { - throw Errors.unknownExecutable("too many executable artifacts found for \(product.name)") - } - guard let artifact = result.builtArtifacts.first else { + guard let artifact = result.executableArtifact(for: product) else { throw Errors.unknownExecutable("no executable artifacts found for \(product.name)") } results[.init(product)] = artifact.path @@ -170,14 +165,21 @@ struct AWSLambdaPackager: CommandPlugin { } // prep zipfile location - let zipfilePath = outputDirectory.appending(product.name, "\(product.name).zip") - if FileManager.default.fileExists(atPath: zipfilePath.string) { - try FileManager.default.removeItem(atPath: zipfilePath.string) + let workingDirectory = outputDirectory.appending(product.name) + let zipfilePath = workingDirectory.appending("\(product.name).zip") + if FileManager.default.fileExists(atPath: workingDirectory.string) { + try FileManager.default.removeItem(atPath: workingDirectory.string) } - try FileManager.default.createDirectory(atPath: zipfilePath.removingLastComponent().string, withIntermediateDirectories: true) + try FileManager.default.createDirectory(atPath: workingDirectory.string, withIntermediateDirectories: true) + + // rename artifact to "bootstrap" + let relocatedArtifactPath = workingDirectory.appending(artifactPath.lastComponent) + let symbolicLinkPath = workingDirectory.appending("bootstrap") + try FileManager.default.copyItem(atPath: artifactPath.string, toPath: relocatedArtifactPath.string) + try FileManager.default.createSymbolicLink(atPath: symbolicLinkPath.string, withDestinationPath: relocatedArtifactPath.lastComponent) #if os(macOS) || os(Linux) - let arguments = ["--junk-paths", zipfilePath.string, artifactPath.string] + let arguments = ["--junk-paths", "--symlinks", zipfilePath.string, relocatedArtifactPath.string, symbolicLinkPath.string] #else throw Error.unsupportedPlatform("can't or don't know how to create a zipfile on this platform") #endif @@ -190,22 +192,6 @@ struct AWSLambdaPackager: CommandPlugin { ) archives[product] = zipfilePath - - /* - - target=".build/lambda/$executable" - rm -rf "$target" - mkdir -p "$target" - cp ".build/release/$executable" "$target/" - # add the target deps based on ldd - ldd ".build/release/$executable" | grep swift | awk '{print $3}' | xargs cp -Lv -t "$target" - cd "$target" - ln -s "$executable" "bootstrap" - zip --symlinks lambda.zip * - - */ - // docker run --rm -v "$workspace":/workspace -w /workspace/Examples/Deployment builder \ - // bash -cl "./scripts/package.sh $executable" } return archives } @@ -223,25 +209,25 @@ struct AWSLambdaPackager: CommandPlugin { let sync = DispatchGroup() var output = "" - let outputLock = NSLock() - let outputHandler = { (fileHandle: FileHandle) in + let outputQueue = DispatchQueue(label: "AWSLambdaPackager.output") + let outputHandler = { (data: Data?) in + dispatchPrecondition(condition: .onQueue(outputQueue)) + guard let _output = data.flatMap({ String(data: $0, encoding: .utf8)?.trimmingCharacters(in: CharacterSet(["\n"])) }), !_output.isEmpty else { + return + } sync.enter() defer { sync.leave() } - if !fileHandle.availableData.isEmpty, let _output = String(data: fileHandle.availableData, encoding: .utf8)?.trimmingCharacters(in: CharacterSet(["\n"])) { - if verboseLogging { - print(_output) - fflush(stdout) - } - outputLock.lock() - output += _output - outputLock.unlock() + if verboseLogging { + print(_output) + fflush(stdout) } + output += _output } let stdoutPipe = Pipe() - stdoutPipe.fileHandleForReading.readabilityHandler = outputHandler + stdoutPipe.fileHandleForReading.readabilityHandler = { fileHandle in outputQueue.async { outputHandler(fileHandle.availableData) } } let stderrPipe = Pipe() - stderrPipe.fileHandleForReading.readabilityHandler = outputHandler + stderrPipe.fileHandleForReading.readabilityHandler = { fileHandle in outputQueue.async { outputHandler(fileHandle.availableData) } } let process = Process() process.standardOutput = stdoutPipe @@ -252,10 +238,10 @@ struct AWSLambdaPackager: CommandPlugin { process.currentDirectoryURL = URL(fileURLWithPath: workingDirectory.string) } process.terminationHandler = { _ in - // Read and pass on any remaining free-form text output from the plugin. - stderrPipe.fileHandleForReading.readabilityHandler?(stderrPipe.fileHandleForReading) - // Read and pass on any remaining messages from the plugin. - stdoutPipe.fileHandleForReading.readabilityHandler?(stdoutPipe.fileHandleForReading) + outputQueue.async { + outputHandler(try? stdoutPipe.fileHandleForReading.readToEnd()) + outputHandler(try? stderrPipe.fileHandleForReading.readToEnd()) + } } try process.run() @@ -270,13 +256,20 @@ struct AWSLambdaPackager: CommandPlugin { return output } + + private func isAmazonLinux2() -> Bool { + if let data = FileManager.default.contents(atPath: "/etc/system-release"), let release = String(data: data, encoding: .utf8) { + return release.hasPrefix("Amazon Linux release 2") + } else { + return false + } + } } private struct Configuration { public let outputDirectory: Path public let products: [Product] public let buildConfiguration: PackageManager.BuildConfiguration - public let staticallyLinkRuntime: Bool public let verboseLogging: Bool public let baseImage: String public let version: String @@ -286,11 +279,6 @@ private struct Configuration { self.outputDirectory = context.pluginWorkDirectory.appending(subpath: "\(AWSLambdaPackager.self)") // FIXME: read argument self.products = context.package.products.filter { $0 is ExecutableProduct } // FIXME: read argument, filter is ugly self.buildConfiguration = .release // FIXME: read argument - #if os(Linux) - self.staticallyLinkRuntime = true // FIXME: read argument - #else - self.staticallyLinkRuntime = false // FIXME: read argument, warn if set to true - #endif self.verboseLogging = true // FIXME: read argument let swiftVersion = "5.6" // FIXME: read dynamically current version self.baseImage = "swift:\(swiftVersion)-amazonlinux2" // FIXME: read argument @@ -311,7 +299,7 @@ private enum Errors: Error { extension PackageManager.BuildResult { // find the executable produced by the build - func executableArtifact(for product: Product) throws -> PackageManager.BuildResult.BuiltArtifact? { + func executableArtifact(for product: Product) -> PackageManager.BuildResult.BuiltArtifact? { let executables = self.builtArtifacts.filter { $0.kind == .executable && $0.path.lastComponent == product.name } guard !executables.isEmpty else { return nil From 5c035d23c97468eb4b789c355411753f8f33364b Mon Sep 17 00:00:00 2001 From: tom doron Date: Fri, 18 Mar 2022 18:35:50 -0700 Subject: [PATCH 03/18] parse arguments --- Plugins/AWSLambdaPackager/Plugin.swift | 128 ++++++++++++++++++------- 1 file changed, 93 insertions(+), 35 deletions(-) diff --git a/Plugins/AWSLambdaPackager/Plugin.swift b/Plugins/AWSLambdaPackager/Plugin.swift index 0f0cdae5..dda31ad8 100644 --- a/Plugins/AWSLambdaPackager/Plugin.swift +++ b/Plugins/AWSLambdaPackager/Plugin.swift @@ -23,7 +23,7 @@ import Glibc @main struct AWSLambdaPackager: CommandPlugin { func performCommand(context: PackagePlugin.PluginContext, arguments: [String]) async throws { - let configuration = Configuration(context: context, arguments: arguments) + let configuration = try Configuration(context: context, arguments: arguments) guard !configuration.products.isEmpty else { throw Errors.unknownProduct("no appropriate products found to package") } @@ -44,7 +44,7 @@ struct AWSLambdaPackager: CommandPlugin { products: configuration.products, toolsProvider: { name in try context.tool(named: name).path }, outputDirectory: configuration.outputDirectory, - baseImage: configuration.baseImage, + baseImage: configuration.baseDockerImage, buildConfiguration: configuration.buildConfiguration, verboseLogging: configuration.verboseLogging ) @@ -101,11 +101,9 @@ struct AWSLambdaPackager: CommandPlugin { var builtProducts = [LambdaProduct: Path]() for product in products { - if verboseLogging { - print("-------------------------------------------------------------------------") - print("building \"\(product.name)\" in docker") - print("-------------------------------------------------------------------------") - } + print("-------------------------------------------------------------------------") + print("building \"\(product.name)\" in docker") + print("-------------------------------------------------------------------------") // let buildCommand = "swift build --product \(product.name) -c \(buildConfiguration.rawValue) --static-swift-stdlib" try self.execute( @@ -126,11 +124,9 @@ struct AWSLambdaPackager: CommandPlugin { ) throws -> [LambdaProduct: Path] { var results = [LambdaProduct: Path]() for product in products { - if verboseLogging { - print("-------------------------------------------------------------------------") - print("building \"\(product.name)\"") - print("-------------------------------------------------------------------------") - } + print("-------------------------------------------------------------------------") + print("building \"\(product.name)\"") + print("-------------------------------------------------------------------------") var parameters = PackageManager.BuildParameters() parameters.configuration = buildConfiguration parameters.otherSwiftcFlags = ["-static-stdlib"] @@ -148,6 +144,7 @@ struct AWSLambdaPackager: CommandPlugin { return results } + #warning("FIXME: use zlib") private func package( products: [LambdaProduct: Path], toolsProvider: (String) throws -> Path, @@ -158,11 +155,9 @@ struct AWSLambdaPackager: CommandPlugin { var archives = [LambdaProduct: Path]() for (product, artifactPath) in products { - if verboseLogging { - print("-------------------------------------------------------------------------") - print("archiving \"\(product.name)\"") - print("-------------------------------------------------------------------------") - } + print("-------------------------------------------------------------------------") + print("archiving \"\(product.name)\"") + print("-------------------------------------------------------------------------") // prep zipfile location let workingDirectory = outputDirectory.appending(product.name) @@ -217,10 +212,8 @@ struct AWSLambdaPackager: CommandPlugin { } sync.enter() defer { sync.leave() } - if verboseLogging { - print(_output) - fflush(stdout) - } + print(_output) + fflush(stdout) output += _output } @@ -266,28 +259,93 @@ struct AWSLambdaPackager: CommandPlugin { } } -private struct Configuration { +private struct Configuration: CustomStringConvertible { public let outputDirectory: Path public let products: [Product] public let buildConfiguration: PackageManager.BuildConfiguration public let verboseLogging: Bool - public let baseImage: String - public let version: String - public let applicationRoot: String - - public init(context: PluginContext, arguments: [String]) { - self.outputDirectory = context.pluginWorkDirectory.appending(subpath: "\(AWSLambdaPackager.self)") // FIXME: read argument - self.products = context.package.products.filter { $0 is ExecutableProduct } // FIXME: read argument, filter is ugly - self.buildConfiguration = .release // FIXME: read argument - self.verboseLogging = true // FIXME: read argument - let swiftVersion = "5.6" // FIXME: read dynamically current version - self.baseImage = "swift:\(swiftVersion)-amazonlinux2" // FIXME: read argument - self.version = "1.0.0" // FIXME: where can we get this from? argument? - self.applicationRoot = "/app" // FIXME: read argument + public let baseDockerImage: String + + public init( + context: PluginContext, + arguments: [String] + ) throws { + var argumentExtractor = ArgumentExtractor(arguments) + let verboseArgument = argumentExtractor.extractFlag(named: "verbose") > 0 + let outputPathArgument = argumentExtractor.extractOption(named: "output-path") + let productsArgument = argumentExtractor.extractOption(named: "products") + let configurationArgument = argumentExtractor.extractOption(named: "configuration") + let swiftVersionArgument = argumentExtractor.extractOption(named: "swift-version") + let baseDockerImageArgument = argumentExtractor.extractOption(named: "base-docker-image") + + self.verboseLogging = verboseArgument + + if let outputPath = outputPathArgument.first { + var isDirectory: ObjCBool = false + guard FileManager.default.fileExists(atPath: outputPath, isDirectory: &isDirectory), isDirectory.boolValue else { + throw Errors.invalidArgument("invalid output directory \(outputPath)") + } + self.outputDirectory = Path(outputPath) + } else { + self.outputDirectory = context.pluginWorkDirectory.appending(subpath: "\(AWSLambdaPackager.self)") + } + + if !productsArgument.isEmpty { + let products = try context.package.products(named: productsArgument) + for product in products { + guard product is ExecutableProduct else { + throw Errors.invalidArgument("product named \(product.name) is not an executable product") + } + } + self.products = products + + } else { + self.products = context.package.products.filter { $0 is ExecutableProduct } + } + + if let buildConfigurationName = configurationArgument.first { + guard let buildConfiguration = PackageManager.BuildConfiguration(rawValue: buildConfigurationName) else { + throw Errors.invalidArgument("invalid build configuration named \(buildConfigurationName)") + } + self.buildConfiguration = buildConfiguration + } else { + self.buildConfiguration = .release + } + + guard !(!swiftVersionArgument.isEmpty && !baseDockerImageArgument.isEmpty) else { + throw Errors.invalidArgument("--swift-version and --base-docker-image are mutually exclusive") + } + + let swiftVersion = swiftVersionArgument.first ?? Self.getSwiftVersion() + self.baseDockerImage = baseDockerImageArgument.first ?? "swift:\(swiftVersion)-amazonlinux2" + + if self.verboseLogging { + print("-------------------------------------------------------------------------") + print("configuration") + print("-------------------------------------------------------------------------") + print(self) + } + } + + var description: String { + """ + { + outputDirectory: \(self.outputDirectory) + products: \(self.products.map(\.name)) + buildConfiguration: \(self.buildConfiguration) + baseDockerImage: \(self.baseDockerImage) + } + """ + } + + #warning("FIXME: read this programmatically") + private static func getSwiftVersion() -> String { + "5.6" } } private enum Errors: Error { + case invalidArgument(String) case unsupportedPlatform(String) case unknownProduct(String) case unknownExecutable(String) From 4b23bb6c2045fa000a67ffaf5b73bebe00d4bc90 Mon Sep 17 00:00:00 2001 From: tomer doron Date: Mon, 21 Mar 2022 18:25:54 -0700 Subject: [PATCH 04/18] Update Plugins/AWSLambdaPackager/Plugin.swift Co-authored-by: Yim Lee --- Plugins/AWSLambdaPackager/Plugin.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Plugins/AWSLambdaPackager/Plugin.swift b/Plugins/AWSLambdaPackager/Plugin.swift index dda31ad8..c6e573b8 100644 --- a/Plugins/AWSLambdaPackager/Plugin.swift +++ b/Plugins/AWSLambdaPackager/Plugin.swift @@ -176,7 +176,7 @@ struct AWSLambdaPackager: CommandPlugin { #if os(macOS) || os(Linux) let arguments = ["--junk-paths", "--symlinks", zipfilePath.string, relocatedArtifactPath.string, symbolicLinkPath.string] #else - throw Error.unsupportedPlatform("can't or don't know how to create a zipfile on this platform") + throw Error.unsupportedPlatform("can't or don't know how to create a zip file on this platform") #endif // run the zip tool From df4ab5d0a2b3122930df183bb3f3ec9971a23f06 Mon Sep 17 00:00:00 2001 From: tom doron Date: Tue, 12 Apr 2022 21:12:42 -0700 Subject: [PATCH 05/18] fixup --- Plugins/AWSLambdaPackager/Plugin.swift | 90 ++++++++++++++------------ 1 file changed, 50 insertions(+), 40 deletions(-) diff --git a/Plugins/AWSLambdaPackager/Plugin.swift b/Plugins/AWSLambdaPackager/Plugin.swift index c6e573b8..060bd848 100644 --- a/Plugins/AWSLambdaPackager/Plugin.swift +++ b/Plugins/AWSLambdaPackager/Plugin.swift @@ -32,6 +32,7 @@ struct AWSLambdaPackager: CommandPlugin { if self.isAmazonLinux2() { // build directly on the machine builtProducts = try self.build( + packageIdentity: context.package.id, products: configuration.products, buildConfiguration: configuration.buildConfiguration, verboseLogging: configuration.verboseLogging @@ -76,57 +77,51 @@ struct AWSLambdaPackager: CommandPlugin { ) throws -> [LambdaProduct: Path] { let dockerToolPath = try toolsProvider("docker") - /* - if verboseLogging { - print("-------------------------------------------------------------------------") - print("preparing docker build image") - print("-------------------------------------------------------------------------") - } - let packageDockerFilePath = packageDirectory.appending("Dockerfile") - let tempDockerFilePath = outputDirectory.appending("Dockerfile") - try FileManager.default.createDirectory(atPath: tempDockerFilePath.removingLastComponent().string, withIntermediateDirectories: true) - if FileManager.default.fileExists(atPath: packageDockerFilePath.string) { - try FileManager.default.copyItem(atPath: packageDockerFilePath.string, toPath: tempDockerFilePath.string) - } else { - FileManager.default.createFile(atPath: tempDockerFilePath.string, contents: "FROM \(baseImage)".data(using: .utf8)) - } - try self.execute( - executable: dockerToolPath, - arguments: ["build", "-f", tempDockerFilePath.string, packageDirectory.string , "-t", "\(builderImageName)"], - verboseLogging: verboseLogging - ) - */ - - let builderImageName = baseImage + print("-------------------------------------------------------------------------") + print("building \"\(packageIdentity)\" in docker") + print("-------------------------------------------------------------------------") + + // get the build output path + let buildOutputPathCommand = "swift build -c \(buildConfiguration.rawValue) --show-bin-path" + let dockerBuildOutputPath = try self.execute( + executable: dockerToolPath, + arguments: ["run", "--rm", "-v", "\(packageDirectory.string):/workspace", "-w", "/workspace", baseImage, "bash", "-cl", buildOutputPathCommand], + logLevel: verboseLogging ? .debug : .silent + ) + let buildOutputPath = Path(dockerBuildOutputPath.replacingOccurrences(of: "/workspace", with: packageDirectory.string)) + // build the products var builtProducts = [LambdaProduct: Path]() for product in products { - print("-------------------------------------------------------------------------") - print("building \"\(product.name)\" in docker") - print("-------------------------------------------------------------------------") - // - let buildCommand = "swift build --product \(product.name) -c \(buildConfiguration.rawValue) --static-swift-stdlib" + print("building \"\(product.name)\"") + let buildCommand = "swift build -c \(buildConfiguration.rawValue) --product \(product.name) --static-swift-stdlib" try self.execute( executable: dockerToolPath, - arguments: ["run", "--rm", "-v", "\(packageDirectory.string):/workspace", "-w", "/workspace", builderImageName, "bash", "-cl", buildCommand], - verboseLogging: verboseLogging + arguments: ["run", "--rm", "-v", "\(packageDirectory.string):/workspace", "-w", "/workspace", baseImage, "bash", "-cl", buildCommand], + logLevel: verboseLogging ? .debug : .output ) - // TODO: this knows too much about the underlying implementation - builtProducts[.init(product)] = packageDirectory.appending([".build", buildConfiguration.rawValue, product.name]) + let productPath = buildOutputPath.appending(product.name) + guard FileManager.default.fileExists(atPath: productPath.string) else { + throw Errors.productExecutableNotFound("could not find executable for '\(product.name)', expected at '\(productPath)'") + } + builtProducts[.init(product)] = productPath } return builtProducts } private func build( + packageIdentity: Package.ID, products: [Product], buildConfiguration: PackageManager.BuildConfiguration, verboseLogging: Bool ) throws -> [LambdaProduct: Path] { + print("-------------------------------------------------------------------------") + print("building \"\(packageIdentity)\"") + print("-------------------------------------------------------------------------") + var results = [LambdaProduct: Path]() for product in products { - print("-------------------------------------------------------------------------") print("building \"\(product.name)\"") - print("-------------------------------------------------------------------------") var parameters = PackageManager.BuildParameters() parameters.configuration = buildConfiguration parameters.otherSwiftcFlags = ["-static-stdlib"] @@ -183,7 +178,7 @@ struct AWSLambdaPackager: CommandPlugin { try self.execute( executable: zipToolPath, arguments: arguments, - verboseLogging: verboseLogging + logLevel: verboseLogging ? .debug : .silent ) archives[product] = zipfilePath @@ -196,9 +191,9 @@ struct AWSLambdaPackager: CommandPlugin { executable: Path, arguments: [String], customWorkingDirectory: Path? = .none, - verboseLogging: Bool + logLevel: ProcessLogLevel ) throws -> String { - if verboseLogging { + if logLevel >= .debug { print("\(executable.string) \(arguments.joined(separator: " "))") } @@ -207,13 +202,17 @@ struct AWSLambdaPackager: CommandPlugin { let outputQueue = DispatchQueue(label: "AWSLambdaPackager.output") let outputHandler = { (data: Data?) in dispatchPrecondition(condition: .onQueue(outputQueue)) + + sync.enter() + defer { sync.leave() } + guard let _output = data.flatMap({ String(data: $0, encoding: .utf8)?.trimmingCharacters(in: CharacterSet(["\n"])) }), !_output.isEmpty else { return } - sync.enter() - defer { sync.leave() } - print(_output) - fflush(stdout) + if logLevel >= .output { + print(_output) + fflush(stdout) + } output += _output } @@ -350,11 +349,22 @@ private enum Errors: Error { case unknownProduct(String) case unknownExecutable(String) case buildError(String) + case productExecutableNotFound(String) case failedWritingDockerfile case processFailed(Int32) case invalidProcessOutput } +private enum ProcessLogLevel: Int, Comparable { + case silent = 0 + case output = 1 + case debug = 2 + + static func < (lhs: ProcessLogLevel, rhs: ProcessLogLevel) -> Bool { + lhs.rawValue < rhs.rawValue + } +} + extension PackageManager.BuildResult { // find the executable produced by the build func executableArtifact(for product: Product) -> PackageManager.BuildResult.BuiltArtifact? { From ff17559d21f0e3ae81ccedcec41268f61298d198 Mon Sep 17 00:00:00 2001 From: tom doron Date: Fri, 15 Apr 2022 11:42:14 -0700 Subject: [PATCH 06/18] swift version --- Examples/Echo/Package.swift | 15 ++++++--------- Plugins/AWSLambdaPackager/Plugin.swift | 9 ++------- 2 files changed, 8 insertions(+), 16 deletions(-) diff --git a/Examples/Echo/Package.swift b/Examples/Echo/Package.swift index 3040a7af..c0bbaa35 100644 --- a/Examples/Echo/Package.swift +++ b/Examples/Echo/Package.swift @@ -3,14 +3,6 @@ import Foundation import PackageDescription -// this is the dependency on the swift-aws-lambda-runtime library -var dependencies = [Package.Dependency]() -if FileManager.default.fileExists(atPath: "../../Package.swift") { - dependencies.append(Package.Dependency.package(name: "swift-aws-lambda-runtime", path: "../..")) -} else { - dependencies.append(Package.Dependency.package(url: "https://github.com/swift-server/swift-aws-lambda-runtime.git", branch: "main")) -} - let package = Package( name: "swift-aws-lambda-runtime-example", platforms: [ @@ -19,7 +11,12 @@ let package = Package( products: [ .executable(name: "MyLambda", targets: ["MyLambda"]), ], - dependencies: dependencies, + dependencies: [ + // this is the dependency on the swift-aws-lambda-runtime library + // in real-world projects this would say + // .package(url: "https://github.com/swift-server/swift-aws-lambda-runtime.git", from: "1.0.0") + .package(name: "swift-aws-lambda-runtime", path: "../.."), + ], targets: [ .executableTarget( name: "MyLambda", diff --git a/Plugins/AWSLambdaPackager/Plugin.swift b/Plugins/AWSLambdaPackager/Plugin.swift index 060bd848..9b360b66 100644 --- a/Plugins/AWSLambdaPackager/Plugin.swift +++ b/Plugins/AWSLambdaPackager/Plugin.swift @@ -315,8 +315,8 @@ private struct Configuration: CustomStringConvertible { throw Errors.invalidArgument("--swift-version and --base-docker-image are mutually exclusive") } - let swiftVersion = swiftVersionArgument.first ?? Self.getSwiftVersion() - self.baseDockerImage = baseDockerImageArgument.first ?? "swift:\(swiftVersion)-amazonlinux2" + let swiftVersion = swiftVersionArgument.first ?? .none // undefined version will yield the latest docker image + self.baseDockerImage = baseDockerImageArgument.first ?? "swift:\(swiftVersion.map { $0 + "-" } ?? "")amazonlinux2" if self.verboseLogging { print("-------------------------------------------------------------------------") @@ -336,11 +336,6 @@ private struct Configuration: CustomStringConvertible { } """ } - - #warning("FIXME: read this programmatically") - private static func getSwiftVersion() -> String { - "5.6" - } } private enum Errors: Error { From 6a758e74b3e6784d3918c116314eb244f880447b Mon Sep 17 00:00:00 2001 From: tom doron Date: Fri, 15 Apr 2022 12:13:09 -0700 Subject: [PATCH 07/18] cleanup --- Package.swift | 2 +- Plugins/AWSLambdaPackager/Plugin.swift | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Package.swift b/Package.swift index 3a1dd937..14a2d85e 100644 --- a/Package.swift +++ b/Package.swift @@ -53,7 +53,7 @@ let package = Package( ]), .testTarget(name: "AWSLambdaTestingTests", dependencies: ["AWSLambdaTesting"]), // for perf testing - .target(name: "MockServer", dependencies: [ + .executableTarget(name: "MockServer", dependencies: [ .product(name: "NIOHTTP1", package: "swift-nio"), .product(name: "NIO", package: "swift-nio"), ]), diff --git a/Plugins/AWSLambdaPackager/Plugin.swift b/Plugins/AWSLambdaPackager/Plugin.swift index 9b360b66..f725b06e 100644 --- a/Plugins/AWSLambdaPackager/Plugin.swift +++ b/Plugins/AWSLambdaPackager/Plugin.swift @@ -139,7 +139,7 @@ struct AWSLambdaPackager: CommandPlugin { return results } - #warning("FIXME: use zlib") + // TODO: explore using ziplib or similar instead of shelling out private func package( products: [LambdaProduct: Path], toolsProvider: (String) throws -> Path, From 1a249d8af54092ae0be22101d51b94eaa207ccb0 Mon Sep 17 00:00:00 2001 From: tom doron Date: Mon, 18 Apr 2022 09:15:33 -0700 Subject: [PATCH 08/18] cleanup --- Examples/Echo/Package.swift | 1 - 1 file changed, 1 deletion(-) diff --git a/Examples/Echo/Package.swift b/Examples/Echo/Package.swift index c0bbaa35..caae8f03 100644 --- a/Examples/Echo/Package.swift +++ b/Examples/Echo/Package.swift @@ -1,6 +1,5 @@ // swift-tools-version:5.5 -import Foundation import PackageDescription let package = Package( From ff7d30ae5b3591ea24bbafbcd2cabeaf161bf1de Mon Sep 17 00:00:00 2001 From: tom doron Date: Mon, 18 Apr 2022 10:04:03 -0700 Subject: [PATCH 09/18] error handling --- Plugins/AWSLambdaPackager/Plugin.swift | 98 +++++++++++++++----------- 1 file changed, 57 insertions(+), 41 deletions(-) diff --git a/Plugins/AWSLambdaPackager/Plugin.swift b/Plugins/AWSLambdaPackager/Plugin.swift index f725b06e..2c262d65 100644 --- a/Plugins/AWSLambdaPackager/Plugin.swift +++ b/Plugins/AWSLambdaPackager/Plugin.swift @@ -102,7 +102,7 @@ struct AWSLambdaPackager: CommandPlugin { ) let productPath = buildOutputPath.appending(product.name) guard FileManager.default.fileExists(atPath: productPath.string) else { - throw Errors.productExecutableNotFound("could not find executable for '\(product.name)', expected at '\(productPath)'") + throw Errors.productExecutableNotFound(product.name) } builtProducts[.init(product)] = productPath } @@ -132,7 +132,7 @@ struct AWSLambdaPackager: CommandPlugin { parameters: parameters ) guard let artifact = result.executableArtifact(for: product) else { - throw Errors.unknownExecutable("no executable artifacts found for \(product.name)") + throw Errors.productExecutableNotFound(product.name) } results[.init(product)] = artifact.path } @@ -197,14 +197,14 @@ struct AWSLambdaPackager: CommandPlugin { print("\(executable.string) \(arguments.joined(separator: " "))") } - let sync = DispatchGroup() var output = "" + let outputSync = DispatchGroup() let outputQueue = DispatchQueue(label: "AWSLambdaPackager.output") let outputHandler = { (data: Data?) in dispatchPrecondition(condition: .onQueue(outputQueue)) - sync.enter() - defer { sync.leave() } + outputSync.enter() + defer { outputSync.leave() } guard let _output = data.flatMap({ String(data: $0, encoding: .utf8)?.trimmingCharacters(in: CharacterSet(["\n"])) }), !_output.isEmpty else { return @@ -216,14 +216,12 @@ struct AWSLambdaPackager: CommandPlugin { output += _output } - let stdoutPipe = Pipe() - stdoutPipe.fileHandleForReading.readabilityHandler = { fileHandle in outputQueue.async { outputHandler(fileHandle.availableData) } } - let stderrPipe = Pipe() - stderrPipe.fileHandleForReading.readabilityHandler = { fileHandle in outputQueue.async { outputHandler(fileHandle.availableData) } } + let pipe = Pipe() + pipe.fileHandleForReading.readabilityHandler = { fileHandle in outputQueue.async { outputHandler(fileHandle.availableData) } } let process = Process() - process.standardOutput = stdoutPipe - process.standardError = stderrPipe + process.standardOutput = pipe + process.standardError = pipe process.executableURL = URL(fileURLWithPath: executable.string) process.arguments = arguments if let workingDirectory = customWorkingDirectory { @@ -231,8 +229,7 @@ struct AWSLambdaPackager: CommandPlugin { } process.terminationHandler = { _ in outputQueue.async { - outputHandler(try? stdoutPipe.fileHandleForReading.readToEnd()) - outputHandler(try? stderrPipe.fileHandleForReading.readToEnd()) + outputHandler(try? pipe.fileHandleForReading.readToEnd()) } } @@ -240,10 +237,15 @@ struct AWSLambdaPackager: CommandPlugin { process.waitUntilExit() // wait for output to be full processed - sync.wait() + outputSync.wait() if process.terminationStatus != 0 { - throw Errors.processFailed(process.terminationStatus) + // print output on failure and if not already printed + if logLevel < .output { + print(output) + fflush(stdout) + } + throw Errors.processFailed([executable.string] + arguments, process.terminationStatus) } return output @@ -282,7 +284,7 @@ private struct Configuration: CustomStringConvertible { if let outputPath = outputPathArgument.first { var isDirectory: ObjCBool = false guard FileManager.default.fileExists(atPath: outputPath, isDirectory: &isDirectory), isDirectory.boolValue else { - throw Errors.invalidArgument("invalid output directory \(outputPath)") + throw Errors.invalidArgument("invalid output directory '\(outputPath)'") } self.outputDirectory = Path(outputPath) } else { @@ -293,7 +295,7 @@ private struct Configuration: CustomStringConvertible { let products = try context.package.products(named: productsArgument) for product in products { guard product is ExecutableProduct else { - throw Errors.invalidArgument("product named \(product.name) is not an executable product") + throw Errors.invalidArgument("product named '\(product.name)' is not an executable product") } } self.products = products @@ -304,7 +306,7 @@ private struct Configuration: CustomStringConvertible { if let buildConfigurationName = configurationArgument.first { guard let buildConfiguration = PackageManager.BuildConfiguration(rawValue: buildConfigurationName) else { - throw Errors.invalidArgument("invalid build configuration named \(buildConfigurationName)") + throw Errors.invalidArgument("invalid build configuration named '\(buildConfigurationName)'") } self.buildConfiguration = buildConfiguration } else { @@ -338,18 +340,6 @@ private struct Configuration: CustomStringConvertible { } } -private enum Errors: Error { - case invalidArgument(String) - case unsupportedPlatform(String) - case unknownProduct(String) - case unknownExecutable(String) - case buildError(String) - case productExecutableNotFound(String) - case failedWritingDockerfile - case processFailed(Int32) - case invalidProcessOutput -} - private enum ProcessLogLevel: Int, Comparable { case silent = 0 case output = 1 @@ -360,21 +350,33 @@ private enum ProcessLogLevel: Int, Comparable { } } -extension PackageManager.BuildResult { - // find the executable produced by the build - func executableArtifact(for product: Product) -> PackageManager.BuildResult.BuiltArtifact? { - let executables = self.builtArtifacts.filter { $0.kind == .executable && $0.path.lastComponent == product.name } - guard !executables.isEmpty else { - return nil - } - guard executables.count == 1, let executable = executables.first else { - return nil +private enum Errors: Error, CustomStringConvertible { + case invalidArgument(String) + case unsupportedPlatform(String) + case unknownProduct(String) + case productExecutableNotFound(String) + case failedWritingDockerfile + case processFailed([String], Int32) + + var description: String { + switch self { + case .invalidArgument(let description): + return description + case .unsupportedPlatform(let description): + return description + case .unknownProduct(let description): + return description + case .productExecutableNotFound(let product): + return "product executable not found '\(product)'" + case .failedWritingDockerfile: + return "failed writing dockerfile" + case .processFailed(let arguments, let code): + return "\(arguments.joined(separator: " ")) failed with code \(code)" } - return executable } } -struct LambdaProduct: Hashable { +private struct LambdaProduct: Hashable { let underlying: Product init(_ underlying: Product) { @@ -393,3 +395,17 @@ struct LambdaProduct: Hashable { lhs.underlying.id == rhs.underlying.id } } + +extension PackageManager.BuildResult { + // find the executable produced by the build + func executableArtifact(for product: Product) -> PackageManager.BuildResult.BuiltArtifact? { + let executables = self.builtArtifacts.filter { $0.kind == .executable && $0.path.lastComponent == product.name } + guard !executables.isEmpty else { + return nil + } + guard executables.count == 1, let executable = executables.first else { + return nil + } + return executable + } +} From 6b63d0e32e309241d2b8834febfd22b9106fc374 Mon Sep 17 00:00:00 2001 From: Fabian Fett Date: Thu, 21 Apr 2022 17:04:07 +0200 Subject: [PATCH 10/18] Fix Package.swift symlink issues (#1) --- Package.swift | 6 ++---- Package@swift-5.5.swift | 1 - ...age@swift-5.4.swift => Package@swift-5.6.swift | 15 +++++++++++++-- 3 files changed, 15 insertions(+), 7 deletions(-) delete mode 120000 Package@swift-5.5.swift rename Package@swift-5.4.swift => Package@swift-5.6.swift (81%) diff --git a/Package.swift b/Package.swift index 14a2d85e..26e3ca11 100644 --- a/Package.swift +++ b/Package.swift @@ -1,4 +1,4 @@ -// swift-tools-version:5.6 +// swift-tools-version:5.4 import PackageDescription @@ -9,8 +9,6 @@ let package = Package( .library(name: "AWSLambdaRuntime", targets: ["AWSLambdaRuntime"]), // this has all the main functionality for lambda and it does not link Foundation .library(name: "AWSLambdaRuntimeCore", targets: ["AWSLambdaRuntimeCore"]), - // plugin to package the lambda, creating an archive that can be uploaded to AWS - .plugin(name: "AWSLambdaPackager", targets: ["AWSLambdaPackager"]), // for testing only .library(name: "AWSLambdaTesting", targets: ["AWSLambdaTesting"]), ], @@ -53,7 +51,7 @@ let package = Package( ]), .testTarget(name: "AWSLambdaTestingTests", dependencies: ["AWSLambdaTesting"]), // for perf testing - .executableTarget(name: "MockServer", dependencies: [ + .target(name: "MockServer", dependencies: [ .product(name: "NIOHTTP1", package: "swift-nio"), .product(name: "NIO", package: "swift-nio"), ]), diff --git a/Package@swift-5.5.swift b/Package@swift-5.5.swift deleted file mode 120000 index bd1da396..00000000 --- a/Package@swift-5.5.swift +++ /dev/null @@ -1 +0,0 @@ -Package@swift-5.4.swift \ No newline at end of file diff --git a/Package@swift-5.4.swift b/Package@swift-5.6.swift similarity index 81% rename from Package@swift-5.4.swift rename to Package@swift-5.6.swift index 90ace14b..9d6a9db3 100644 --- a/Package@swift-5.4.swift +++ b/Package@swift-5.6.swift @@ -1,4 +1,4 @@ -// swift-tools-version:5.4 +// swift-tools-version:5.6 import PackageDescription @@ -9,6 +9,8 @@ let package = Package( .library(name: "AWSLambdaRuntime", targets: ["AWSLambdaRuntime"]), // this has all the main functionality for lambda and it does not link Foundation .library(name: "AWSLambdaRuntimeCore", targets: ["AWSLambdaRuntimeCore"]), + // plugin to package the lambda, creating an archive that can be uploaded to AWS + .plugin(name: "AWSLambdaPackager", targets: ["AWSLambdaPackager"]), // for testing only .library(name: "AWSLambdaTesting", targets: ["AWSLambdaTesting"]), ], @@ -31,6 +33,15 @@ let package = Package( .product(name: "NIOConcurrencyHelpers", package: "swift-nio"), .product(name: "NIOPosix", package: "swift-nio"), ]), + .plugin( + name: "AWSLambdaPackager", + capability: .command( + intent: .custom( + verb: "archive", + description: "Archive the Lambda binary and prepare it for uploading to AWS. Requires docker on macOS." + ) + ) + ), .testTarget(name: "AWSLambdaRuntimeCoreTests", dependencies: [ .byName(name: "AWSLambdaRuntimeCore"), .product(name: "NIOTestUtils", package: "swift-nio"), @@ -47,7 +58,7 @@ let package = Package( ]), .testTarget(name: "AWSLambdaTestingTests", dependencies: ["AWSLambdaTesting"]), // for perf testing - .target(name: "MockServer", dependencies: [ + .executableTarget(name: "MockServer", dependencies: [ .product(name: "NIOHTTP1", package: "swift-nio"), .product(name: "NIO", package: "swift-nio"), ]), From f43f914b6e11632afb83cc75c4344896335f2ba7 Mon Sep 17 00:00:00 2001 From: tom doron Date: Thu, 21 Apr 2022 09:37:36 -0700 Subject: [PATCH 11/18] cleanup --- Package@swift-5.6.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Package@swift-5.6.swift b/Package@swift-5.6.swift index 9d6a9db3..d14a6685 100644 --- a/Package@swift-5.6.swift +++ b/Package@swift-5.6.swift @@ -37,7 +37,7 @@ let package = Package( name: "AWSLambdaPackager", capability: .command( intent: .custom( - verb: "archive", + verb: "archive", description: "Archive the Lambda binary and prepare it for uploading to AWS. Requires docker on macOS." ) ) From 0ddf8fbb7977d0979df36655731b41c878eb5f25 Mon Sep 17 00:00:00 2001 From: tom doron Date: Mon, 16 May 2022 09:49:06 -0700 Subject: [PATCH 12/18] info about products, allways pull latest docker iamge --- Plugins/AWSLambdaPackager/Plugin.swift | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/Plugins/AWSLambdaPackager/Plugin.swift b/Plugins/AWSLambdaPackager/Plugin.swift index 2c262d65..9a474f60 100644 --- a/Plugins/AWSLambdaPackager/Plugin.swift +++ b/Plugins/AWSLambdaPackager/Plugin.swift @@ -28,6 +28,11 @@ struct AWSLambdaPackager: CommandPlugin { throw Errors.unknownProduct("no appropriate products found to package") } + if configuration.products.count > 1 && !configuration.explicitProducts { + let productNames = configuration.products.map(\.name) + print("No explicit products named, building all executable products: '\(productNames.joined(separator: "', '"))'") + } + let builtProducts: [LambdaProduct: Path] if self.isAmazonLinux2() { // build directly on the machine @@ -85,7 +90,7 @@ struct AWSLambdaPackager: CommandPlugin { let buildOutputPathCommand = "swift build -c \(buildConfiguration.rawValue) --show-bin-path" let dockerBuildOutputPath = try self.execute( executable: dockerToolPath, - arguments: ["run", "--rm", "-v", "\(packageDirectory.string):/workspace", "-w", "/workspace", baseImage, "bash", "-cl", buildOutputPathCommand], + arguments: ["run", "--rm", "--pull", "always", "-v", "\(packageDirectory.string):/workspace", "-w", "/workspace", baseImage, "bash", "-cl", buildOutputPathCommand], logLevel: verboseLogging ? .debug : .silent ) let buildOutputPath = Path(dockerBuildOutputPath.replacingOccurrences(of: "/workspace", with: packageDirectory.string)) @@ -263,6 +268,7 @@ struct AWSLambdaPackager: CommandPlugin { private struct Configuration: CustomStringConvertible { public let outputDirectory: Path public let products: [Product] + public let explicitProducts: Bool public let buildConfiguration: PackageManager.BuildConfiguration public let verboseLogging: Bool public let baseDockerImage: String @@ -291,7 +297,8 @@ private struct Configuration: CustomStringConvertible { self.outputDirectory = context.pluginWorkDirectory.appending(subpath: "\(AWSLambdaPackager.self)") } - if !productsArgument.isEmpty { + self.explicitProducts = !productsArgument.isEmpty + if self.explicitProducts { let products = try context.package.products(named: productsArgument) for product in products { guard product is ExecutableProduct else { From 0eed404049b34440685b65ad2c5facdfc5641743 Mon Sep 17 00:00:00 2001 From: tom doron Date: Mon, 16 May 2022 10:03:43 -0700 Subject: [PATCH 13/18] debug --- Plugins/AWSLambdaPackager/Plugin.swift | 1 + 1 file changed, 1 insertion(+) diff --git a/Plugins/AWSLambdaPackager/Plugin.swift b/Plugins/AWSLambdaPackager/Plugin.swift index 9a474f60..4f7daf0f 100644 --- a/Plugins/AWSLambdaPackager/Plugin.swift +++ b/Plugins/AWSLambdaPackager/Plugin.swift @@ -107,6 +107,7 @@ struct AWSLambdaPackager: CommandPlugin { ) let productPath = buildOutputPath.appending(product.name) guard FileManager.default.fileExists(atPath: productPath.string) else { + print("expected '\(product.name)' binary at \"\(productPath.string)\"") throw Errors.productExecutableNotFound(product.name) } builtProducts[.init(product)] = productPath From b2842ba59d7337139d50487c013c202f16861ba3 Mon Sep 17 00:00:00 2001 From: tom doron Date: Mon, 16 May 2022 10:17:41 -0700 Subject: [PATCH 14/18] error handling --- Plugins/AWSLambdaPackager/Plugin.swift | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/Plugins/AWSLambdaPackager/Plugin.swift b/Plugins/AWSLambdaPackager/Plugin.swift index 4f7daf0f..418bd857 100644 --- a/Plugins/AWSLambdaPackager/Plugin.swift +++ b/Plugins/AWSLambdaPackager/Plugin.swift @@ -93,7 +93,10 @@ struct AWSLambdaPackager: CommandPlugin { arguments: ["run", "--rm", "--pull", "always", "-v", "\(packageDirectory.string):/workspace", "-w", "/workspace", baseImage, "bash", "-cl", buildOutputPathCommand], logLevel: verboseLogging ? .debug : .silent ) - let buildOutputPath = Path(dockerBuildOutputPath.replacingOccurrences(of: "/workspace", with: packageDirectory.string)) + guard let buildPathOutput = dockerBuildOutputPath.split(separator: "\n").last else { + throw Errors.failedParsingDockerOutput(dockerBuildOutputPath) + } + let buildOutputPath = Path(buildPathOutput.replacingOccurrences(of: "/workspace", with: packageDirectory.string)) // build the products var builtProducts = [LambdaProduct: Path]() @@ -107,7 +110,7 @@ struct AWSLambdaPackager: CommandPlugin { ) let productPath = buildOutputPath.appending(product.name) guard FileManager.default.fileExists(atPath: productPath.string) else { - print("expected '\(product.name)' binary at \"\(productPath.string)\"") + Diagnostics.error("expected '\(product.name)' binary at \"\(productPath.string)\"") throw Errors.productExecutableNotFound(product.name) } builtProducts[.init(product)] = productPath @@ -219,7 +222,7 @@ struct AWSLambdaPackager: CommandPlugin { print(_output) fflush(stdout) } - output += _output + output += _output + "\n" } let pipe = Pipe() @@ -364,6 +367,7 @@ private enum Errors: Error, CustomStringConvertible { case unknownProduct(String) case productExecutableNotFound(String) case failedWritingDockerfile + case failedParsingDockerOutput(String) case processFailed([String], Int32) var description: String { @@ -378,6 +382,8 @@ private enum Errors: Error, CustomStringConvertible { return "product executable not found '\(product)'" case .failedWritingDockerfile: return "failed writing dockerfile" + case .failedParsingDockerOutput(let output): + return "failed parsing docker output: '\(output)'" case .processFailed(let arguments, let code): return "\(arguments.joined(separator: " ")) failed with code \(code)" } From 0457a3767e84db8bead430aa9b387da3b6713457 Mon Sep 17 00:00:00 2001 From: tom doron Date: Mon, 16 May 2022 11:14:24 -0700 Subject: [PATCH 15/18] fixup --- Package.swift | 4 ---- 1 file changed, 4 deletions(-) diff --git a/Package.swift b/Package.swift index 26e3ca11..90ace14b 100644 --- a/Package.swift +++ b/Package.swift @@ -31,10 +31,6 @@ let package = Package( .product(name: "NIOConcurrencyHelpers", package: "swift-nio"), .product(name: "NIOPosix", package: "swift-nio"), ]), - .plugin( - name: "AWSLambdaPackager", - capability: .command(intent: .custom(verb: "archive", description: "Archive the Lambda binary and prepare it for uploading to AWS. Requires docker on macOS.")) - ), .testTarget(name: "AWSLambdaRuntimeCoreTests", dependencies: [ .byName(name: "AWSLambdaRuntimeCore"), .product(name: "NIOTestUtils", package: "swift-nio"), From 820eb62027054e87dcca22ef6a094f498b3686cd Mon Sep 17 00:00:00 2001 From: tomer doron Date: Mon, 16 May 2022 18:53:42 -0700 Subject: [PATCH 16/18] Update Package@swift-5.6.swift Co-authored-by: Fabian Fett --- Package@swift-5.6.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Package@swift-5.6.swift b/Package@swift-5.6.swift index d14a6685..e0b7aaf2 100644 --- a/Package@swift-5.6.swift +++ b/Package@swift-5.6.swift @@ -38,7 +38,7 @@ let package = Package( capability: .command( intent: .custom( verb: "archive", - description: "Archive the Lambda binary and prepare it for uploading to AWS. Requires docker on macOS." + description: "Archive the Lambda binary and prepare it for uploading to AWS. Requires docker on macOS or non Amazonlinux 2 distributions." ) ) ), From 936eb8feb6acf7caf3278ee1773d0f15298431ac Mon Sep 17 00:00:00 2001 From: tom doron Date: Thu, 26 May 2022 10:44:10 -0700 Subject: [PATCH 17/18] better way to pull --- Plugins/AWSLambdaPackager/Plugin.swift | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/Plugins/AWSLambdaPackager/Plugin.swift b/Plugins/AWSLambdaPackager/Plugin.swift index 418bd857..5efa0a15 100644 --- a/Plugins/AWSLambdaPackager/Plugin.swift +++ b/Plugins/AWSLambdaPackager/Plugin.swift @@ -86,11 +86,19 @@ struct AWSLambdaPackager: CommandPlugin { print("building \"\(packageIdentity)\" in docker") print("-------------------------------------------------------------------------") + // update the underlying docker image, if necessary + print("updating \"\(baseImage)\" docker image") + try self.execute( + executable: dockerToolPath, + arguments: ["pull", baseImage], + logLevel: .output + ) + // get the build output path let buildOutputPathCommand = "swift build -c \(buildConfiguration.rawValue) --show-bin-path" let dockerBuildOutputPath = try self.execute( executable: dockerToolPath, - arguments: ["run", "--rm", "--pull", "always", "-v", "\(packageDirectory.string):/workspace", "-w", "/workspace", baseImage, "bash", "-cl", buildOutputPathCommand], + arguments: ["run", "--rm", "-v", "\(packageDirectory.string):/workspace", "-w", "/workspace", baseImage, "bash", "-cl", buildOutputPathCommand], logLevel: verboseLogging ? .debug : .silent ) guard let buildPathOutput = dockerBuildOutputPath.split(separator: "\n").last else { From 03ff0661988d314070ac6051c4f2d6b3e87a5cac Mon Sep 17 00:00:00 2001 From: tom doron Date: Thu, 26 May 2022 16:41:17 -0700 Subject: [PATCH 18/18] nicer output --- Plugins/AWSLambdaPackager/Plugin.swift | 39 +++++++++++++++++++++----- 1 file changed, 32 insertions(+), 7 deletions(-) diff --git a/Plugins/AWSLambdaPackager/Plugin.swift b/Plugins/AWSLambdaPackager/Plugin.swift index 5efa0a15..9b4f318c 100644 --- a/Plugins/AWSLambdaPackager/Plugin.swift +++ b/Plugins/AWSLambdaPackager/Plugin.swift @@ -226,11 +226,17 @@ struct AWSLambdaPackager: CommandPlugin { guard let _output = data.flatMap({ String(data: $0, encoding: .utf8)?.trimmingCharacters(in: CharacterSet(["\n"])) }), !_output.isEmpty else { return } - if logLevel >= .output { + + output += _output + "\n" + + switch logLevel { + case .silent: + break + case .debug(let outputIndent), .output(let outputIndent): + print(String(repeating: " ", count: outputIndent), terminator: "") print(_output) fflush(stdout) } - output += _output + "\n" } let pipe = Pipe() @@ -359,13 +365,32 @@ private struct Configuration: CustomStringConvertible { } } -private enum ProcessLogLevel: Int, Comparable { - case silent = 0 - case output = 1 - case debug = 2 +private enum ProcessLogLevel: Comparable { + case silent + case output(outputIndent: Int) + case debug(outputIndent: Int) + + var naturalOrder: Int { + switch self { + case .silent: + return 0 + case .output: + return 1 + case .debug: + return 2 + } + } + + static var output: Self { + .output(outputIndent: 2) + } + + static var debug: Self { + .debug(outputIndent: 2) + } static func < (lhs: ProcessLogLevel, rhs: ProcessLogLevel) -> Bool { - lhs.rawValue < rhs.rawValue + lhs.naturalOrder < rhs.naturalOrder } }