From ceb42814649399d556b5e3165c21ea5f1f3b549b Mon Sep 17 00:00:00 2001 From: anandu-kv Date: Fri, 14 Mar 2025 14:43:10 +0530 Subject: [PATCH 1/2] fix: Add support for web identity in s3 config --- awss3-storage/build.gradle | 3 +++ .../com/netflix/conductor/s3/config/S3Configuration.java | 6 ++++++ .../java/com/netflix/conductor/s3/config/S3Properties.java | 5 +++++ docker/server/config/config-postgres.properties | 5 +++-- 4 files changed, 17 insertions(+), 2 deletions(-) diff --git a/awss3-storage/build.gradle b/awss3-storage/build.gradle index 57e9d4fc3a..350ae8bd97 100644 --- a/awss3-storage/build.gradle +++ b/awss3-storage/build.gradle @@ -18,4 +18,7 @@ dependencies { implementation "com.amazonaws:aws-java-sdk-s3:${revAwsSdk}" implementation "org.apache.commons:commons-lang3" + implementation 'software.amazon.awssdk:s3:2.20.146' + implementation 'software.amazon.awssdk:sts:2.20.146' + implementation 'com.amazonaws:aws-java-sdk-sts:1.12.782' } diff --git a/awss3-storage/src/main/java/com/netflix/conductor/s3/config/S3Configuration.java b/awss3-storage/src/main/java/com/netflix/conductor/s3/config/S3Configuration.java index 106f61664d..44dc8a43ed 100644 --- a/awss3-storage/src/main/java/com/netflix/conductor/s3/config/S3Configuration.java +++ b/awss3-storage/src/main/java/com/netflix/conductor/s3/config/S3Configuration.java @@ -21,6 +21,7 @@ import com.netflix.conductor.core.utils.IDGenerator; import com.netflix.conductor.s3.storage.S3PayloadStorage; +import com.amazonaws.auth.WebIdentityTokenCredentialsProvider; import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.AmazonS3ClientBuilder; @@ -41,6 +42,11 @@ public ExternalPayloadStorage s3ExternalPayloadStorage( matchIfMissing = true) @Bean public AmazonS3 amazonS3(S3Properties properties) { + if (properties.getWebIdentity()) { + return AmazonS3ClientBuilder.standard() + .withCredentials(WebIdentityTokenCredentialsProvider.builder().build()) + .build(); + } return AmazonS3ClientBuilder.standard().withRegion(properties.getRegion()).build(); // TODO: Add localstack support to test locally // return AmazonS3ClientBuilder.standard() diff --git a/awss3-storage/src/main/java/com/netflix/conductor/s3/config/S3Properties.java b/awss3-storage/src/main/java/com/netflix/conductor/s3/config/S3Properties.java index f399cd0d7d..f150e97c1d 100644 --- a/awss3-storage/src/main/java/com/netflix/conductor/s3/config/S3Properties.java +++ b/awss3-storage/src/main/java/com/netflix/conductor/s3/config/S3Properties.java @@ -24,6 +24,8 @@ public class S3Properties { /** The s3 bucket name where the payloads will be stored */ private String bucketName = "conductor_payloads"; + private Boolean webIdentity = true; + /** The time (in seconds) for which the signed url will be valid */ @DurationUnit(ChronoUnit.SECONDS) private Duration signedUrlExpirationDuration = Duration.ofSeconds(5); @@ -55,6 +57,9 @@ public void setRegion(String region) { this.region = region; } + public Boolean getWebIdentity() { + return webIdentity; + } // TODO: Add localstack support to test locally // private String endpoint = "http://s3.localhost.localstack.cloud:4566"; // diff --git a/docker/server/config/config-postgres.properties b/docker/server/config/config-postgres.properties index 04b51a58c9..9495729fc1 100755 --- a/docker/server/config/config-postgres.properties +++ b/docker/server/config/config-postgres.properties @@ -1,13 +1,14 @@ # Database persistence type. conductor.db.type=postgres conductor.queue.type=postgres -conductor.external-payload-storage.type=postgres +conductor.external-payload-storage.type=S3 # Database connectivity spring.datasource.url=jdbc:postgresql://postgresdb:5432/postgres spring.datasource.username=conductor spring.datasource.password=conductor +conductor.external-payload-storage.s3.web-identity=false # Indexing Properties conductor.indexing.enabled=true @@ -20,4 +21,4 @@ conductor.metrics-prometheus.enabled=true management.endpoints.web.exposure.include=prometheus # Load sample kitchen-sink workflow -loadSample=true \ No newline at end of file +loadSample=false \ No newline at end of file From 915f300b173258b9c03809ada1e4940fa9c70c24 Mon Sep 17 00:00:00 2001 From: anandu-kv Date: Tue, 18 Mar 2025 14:48:58 +0530 Subject: [PATCH 2/2] fix: Change to default properties file --- .../java/com/netflix/conductor/s3/config/S3Properties.java | 2 +- docker/server/config/config-postgres.properties | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/awss3-storage/src/main/java/com/netflix/conductor/s3/config/S3Properties.java b/awss3-storage/src/main/java/com/netflix/conductor/s3/config/S3Properties.java index f150e97c1d..df28f94c30 100644 --- a/awss3-storage/src/main/java/com/netflix/conductor/s3/config/S3Properties.java +++ b/awss3-storage/src/main/java/com/netflix/conductor/s3/config/S3Properties.java @@ -24,7 +24,7 @@ public class S3Properties { /** The s3 bucket name where the payloads will be stored */ private String bucketName = "conductor_payloads"; - private Boolean webIdentity = true; + private Boolean webIdentity = false; /** The time (in seconds) for which the signed url will be valid */ @DurationUnit(ChronoUnit.SECONDS) diff --git a/docker/server/config/config-postgres.properties b/docker/server/config/config-postgres.properties index 9495729fc1..04b51a58c9 100755 --- a/docker/server/config/config-postgres.properties +++ b/docker/server/config/config-postgres.properties @@ -1,14 +1,13 @@ # Database persistence type. conductor.db.type=postgres conductor.queue.type=postgres -conductor.external-payload-storage.type=S3 +conductor.external-payload-storage.type=postgres # Database connectivity spring.datasource.url=jdbc:postgresql://postgresdb:5432/postgres spring.datasource.username=conductor spring.datasource.password=conductor -conductor.external-payload-storage.s3.web-identity=false # Indexing Properties conductor.indexing.enabled=true @@ -21,4 +20,4 @@ conductor.metrics-prometheus.enabled=true management.endpoints.web.exposure.include=prometheus # Load sample kitchen-sink workflow -loadSample=false \ No newline at end of file +loadSample=true \ No newline at end of file