From 5f6aacdfd6316aab9ac6c5f8ceeb630b7d140993 Mon Sep 17 00:00:00 2001 From: Satyam Singh Date: Thu, 24 Aug 2023 12:23:11 +0530 Subject: [PATCH 1/2] Fetch from task iam when env var is found --- server/src/storage/s3.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/server/src/storage/s3.rs b/server/src/storage/s3.rs index f5d0d3a66..a46d610d6 100644 --- a/server/src/storage/s3.rs +++ b/server/src/storage/s3.rs @@ -25,7 +25,7 @@ use datafusion::datasource::object_store::{ use datafusion::execution::runtime_env::RuntimeConfig; use futures::stream::FuturesUnordered; use futures::{StreamExt, TryStreamExt}; -use object_store::aws::{AmazonS3, AmazonS3Builder, Checksum}; +use object_store::aws::{AmazonS3, AmazonS3Builder, AmazonS3ConfigKey, Checksum}; use object_store::limit::LimitStore; use object_store::path::Path as StorePath; use object_store::{ClientOptions, ObjectStore}; @@ -152,6 +152,13 @@ impl S3Config { .with_secret_access_key(secret_key); } + if let Ok(relative_uri) = std::env::var("AWS_CONTAINER_CREDENTIALS_RELATIVE_URI") { + builder = builder.with_config( + AmazonS3ConfigKey::ContainerCredentialsRelativeUri, + relative_uri, + ); + } + if self.imdsv1_fallback { builder = builder.with_imdsv1_fallback() } From c6bb2d2ca91eeee296a5ad6d5b14cb3de1b94e8d Mon Sep 17 00:00:00 2001 From: Satyam Singh Date: Thu, 24 Aug 2023 14:57:03 +0530 Subject: [PATCH 2/2] Fix --- server/src/storage/s3.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/src/storage/s3.rs b/server/src/storage/s3.rs index a46d610d6..72a1404aa 100644 --- a/server/src/storage/s3.rs +++ b/server/src/storage/s3.rs @@ -46,6 +46,7 @@ use super::{object_storage, ObjectStorageProvider}; // in bytes const MULTIPART_UPLOAD_SIZE: usize = 1024 * 1024 * 100; const CONNECT_TIMEOUT_SECS: u64 = 5; +const AWS_CONTAINER_CREDENTIALS_RELATIVE_URI: &str = "AWS_CONTAINER_CREDENTIALS_RELATIVE_URI"; #[derive(Debug, Clone, clap::Args)] #[command( @@ -152,7 +153,7 @@ impl S3Config { .with_secret_access_key(secret_key); } - if let Ok(relative_uri) = std::env::var("AWS_CONTAINER_CREDENTIALS_RELATIVE_URI") { + if let Ok(relative_uri) = std::env::var(AWS_CONTAINER_CREDENTIALS_RELATIVE_URI) { builder = builder.with_config( AmazonS3ConfigKey::ContainerCredentialsRelativeUri, relative_uri,