diff --git a/.github/workflows/MainDistributionPipeline.yml b/.github/workflows/MainDistributionPipeline.yml index d14857f2..9209ee88 100644 --- a/.github/workflows/MainDistributionPipeline.yml +++ b/.github/workflows/MainDistributionPipeline.yml @@ -14,19 +14,19 @@ concurrency: jobs: duckdb-stable-build: name: Build extension binaries - uses: duckdb/extension-ci-tools/.github/workflows/_extension_distribution.yml@v1.2.1 + uses: duckdb/extension-ci-tools/.github/workflows/_extension_distribution.yml@main with: extension_name: httpfs - duckdb_version: v1.2.1 - ci_tools_version: v1.2.1 + duckdb_version: main + ci_tools_version: main duckdb-stable-deploy: name: Deploy extension binaries needs: duckdb-stable-build - uses: duckdb/extension-ci-tools/.github/workflows/_extension_deploy.yml@v1.2.1 + uses: duckdb/extension-ci-tools/.github/workflows/_extension_deploy.yml@main secrets: inherit with: extension_name: httpfs - duckdb_version: v1.2.1 - ci_tools_version: v1.2.1 + duckdb_version: main + ci_tools_version: main deploy_latest: ${{ startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' }} \ No newline at end of file diff --git a/duckdb b/duckdb index 8e52ec43..e1dcf13e 160000 --- a/duckdb +++ b/duckdb @@ -1 +1 @@ -Subproject commit 8e52ec43959ab363643d63cb78ee214577111da4 +Subproject commit e1dcf13ed533d2f071d94597b4144d83242e78b9 diff --git a/extension-ci-tools b/extension-ci-tools index 58970c53..cca140d4 160000 --- a/extension-ci-tools +++ b/extension-ci-tools @@ -1 +1 @@ -Subproject commit 58970c538d35919db875096460c05806056f4de0 +Subproject commit cca140d4cc47f3f3e40f29b49c305bd92845771f diff --git a/extension/httpfs/create_secret_functions.cpp b/extension/httpfs/create_secret_functions.cpp index 52fdeae0..c93e8baa 100644 --- a/extension/httpfs/create_secret_functions.cpp +++ b/extension/httpfs/create_secret_functions.cpp @@ -116,10 +116,12 @@ unique_ptr CreateS3SecretFunctions::CreateSecretFunctionInternal(Cli return std::move(secret); } -CreateSecretInfo CreateS3SecretFunctions::GenerateRefreshSecretInfo(const SecretEntry &secret_entry, Value &refresh_info) { +CreateSecretInput CreateS3SecretFunctions::GenerateRefreshSecretInfo(const SecretEntry &secret_entry, Value &refresh_info) { const auto &kv_secret = dynamic_cast(*secret_entry.secret); - CreateSecretInfo result(OnCreateConflict::REPLACE_ON_CONFLICT, secret_entry.persist_type); + CreateSecretInput result; + result.on_conflict = OnCreateConflict::REPLACE_ON_CONFLICT; + result.persist_type = SecretPersistType::TEMPORARY; result.type = kv_secret.GetType(); result.name = kv_secret.GetName(); diff --git a/extension/httpfs/httpfs.cpp b/extension/httpfs/httpfs.cpp index e18ca418..28ad64f6 100644 --- a/extension/httpfs/httpfs.cpp +++ b/extension/httpfs/httpfs.cpp @@ -629,6 +629,11 @@ time_t HTTPFileSystem::GetLastModifiedTime(FileHandle &handle) { return sfh.last_modified; } +string HTTPFileSystem::GetVersionTag(FileHandle &handle) { + auto &sfh = handle.Cast(); + return sfh.etag; +} + bool HTTPFileSystem::FileExists(const string &filename, optional_ptr opener) { try { auto handle = OpenFile(filename, FileFlags::FILE_FLAGS_READ, opener); @@ -729,6 +734,7 @@ void HTTPFileHandle::Initialize(optional_ptr opener) { if (found) { last_modified = value.last_modified; length = value.length; + etag = value.etag; if (flags.OpenForReading()) { read_buffer = duckdb::unique_ptr(new data_t[READ_BUFFER_LEN]); @@ -815,9 +821,12 @@ void HTTPFileHandle::Initialize(optional_ptr opener) { last_modified = mktime(&tm); } } + if (!res->headers["Etag"].empty()) { + etag = res->headers["Etag"]; + } if (should_write_cache) { - current_cache->Insert(path, {length, last_modified}); + current_cache->Insert(path, {length, last_modified, etag}); } } diff --git a/extension/httpfs/include/create_secret_functions.hpp b/extension/httpfs/include/create_secret_functions.hpp index e0252f51..54b7566d 100644 --- a/extension/httpfs/include/create_secret_functions.hpp +++ b/extension/httpfs/include/create_secret_functions.hpp @@ -7,7 +7,6 @@ struct CreateSecretInput; struct S3AuthParams; class CreateSecretFunction; class BaseSecret; -struct CreateSecretInfo; struct SecretEntry; struct CreateS3SecretFunctions { @@ -16,7 +15,7 @@ struct CreateS3SecretFunctions { static void Register(DatabaseInstance &instance); //! Secret refreshing mechanisms - static CreateSecretInfo GenerateRefreshSecretInfo(const SecretEntry &secret_entry, Value &refresh_info); + static CreateSecretInput GenerateRefreshSecretInfo(const SecretEntry &secret_entry, Value &refresh_info); static bool TryRefreshS3Secret(ClientContext &context, const SecretEntry &secret_to_refresh); protected: diff --git a/extension/httpfs/include/http_metadata_cache.hpp b/extension/httpfs/include/http_metadata_cache.hpp index 73d032b0..8fc7909c 100644 --- a/extension/httpfs/include/http_metadata_cache.hpp +++ b/extension/httpfs/include/http_metadata_cache.hpp @@ -18,6 +18,7 @@ namespace duckdb { struct HTTPMetadataCacheEntry { idx_t length; time_t last_modified; + string etag; }; // Simple cache with a max age for an entry to be valid diff --git a/extension/httpfs/include/httpfs.hpp b/extension/httpfs/include/httpfs.hpp index 93258cec..f8620a24 100644 --- a/extension/httpfs/include/httpfs.hpp +++ b/extension/httpfs/include/httpfs.hpp @@ -103,6 +103,7 @@ class HTTPFileHandle : public FileHandle { FileOpenFlags flags; idx_t length; time_t last_modified; + string etag; // When using full file download, the full file will be written to a cached file handle unique_ptr cached_file_handle; @@ -183,6 +184,7 @@ class HTTPFileSystem : public FileSystem { void FileSync(FileHandle &handle) override; int64_t GetFileSize(FileHandle &handle) override; time_t GetLastModifiedTime(FileHandle &handle) override; + string GetVersionTag(FileHandle &handle) override; bool FileExists(const string &filename, optional_ptr opener) override; void Seek(FileHandle &handle, idx_t location) override; idx_t SeekPosition(FileHandle &handle) override;