Add patch for external file cache

.github/workflows/MainDistributionPipeline.yml

@@ -14,19 +14,19 @@ concurrency:
 jobs:
   duckdb-stable-build:
     name: Build extension binaries
-    uses: duckdb/extension-ci-tools/.github/workflows/_extension_distribution.yml@v1.2.1
+    uses: duckdb/extension-ci-tools/.github/workflows/_extension_distribution.yml@main
     with:
       extension_name: httpfs
-      duckdb_version: v1.2.1
-      ci_tools_version: v1.2.1
+      duckdb_version: main
+      ci_tools_version: main
 
   duckdb-stable-deploy:
     name: Deploy extension binaries
     needs: duckdb-stable-build
-    uses: duckdb/extension-ci-tools/.github/workflows/_extension_deploy.yml@v1.2.1
+    uses: duckdb/extension-ci-tools/.github/workflows/_extension_deploy.yml@main
     secrets: inherit
     with:
       extension_name: httpfs
-      duckdb_version: v1.2.1
-      ci_tools_version: v1.2.1
+      duckdb_version: main
+      ci_tools_version: main
       deploy_latest: ${{ startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' }}

extension/httpfs/create_secret_functions.cpp

@@ -116,10 +116,12 @@ unique_ptr<BaseSecret> CreateS3SecretFunctions::CreateSecretFunctionInternal(Cli
 	return std::move(secret);
 }
 
-CreateSecretInfo CreateS3SecretFunctions::GenerateRefreshSecretInfo(const SecretEntry &secret_entry, Value &refresh_info) {
+CreateSecretInput CreateS3SecretFunctions::GenerateRefreshSecretInfo(const SecretEntry &secret_entry, Value &refresh_info) {
 	const auto &kv_secret = dynamic_cast<const KeyValueSecret&>(*secret_entry.secret);
 
-	CreateSecretInfo result(OnCreateConflict::REPLACE_ON_CONFLICT, secret_entry.persist_type);
+	CreateSecretInput result;
+	result.on_conflict = OnCreateConflict::REPLACE_ON_CONFLICT;
+	result.persist_type = SecretPersistType::TEMPORARY;
 
 	result.type = kv_secret.GetType();
 	result.name = kv_secret.GetName();

extension/httpfs/httpfs.cpp

@@ -629,6 +629,11 @@ time_t HTTPFileSystem::GetLastModifiedTime(FileHandle &handle) {
 	return sfh.last_modified;
 }
 
+string HTTPFileSystem::GetVersionTag(FileHandle &handle) {
+	auto &sfh = handle.Cast<HTTPFileHandle>();
+	return sfh.etag;
+}
+
 bool HTTPFileSystem::FileExists(const string &filename, optional_ptr<FileOpener> opener) {
 	try {
 		auto handle = OpenFile(filename, FileFlags::FILE_FLAGS_READ, opener);
@@ -729,6 +734,7 @@ void HTTPFileHandle::Initialize(optional_ptr<FileOpener> opener) {
 		if (found) {
 			last_modified = value.last_modified;
 			length = value.length;
+			etag = value.etag;
 
 			if (flags.OpenForReading()) {
 				read_buffer = duckdb::unique_ptr<data_t[]>(new data_t[READ_BUFFER_LEN]);
@@ -815,9 +821,12 @@ void HTTPFileHandle::Initialize(optional_ptr<FileOpener> opener) {
 			last_modified = mktime(&tm);
 		}
 	}
+	if (!res->headers["Etag"].empty()) {
+		etag = res->headers["Etag"];
+	}
 
 	if (should_write_cache) {
-		current_cache->Insert(path, {length, last_modified});
+		current_cache->Insert(path, {length, last_modified, etag});
 	}
 }
 

extension/httpfs/include/create_secret_functions.hpp

@@ -7,7 +7,6 @@ struct CreateSecretInput;
 struct S3AuthParams;
 class CreateSecretFunction;
 class BaseSecret;
-struct CreateSecretInfo;
 struct SecretEntry;
 
 struct CreateS3SecretFunctions {
@@ -16,7 +15,7 @@ struct CreateS3SecretFunctions {
 	static void Register(DatabaseInstance &instance);
 
 	//! Secret refreshing mechanisms
-	static CreateSecretInfo GenerateRefreshSecretInfo(const SecretEntry &secret_entry, Value &refresh_info);
+	static CreateSecretInput GenerateRefreshSecretInfo(const SecretEntry &secret_entry, Value &refresh_info);
 	static bool TryRefreshS3Secret(ClientContext &context, const SecretEntry &secret_to_refresh);
 
 protected:

extension/httpfs/include/http_metadata_cache.hpp

@@ -18,6 +18,7 @@ namespace duckdb {
 struct HTTPMetadataCacheEntry {
 	idx_t length;
 	time_t last_modified;
+	string etag;
 };
 
 // Simple cache with a max age for an entry to be valid

extension/httpfs/include/httpfs.hpp

@@ -103,6 +103,7 @@ class HTTPFileHandle : public FileHandle {
 	FileOpenFlags flags;
 	idx_t length;
 	time_t last_modified;
+	string etag;
 
 	// When using full file download, the full file will be written to a cached file handle
 	unique_ptr<CachedFileHandle> cached_file_handle;
@@ -183,6 +184,7 @@ class HTTPFileSystem : public FileSystem {
 	void FileSync(FileHandle &handle) override;
 	int64_t GetFileSize(FileHandle &handle) override;
 	time_t GetLastModifiedTime(FileHandle &handle) override;
+	string GetVersionTag(FileHandle &handle) override;
 	bool FileExists(const string &filename, optional_ptr<FileOpener> opener) override;
 	void Seek(FileHandle &handle, idx_t location) override;
 	idx_t SeekPosition(FileHandle &handle) override;