From 9e49f078b18d2217aca5965f6e2b1c24bc17d8b1 Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Thu, 15 May 2025 00:03:40 +0200 Subject: [PATCH 1/2] Apply changes from https://github.com/duckdb/duckdb/pull/17486 --- extension/httpfs/hffs.cpp | 15 ++- extension/httpfs/httpfs.cpp | 79 ++++++------ extension/httpfs/httpfs_client.cpp | 133 +-------------------- extension/httpfs/include/hffs.hpp | 4 +- extension/httpfs/include/httpfs.hpp | 5 +- extension/httpfs/include/httpfs_client.hpp | 10 +- extension/httpfs/include/s3fs.hpp | 4 +- extension/httpfs/s3fs.cpp | 20 ++-- 8 files changed, 72 insertions(+), 198 deletions(-) diff --git a/extension/httpfs/hffs.cpp b/extension/httpfs/hffs.cpp index a7c70d87..d91e2cf2 100644 --- a/extension/httpfs/hffs.cpp +++ b/extension/httpfs/hffs.cpp @@ -45,7 +45,7 @@ static string ParseNextUrlFromLinkHeader(const string &link_header_content) { HFFileHandle::~HFFileHandle() {}; unique_ptr HFFileHandle::CreateClient() { - return http_params.http_util->InitializeClient(http_params, parsed_url.endpoint); + return http_params.http_util.InitializeClient(http_params, parsed_url.endpoint); } string HuggingFaceFileSystem::ListHFRequest(ParsedHFUrl &url, HTTPFSParams &http_params, string &next_page_url, @@ -69,7 +69,7 @@ string HuggingFaceFileSystem::ListHFRequest(ParsedHFUrl &url, HTTPFSParams &http response << string(const_char_ptr_cast(data), data_length); return true; }); - auto res = http_params.http_util->Request(get_request); + auto res = http_params.http_util.Request(get_request); if (res->status != HTTPStatusCode::OK_200) { throw IOException(res->GetError() + " error for HTTP GET to '" + next_page_url + "'"); } @@ -205,7 +205,9 @@ vector HuggingFaceFileSystem::Glob(const string &path, FileOpener FileOpenerInfo info; info.file_path = path; - auto http_params = HTTPFSParams::ReadFrom(opener, info); + auto http_util = HTTPFSUtil::GetHTTPUtil(opener); + auto params = http_util->InitializeParameters(opener, info); + auto &http_params = params->Cast(); SetParams(http_params, path, opener); auto http_state = HTTPState::TryGetState(opener).get(); @@ -278,10 +280,11 @@ unique_ptr HuggingFaceFileSystem::CreateHandle(const OpenFileInf FileOpenerInfo info; info.file_path = file.path; - auto params = HTTPFSParams::ReadFrom(opener, info); - SetParams(params, file.path, opener); + auto http_util = HTTPFSUtil::GetHTTPUtil(opener); + auto params = http_util->InitializeParameters(opener, info); + SetParams(params->Cast(), file.path, opener); - return duckdb::make_uniq(*this, std::move(parsed_url), file, flags, params); + return duckdb::make_uniq(*this, std::move(parsed_url), file, flags, std::move(params)); } void HuggingFaceFileSystem::SetParams(HTTPFSParams ¶ms, const string &path, optional_ptr opener) { diff --git a/extension/httpfs/httpfs.cpp b/extension/httpfs/httpfs.cpp index 24123b26..b1115629 100644 --- a/extension/httpfs/httpfs.cpp +++ b/extension/httpfs/httpfs.cpp @@ -21,7 +21,7 @@ namespace duckdb { -shared_ptr GetHTTPUtil(optional_ptr opener) { +shared_ptr HTTPFSUtil::GetHTTPUtil(optional_ptr opener) { if (opener) { auto db = opener->TryGetDatabase(); if (db) { @@ -32,53 +32,43 @@ shared_ptr GetHTTPUtil(optional_ptr opener) { return make_shared_ptr(); } -HTTPFSParams HTTPFSParams::ReadFrom(optional_ptr opener, optional_ptr info) { - HTTPFSParams result; - result.http_util = GetHTTPUtil(opener); +unique_ptr HTTPFSUtil::InitializeParameters(optional_ptr opener, optional_ptr info) { + auto result = make_uniq(*this); + result->Initialize(opener); // No point in continueing without an opener if (!opener) { - return result; + return std::move(result); } Value value; // Setting lookups - FileOpener::TryGetCurrentSetting(opener, "http_timeout", result.timeout, info); - FileOpener::TryGetCurrentSetting(opener, "force_download", result.force_download, info); - FileOpener::TryGetCurrentSetting(opener, "http_retries", result.retries, info); - FileOpener::TryGetCurrentSetting(opener, "http_retry_wait_ms", result.retry_wait_ms, info); - FileOpener::TryGetCurrentSetting(opener, "http_retry_backoff", result.retry_backoff, info); - FileOpener::TryGetCurrentSetting(opener, "http_keep_alive", result.keep_alive, info); - FileOpener::TryGetCurrentSetting(opener, "enable_server_cert_verification", result.enable_server_cert_verification, + FileOpener::TryGetCurrentSetting(opener, "http_timeout", result->timeout, info); + FileOpener::TryGetCurrentSetting(opener, "force_download", result->force_download, info); + FileOpener::TryGetCurrentSetting(opener, "http_retries", result->retries, info); + FileOpener::TryGetCurrentSetting(opener, "http_retry_wait_ms", result->retry_wait_ms, info); + FileOpener::TryGetCurrentSetting(opener, "http_retry_backoff", result->retry_backoff, info); + FileOpener::TryGetCurrentSetting(opener, "http_keep_alive", result->keep_alive, info); + FileOpener::TryGetCurrentSetting(opener, "enable_server_cert_verification", result->enable_server_cert_verification, info); - FileOpener::TryGetCurrentSetting(opener, "ca_cert_file", result.ca_cert_file, info); - FileOpener::TryGetCurrentSetting(opener, "hf_max_per_page", result.hf_max_per_page, info); + FileOpener::TryGetCurrentSetting(opener, "ca_cert_file", result->ca_cert_file, info); + FileOpener::TryGetCurrentSetting(opener, "hf_max_per_page", result->hf_max_per_page, info); // HTTP Secret lookups KeyValueSecretReader settings_reader(*opener, info, "http"); - auto client_context = FileOpener::TryGetClientContext(opener); - if (client_context) { - result.Initialize(*client_context); - } else { - auto db = FileOpener::TryGetDatabase(opener); - if (db) { - result.Initialize(*db); - } - } - string proxy_setting; if (settings_reader.TryGetSecretKey("http_proxy", proxy_setting) && !proxy_setting.empty()) { idx_t port; string host; HTTPUtil::ParseHTTPProxyHost(proxy_setting, host, port); - result.http_proxy = host; - result.http_proxy_port = port; + result->http_proxy = host; + result->http_proxy_port = port; } - settings_reader.TryGetSecretKey("http_proxy_username", result.http_proxy_username); - settings_reader.TryGetSecretKey("http_proxy_password", result.http_proxy_password); - settings_reader.TryGetSecretKey("bearer_token", result.bearer_token); + settings_reader.TryGetSecretKey("http_proxy_username", result->http_proxy_username); + settings_reader.TryGetSecretKey("http_proxy_password", result->http_proxy_password); + settings_reader.TryGetSecretKey("bearer_token", result->bearer_token); Value extra_headers; if (settings_reader.TryGetSecretKey("extra_http_headers", extra_headers)) { @@ -86,11 +76,11 @@ HTTPFSParams HTTPFSParams::ReadFrom(optional_ptr opener, optional_pt for (const auto &child : children) { auto kv = StructValue::GetChildren(child); D_ASSERT(kv.size() == 2); - result.extra_headers[kv[0].GetValue()] = kv[1].GetValue(); + result->extra_headers[kv[0].GetValue()] = kv[1].GetValue(); } } - return result; + return std::move(result); } unique_ptr HTTPClientCache::GetClient() { @@ -113,7 +103,7 @@ unique_ptr HTTPFileSystem::PostRequest(FileHandle &handle, string string &buffer_out, char *buffer_in, idx_t buffer_in_len, string params) { auto &hfh = handle.Cast(); - auto &http_util = *hfh.http_params.http_util; + auto &http_util = hfh.http_params.http_util; PostRequestInfo post_request(url, header_map, hfh.http_params, const_data_ptr_cast(buffer_in), buffer_in_len); auto result = http_util.Request(post_request); buffer_out = std::move(post_request.buffer_out); @@ -123,7 +113,7 @@ unique_ptr HTTPFileSystem::PostRequest(FileHandle &handle, string unique_ptr HTTPFileSystem::PutRequest(FileHandle &handle, string url, HTTPHeaders header_map, char *buffer_in, idx_t buffer_in_len, string params) { auto &hfh = handle.Cast(); - auto &http_util = *hfh.http_params.http_util; + auto &http_util = hfh.http_params.http_util; string content_type = "application/octet-stream"; PutRequestInfo put_request(url, header_map, hfh.http_params, (const_data_ptr_t)buffer_in, buffer_in_len, content_type); @@ -132,7 +122,7 @@ unique_ptr HTTPFileSystem::PutRequest(FileHandle &handle, string u unique_ptr HTTPFileSystem::HeadRequest(FileHandle &handle, string url, HTTPHeaders header_map) { auto &hfh = handle.Cast(); - auto &http_util = *hfh.http_params.http_util; + auto &http_util = hfh.http_params.http_util; auto http_client = hfh.GetClient(); HeadRequestInfo head_request(url, header_map, hfh.http_params); @@ -144,7 +134,7 @@ unique_ptr HTTPFileSystem::HeadRequest(FileHandle &handle, string unique_ptr HTTPFileSystem::DeleteRequest(FileHandle &handle, string url, HTTPHeaders header_map) { auto &hfh = handle.Cast(); - auto &http_util = *hfh.http_params.http_util; + auto &http_util = hfh.http_params.http_util; auto http_client = hfh.GetClient(); DeleteRequestInfo delete_request(url, header_map, hfh.http_params); auto response = http_util.Request(delete_request, http_client); @@ -166,7 +156,7 @@ HTTPException HTTPFileSystem::GetHTTPError(FileHandle &, const HTTPResponse &res unique_ptr HTTPFileSystem::GetRequest(FileHandle &handle, string url, HTTPHeaders header_map) { auto &hfh = handle.Cast(); - auto &http_util = *hfh.http_params.http_util; + auto &http_util = hfh.http_params.http_util; D_ASSERT(hfh.cached_file_handle); @@ -215,7 +205,7 @@ unique_ptr HTTPFileSystem::GetRequest(FileHandle &handle, string u unique_ptr HTTPFileSystem::GetRangeRequest(FileHandle &handle, string url, HTTPHeaders header_map, idx_t file_offset, char *buffer_out, idx_t buffer_out_len) { auto &hfh = handle.Cast(); - auto &http_util = *hfh.http_params.http_util; + auto &http_util = hfh.http_params.http_util; // send the Range header to read only subset of file string range_expr = "bytes=" + to_string(file_offset) + "-" + to_string(file_offset + buffer_out_len - 1); @@ -286,8 +276,8 @@ void TimestampToTimeT(timestamp_t timestamp, time_t &result) { } HTTPFileHandle::HTTPFileHandle(FileSystem &fs, const OpenFileInfo &file, FileOpenFlags flags, - HTTPFSParams http_params_p) - : FileHandle(fs, file.path, flags), http_params(std::move(http_params_p)), flags(flags), length(0), + unique_ptr params_p) + : FileHandle(fs, file.path, flags), params(std::move(params_p)), http_params(params->Cast()), flags(flags), length(0), buffer_available(0), buffer_idx(0), file_offset(0), buffer_start(0), buffer_end(0) { // check if the handle has extended properties that can be set directly in the handle // if we have these properties we don't need to do a head request to obtain them later @@ -318,7 +308,9 @@ unique_ptr HTTPFileSystem::CreateHandle(const OpenFileInfo &file FileOpenerInfo info; info.file_path = file.path; - auto params = HTTPFSParams::ReadFrom(opener, info); + + auto http_util = HTTPFSUtil::GetHTTPUtil(opener); + auto params = http_util->InitializeParameters(opener, info); auto secret_manager = FileOpener::TryGetSecretManager(opener); auto transaction = FileOpener::TryGetCatalogTransaction(opener); @@ -327,10 +319,11 @@ unique_ptr HTTPFileSystem::CreateHandle(const OpenFileInfo &file if (secret_match.HasMatch()) { const auto &kv_secret = dynamic_cast(*secret_match.secret_entry->secret); - params.bearer_token = kv_secret.TryGetValue("token", true).ToString(); + auto &httpfs_params = params->Cast(); + httpfs_params.bearer_token = kv_secret.TryGetValue("token", true).ToString(); } } - return duckdb::make_uniq(*this, file, flags, params); + return duckdb::make_uniq(*this, file, flags, std::move(params)); } unique_ptr HTTPFileSystem::OpenFileExtended(const OpenFileInfo &file, FileOpenFlags flags, @@ -711,7 +704,7 @@ unique_ptr HTTPFileHandle::CreateClient() { // Create a new client string path_out, proto_host_port; HTTPUtil::DecomposeURL(path, path_out, proto_host_port); - return http_params.http_util->InitializeClient(http_params, proto_host_port); + return http_params.http_util.InitializeClient(http_params, proto_host_port); } void HTTPFileHandle::StoreClient(unique_ptr client) { diff --git a/extension/httpfs/httpfs_client.cpp b/extension/httpfs/httpfs_client.cpp index 6fb33d92..bf87668e 100644 --- a/extension/httpfs/httpfs_client.cpp +++ b/extension/httpfs/httpfs_client.cpp @@ -167,137 +167,8 @@ unordered_map HTTPFSUtil::ParseGetParameters(const string &text) return result; } -string HTTPFSUtil::GetStatusMessage(HTTPStatusCode status) { - switch (status) { - case HTTPStatusCode::Continue_100: - return "Continue"; - case HTTPStatusCode::SwitchingProtocol_101: - return "Switching Protocol"; - case HTTPStatusCode::Processing_102: - return "Processing"; - case HTTPStatusCode::EarlyHints_103: - return "Early Hints"; - case HTTPStatusCode::OK_200: - return "OK"; - case HTTPStatusCode::Created_201: - return "Created"; - case HTTPStatusCode::Accepted_202: - return "Accepted"; - case HTTPStatusCode::NonAuthoritativeInformation_203: - return "Non-Authoritative Information"; - case HTTPStatusCode::NoContent_204: - return "No Content"; - case HTTPStatusCode::ResetContent_205: - return "Reset Content"; - case HTTPStatusCode::PartialContent_206: - return "Partial Content"; - case HTTPStatusCode::MultiStatus_207: - return "Multi-Status"; - case HTTPStatusCode::AlreadyReported_208: - return "Already Reported"; - case HTTPStatusCode::IMUsed_226: - return "IM Used"; - case HTTPStatusCode::MultipleChoices_300: - return "Multiple Choices"; - case HTTPStatusCode::MovedPermanently_301: - return "Moved Permanently"; - case HTTPStatusCode::Found_302: - return "Found"; - case HTTPStatusCode::SeeOther_303: - return "See Other"; - case HTTPStatusCode::NotModified_304: - return "Not Modified"; - case HTTPStatusCode::UseProxy_305: - return "Use Proxy"; - case HTTPStatusCode::unused_306: - return "unused"; - case HTTPStatusCode::TemporaryRedirect_307: - return "Temporary Redirect"; - case HTTPStatusCode::PermanentRedirect_308: - return "Permanent Redirect"; - case HTTPStatusCode::BadRequest_400: - return "Bad Request"; - case HTTPStatusCode::Unauthorized_401: - return "Unauthorized"; - case HTTPStatusCode::PaymentRequired_402: - return "Payment Required"; - case HTTPStatusCode::Forbidden_403: - return "Forbidden"; - case HTTPStatusCode::NotFound_404: - return "Not Found"; - case HTTPStatusCode::MethodNotAllowed_405: - return "Method Not Allowed"; - case HTTPStatusCode::NotAcceptable_406: - return "Not Acceptable"; - case HTTPStatusCode::ProxyAuthenticationRequired_407: - return "Proxy Authentication Required"; - case HTTPStatusCode::RequestTimeout_408: - return "Request Timeout"; - case HTTPStatusCode::Conflict_409: - return "Conflict"; - case HTTPStatusCode::Gone_410: - return "Gone"; - case HTTPStatusCode::LengthRequired_411: - return "Length Required"; - case HTTPStatusCode::PreconditionFailed_412: - return "Precondition Failed"; - case HTTPStatusCode::PayloadTooLarge_413: - return "Payload Too Large"; - case HTTPStatusCode::UriTooLong_414: - return "URI Too Long"; - case HTTPStatusCode::UnsupportedMediaType_415: - return "Unsupported Media Type"; - case HTTPStatusCode::RangeNotSatisfiable_416: - return "Range Not Satisfiable"; - case HTTPStatusCode::ExpectationFailed_417: - return "Expectation Failed"; - case HTTPStatusCode::ImATeapot_418: - return "I'm a teapot"; - case HTTPStatusCode::MisdirectedRequest_421: - return "Misdirected Request"; - case HTTPStatusCode::UnprocessableContent_422: - return "Unprocessable Content"; - case HTTPStatusCode::Locked_423: - return "Locked"; - case HTTPStatusCode::FailedDependency_424: - return "Failed Dependency"; - case HTTPStatusCode::TooEarly_425: - return "Too Early"; - case HTTPStatusCode::UpgradeRequired_426: - return "Upgrade Required"; - case HTTPStatusCode::PreconditionRequired_428: - return "Precondition Required"; - case HTTPStatusCode::TooManyRequests_429: - return "Too Many Requests"; - case HTTPStatusCode::RequestHeaderFieldsTooLarge_431: - return "Request Header Fields Too Large"; - case HTTPStatusCode::UnavailableForLegalReasons_451: - return "Unavailable For Legal Reasons"; - case HTTPStatusCode::NotImplemented_501: - return "Not Implemented"; - case HTTPStatusCode::BadGateway_502: - return "Bad Gateway"; - case HTTPStatusCode::ServiceUnavailable_503: - return "Service Unavailable"; - case HTTPStatusCode::GatewayTimeout_504: - return "Gateway Timeout"; - case HTTPStatusCode::HttpVersionNotSupported_505: - return "HTTP Version Not Supported"; - case HTTPStatusCode::VariantAlsoNegotiates_506: - return "Variant Also Negotiates"; - case HTTPStatusCode::InsufficientStorage_507: - return "Insufficient Storage"; - case HTTPStatusCode::LoopDetected_508: - return "Loop Detected"; - case HTTPStatusCode::NotExtended_510: - return "Not Extended"; - case HTTPStatusCode::NetworkAuthenticationRequired_511: - return "Network Authentication Required"; - - default: - case HTTPStatusCode::InternalServerError_500: - return "Internal Server Error"; - } +string HTTPFSUtil::GetName() const { + return "HTTPFS"; } } // namespace duckdb diff --git a/extension/httpfs/include/hffs.hpp b/extension/httpfs/include/hffs.hpp index a5901e52..7fe1af11 100644 --- a/extension/httpfs/include/hffs.hpp +++ b/extension/httpfs/include/hffs.hpp @@ -57,8 +57,8 @@ class HFFileHandle : public HTTPFileHandle { public: HFFileHandle(FileSystem &fs, ParsedHFUrl hf_url, const OpenFileInfo &file, FileOpenFlags flags, - const HTTPFSParams &http_params) - : HTTPFileHandle(fs, file, flags, http_params), parsed_url(std::move(hf_url)) { + unique_ptr http_params) + : HTTPFileHandle(fs, file, flags, std::move(http_params)), parsed_url(std::move(hf_url)) { } ~HFFileHandle() override; diff --git a/extension/httpfs/include/httpfs.hpp b/extension/httpfs/include/httpfs.hpp index 37773fe0..a6b8570f 100644 --- a/extension/httpfs/include/httpfs.hpp +++ b/extension/httpfs/include/httpfs.hpp @@ -32,7 +32,7 @@ class HTTPFileSystem; class HTTPFileHandle : public FileHandle { public: - HTTPFileHandle(FileSystem &fs, const OpenFileInfo &file, FileOpenFlags flags, HTTPFSParams params); + HTTPFileHandle(FileSystem &fs, const OpenFileInfo &file, FileOpenFlags flags, unique_ptr params); ~HTTPFileHandle() override; // This two-phase construction allows subclasses more flexible setup. virtual void Initialize(optional_ptr opener); @@ -40,7 +40,8 @@ class HTTPFileHandle : public FileHandle { // We keep an http client stored for connection reuse with keep-alive headers HTTPClientCache client_cache; - HTTPFSParams http_params; + unique_ptr params; + HTTPFSParams &http_params; // File handle info FileOpenFlags flags; diff --git a/extension/httpfs/include/httpfs_client.hpp b/extension/httpfs/include/httpfs_client.hpp index dff9d185..16545e54 100644 --- a/extension/httpfs/include/httpfs_client.hpp +++ b/extension/httpfs/include/httpfs_client.hpp @@ -7,6 +7,8 @@ struct FileOpenerInfo; class HTTPState; struct HTTPFSParams : public HTTPParams { + HTTPFSParams(HTTPUtil &http_util) : HTTPParams(http_util) {} + static constexpr bool DEFAULT_ENABLE_SERVER_CERT_VERIFICATION = false; static constexpr uint64_t DEFAULT_HF_MAX_PER_PAGE = 0; static constexpr bool DEFAULT_FORCE_DOWNLOAD = false; @@ -16,18 +18,18 @@ struct HTTPFSParams : public HTTPParams { idx_t hf_max_per_page = DEFAULT_HF_MAX_PER_PAGE; string ca_cert_file; string bearer_token; - shared_ptr http_util; shared_ptr state; - - static HTTPFSParams ReadFrom(optional_ptr opener, optional_ptr info); }; class HTTPFSUtil : public HTTPUtil { public: + unique_ptr InitializeParameters(optional_ptr opener, optional_ptr info) override; unique_ptr InitializeClient(HTTPParams &http_params, const string &proto_host_port) override; static unordered_map ParseGetParameters(const string &text); - static string GetStatusMessage(HTTPStatusCode status); + static shared_ptr GetHTTPUtil(optional_ptr opener); + + string GetName() const override; }; } // namespace duckdb diff --git a/extension/httpfs/include/s3fs.hpp b/extension/httpfs/include/s3fs.hpp index b15e0d23..83634297 100644 --- a/extension/httpfs/include/s3fs.hpp +++ b/extension/httpfs/include/s3fs.hpp @@ -108,7 +108,7 @@ class S3FileHandle : public HTTPFileHandle { friend class S3FileSystem; public: - S3FileHandle(FileSystem &fs, const OpenFileInfo &file, FileOpenFlags flags, HTTPFSParams http_params_p, + S3FileHandle(FileSystem &fs, const OpenFileInfo &file, FileOpenFlags flags, unique_ptr http_params_p, const S3AuthParams &auth_params_p, const S3ConfigParams &config_params_p) : HTTPFileHandle(fs, file, flags, std::move(http_params_p)), auth_params(auth_params_p), config_params(config_params_p), uploads_in_progress(0), parts_uploaded(0), upload_finalized(false), @@ -238,7 +238,7 @@ class S3FileSystem : public HTTPFileSystem { // Helper class to do s3 ListObjectV2 api call https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjectsV2.html struct AWSListObjectV2 { - static string Request(string &path, HTTPFSParams &http_params, S3AuthParams &s3_auth_params, + static string Request(string &path, HTTPParams &http_params, S3AuthParams &s3_auth_params, string &continuation_token, optional_ptr state, bool use_delimiter = false); static void ParseFileList(string &aws_response, vector &result); static vector ParseCommonPrefix(string &aws_response); diff --git a/extension/httpfs/s3fs.cpp b/extension/httpfs/s3fs.cpp index 700193d7..ce0710fc 100644 --- a/extension/httpfs/s3fs.cpp +++ b/extension/httpfs/s3fs.cpp @@ -275,7 +275,7 @@ unique_ptr S3FileHandle::CreateClient() { auto parsed_url = S3FileSystem::S3UrlParse(path, this->auth_params); string proto_host_port = parsed_url.http_proto + parsed_url.host; - return http_params.http_util->InitializeClient(http_params, proto_host_port); + return http_params.http_util.InitializeClient(http_params, proto_host_port); } // Opens the multipart upload and returns the ID @@ -706,7 +706,10 @@ unique_ptr S3FileSystem::CreateHandle(const OpenFileInfo &file, auto parsed_s3_url = S3UrlParse(file.path, auth_params); ReadQueryParams(parsed_s3_url.query_param, auth_params); - return duckdb::make_uniq(*this, file, flags, HTTPFSParams::ReadFrom(opener, info), auth_params, + auto http_util = HTTPFSUtil::GetHTTPUtil(opener); + auto params = http_util->InitializeParameters(opener, info); + + return duckdb::make_uniq(*this, file, flags, std::move(params), auth_params, S3ConfigParams::ReadFrom(opener)); } @@ -887,7 +890,8 @@ vector S3FileSystem::Glob(const string &glob_pattern, FileOpener * } string shared_path = parsed_glob_url.substr(0, first_wildcard_pos); - auto http_params = HTTPFSParams::ReadFrom(opener, info); + auto http_util = HTTPFSUtil::GetHTTPUtil(opener); + auto http_params = http_util->InitializeParameters(opener, info); ReadQueryParams(parsed_s3_url.query_param, s3_auth_params); @@ -898,7 +902,7 @@ vector S3FileSystem::Glob(const string &glob_pattern, FileOpener * // Main paging loop do { // main listobject call, may - string response_str = AWSListObjectV2::Request(shared_path, http_params, s3_auth_params, + string response_str = AWSListObjectV2::Request(shared_path, *http_params, s3_auth_params, main_continuation_token, HTTPState::TryGetState(opener).get()); main_continuation_token = AWSListObjectV2::ParseContinuationToken(response_str); AWSListObjectV2::ParseFileList(response_str, s3_keys); @@ -914,7 +918,7 @@ vector S3FileSystem::Glob(const string &glob_pattern, FileOpener * string common_prefix_continuation_token; do { auto prefix_res = - AWSListObjectV2::Request(prefix_path, http_params, s3_auth_params, common_prefix_continuation_token, + AWSListObjectV2::Request(prefix_path, *http_params, s3_auth_params, common_prefix_continuation_token, HTTPState::TryGetState(opener).get()); AWSListObjectV2::ParseFileList(prefix_res, s3_keys); auto more_prefixes = AWSListObjectV2::ParseCommonPrefix(prefix_res); @@ -986,7 +990,7 @@ HTTPException S3FileSystem::GetHTTPError(FileHandle &handle, const HTTPResponse } return HTTPFileSystem::GetHTTPError(handle, response, url); } -string AWSListObjectV2::Request(string &path, HTTPFSParams &http_params, S3AuthParams &s3_auth_params, +string AWSListObjectV2::Request(string &path, HTTPParams &http_params, S3AuthParams &s3_auth_params, string &continuation_token, optional_ptr state, bool use_delimiter) { auto parsed_url = S3FileSystem::S3UrlParse(path, s3_auth_params); @@ -1011,7 +1015,7 @@ string AWSListObjectV2::Request(string &path, HTTPFSParams &http_params, S3AuthP create_s3_header(req_path, req_params, parsed_url.host, "s3", "GET", s3_auth_params, "", "", "", ""); // Get requests use fresh connection - auto client = http_params.http_util->InitializeClient(http_params, parsed_url.http_proto + parsed_url.host); + auto client = http_params.http_util.InitializeClient(http_params, parsed_url.http_proto + parsed_url.host); std::stringstream response; GetRequestInfo get_request( parsed_url.host, listobjectv2_url, header_map, http_params, @@ -1028,7 +1032,7 @@ string AWSListObjectV2::Request(string &path, HTTPFSParams &http_params, S3AuthP response << string(const_char_ptr_cast(data), data_length); return true; }); - auto result = http_params.http_util->Request(get_request); + auto result = http_params.http_util.Request(get_request); if (result->HasRequestError()) { throw IOException("%s error for HTTP GET to '%s'", result->GetRequestError(), listobjectv2_url); } From 6a099823db190f06fa8233351d453a732c696ec9 Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Thu, 15 May 2025 08:49:02 +0200 Subject: [PATCH 2/2] Bump to latest duckdb --- duckdb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/duckdb b/duckdb index e36f0286..3c0d4669 160000 --- a/duckdb +++ b/duckdb @@ -1 +1 @@ -Subproject commit e36f0286abb1044940a6fcd7e99cbad52ba8d5d4 +Subproject commit 3c0d466943444de3cfb5fcf2db08e5c4a6eb6c5e