diff --git a/extension/httpfs/httpfs.cpp b/extension/httpfs/httpfs.cpp index 13ac1d22..1bf017bb 100644 --- a/extension/httpfs/httpfs.cpp +++ b/extension/httpfs/httpfs.cpp @@ -391,6 +391,16 @@ unique_ptr HTTPFileSystem::GetRequest(FileHandle &handle, strin return response; } +HTTPException HTTPFileSystem::GetHTTPError(FileHandle &, const duckdb_httplib_openssl::Response &response, const string &url) { + auto status_message = duckdb_httplib_openssl::status_message(response.status); + string error = "HTTP GET error on '" + url + "' (HTTP " + to_string(response.status) + " " + status_message + ")"; + if (response.status == 416) { + error += " This could mean the file was changed. Try disabling the duckdb http metadata cache " + "if enabled, and confirm the server supports range requests."; + } + return HTTPException(response, error); +} + unique_ptr HTTPFileSystem::GetRangeRequest(FileHandle &handle, string url, HeaderMap header_map, idx_t file_offset, char *buffer_out, idx_t buffer_out_len) { auto &hfh = handle.Cast(); @@ -414,12 +424,7 @@ unique_ptr HTTPFileSystem::GetRangeRequest(FileHandle &handle, path.c_str(), *headers, [&](const duckdb_httplib_openssl::Response &response) { if (response.status >= 400) { - string error = "HTTP GET error on '" + url + "' (HTTP " + to_string(response.status) + ")"; - if (response.status == 416) { - error += " This could mean the file was changed. Try disabling the duckdb http metadata cache " - "if enabled, and confirm the server supports range requests."; - } - throw HTTPException(response, error); + throw GetHTTPError(handle, response, url); } if (response.status < 300) { // done redirecting out_offset = 0; diff --git a/extension/httpfs/include/httpfs.hpp b/extension/httpfs/include/httpfs.hpp index 411de2dd..0a083d18 100644 --- a/extension/httpfs/include/httpfs.hpp +++ b/extension/httpfs/include/httpfs.hpp @@ -2,6 +2,7 @@ #include "duckdb/common/case_insensitive_map.hpp" #include "duckdb/common/file_system.hpp" +#include "duckdb/common/exception/http_exception.hpp" #include "http_state.hpp" #include "duckdb/common/pair.hpp" #include "duckdb/common/unordered_map.hpp" @@ -224,6 +225,8 @@ class HTTPFileSystem : public FileSystem { RunRequestWithRetry(const std::function &request, string &url, string method, const HTTPParams ¶ms, const std::function &retry_cb = {}); + virtual HTTPException GetHTTPError(FileHandle &handle, const duckdb_httplib_openssl::Response &response, const string &url); + private: // Global cache mutex global_cache_lock; diff --git a/extension/httpfs/include/s3fs.hpp b/extension/httpfs/include/s3fs.hpp index 229c79e9..f46cdc98 100644 --- a/extension/httpfs/include/s3fs.hpp +++ b/extension/httpfs/include/s3fs.hpp @@ -230,6 +230,8 @@ class S3FileSystem : public HTTPFileSystem { return true; } + static HTTPException GetS3Error(S3AuthParams &s3_auth_params, const duckdb_httplib_openssl::Response &response, const string &url); + protected: static void NotifyUploadsInProgress(S3FileHandle &file_handle); duckdb::unique_ptr CreateHandle(const OpenFileInfo &file, FileOpenFlags flags, @@ -240,6 +242,8 @@ class S3FileSystem : public HTTPFileSystem { // helper for ReadQueryParams void GetQueryParam(const string &key, string ¶m, CPPHTTPLIB_NAMESPACE::Params &query_params); + + HTTPException GetHTTPError(FileHandle &, const duckdb_httplib_openssl::Response &response, const string &url) override; }; // Helper class to do s3 ListObjectV2 api call https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjectsV2.html diff --git a/extension/httpfs/s3fs.cpp b/extension/httpfs/s3fs.cpp index 7ec018c6..ac9faa4c 100644 --- a/extension/httpfs/s3fs.cpp +++ b/extension/httpfs/s3fs.cpp @@ -1076,6 +1076,27 @@ bool S3FileSystem::ListFiles(const string &directory, const std::function(); + return GetS3Error(s3_handle.auth_params, response, url); + } + return HTTPFileSystem::GetHTTPError(handle, response, url); +} + string AWSListObjectV2::Request(string &path, HTTPParams &http_params, S3AuthParams &s3_auth_params, string &continuation_token, optional_ptr state, bool use_delimiter) { auto parsed_url = S3FileSystem::S3UrlParse(path, s3_auth_params); @@ -1108,7 +1129,10 @@ string AWSListObjectV2::Request(string &path, HTTPParams &http_params, S3AuthPar listobjectv2_url.c_str(), *headers, [&](const duckdb_httplib_openssl::Response &response) { if (response.status >= 400) { - throw HTTPException(response, "HTTP GET error on '%s' (HTTP %d)", listobjectv2_url, response.status); + string trimmed_path = path; + StringUtil::RTrim(trimmed_path, "/"); + trimmed_path += listobjectv2_url; + throw S3FileSystem::GetS3Error(s3_auth_params, response, trimmed_path); } return true; },