Better S3 authentication errors

extension/httpfs/httpfs.cpp

@@ -391,6 +391,16 @@ unique_ptr<ResponseWrapper> HTTPFileSystem::GetRequest(FileHandle &handle, strin
 	return response;
 }
 
+HTTPException HTTPFileSystem::GetHTTPError(FileHandle &, const duckdb_httplib_openssl::Response &response, const string &url) {
+	auto status_message = duckdb_httplib_openssl::status_message(response.status);
+	string error = "HTTP GET error on '" + url + "' (HTTP " + to_string(response.status) + " " + status_message + ")";
+	if (response.status == 416) {
+		error += " This could mean the file was changed. Try disabling the duckdb http metadata cache "
+				 "if enabled, and confirm the server supports range requests.";
+	}
+	return HTTPException(response, error);
+}
+
 unique_ptr<ResponseWrapper> HTTPFileSystem::GetRangeRequest(FileHandle &handle, string url, HeaderMap header_map,
                                                             idx_t file_offset, char *buffer_out, idx_t buffer_out_len) {
 	auto &hfh = handle.Cast<HTTPFileHandle>();
@@ -414,12 +424,7 @@ unique_ptr<ResponseWrapper> HTTPFileSystem::GetRangeRequest(FileHandle &handle,
 		    path.c_str(), *headers,
 		    [&](const duckdb_httplib_openssl::Response &response) {
 			    if (response.status >= 400) {
-				    string error = "HTTP GET error on '" + url + "' (HTTP " + to_string(response.status) + ")";
-				    if (response.status == 416) {
-					    error += " This could mean the file was changed. Try disabling the duckdb http metadata cache "
-					             "if enabled, and confirm the server supports range requests.";
-				    }
-				    throw HTTPException(response, error);
+			    	throw GetHTTPError(handle, response, url);
 			    }
 			    if (response.status < 300) { // done redirecting
 				    out_offset = 0;

extension/httpfs/include/httpfs.hpp

@@ -2,6 +2,7 @@
 
 #include "duckdb/common/case_insensitive_map.hpp"
 #include "duckdb/common/file_system.hpp"
+#include "duckdb/common/exception/http_exception.hpp"
 #include "http_state.hpp"
 #include "duckdb/common/pair.hpp"
 #include "duckdb/common/unordered_map.hpp"
@@ -224,6 +225,8 @@ class HTTPFileSystem : public FileSystem {
 	RunRequestWithRetry(const std::function<duckdb_httplib_openssl::Result(void)> &request, string &url, string method,
 	                    const HTTPParams &params, const std::function<void(void)> &retry_cb = {});
 
+	virtual HTTPException GetHTTPError(FileHandle &handle, const duckdb_httplib_openssl::Response &response, const string &url);
+
 private:
 	// Global cache
 	mutex global_cache_lock;

extension/httpfs/include/s3fs.hpp

@@ -230,6 +230,8 @@ class S3FileSystem : public HTTPFileSystem {
 		return true;
 	}
 
+	static HTTPException GetS3Error(S3AuthParams &s3_auth_params, const duckdb_httplib_openssl::Response &response, const string &url);
+
 protected:
 	static void NotifyUploadsInProgress(S3FileHandle &file_handle);
 	duckdb::unique_ptr<HTTPFileHandle> CreateHandle(const OpenFileInfo &file, FileOpenFlags flags,
@@ -240,6 +242,8 @@ class S3FileSystem : public HTTPFileSystem {
 
 	// helper for ReadQueryParams
 	void GetQueryParam(const string &key, string &param, CPPHTTPLIB_NAMESPACE::Params &query_params);
+
+	HTTPException GetHTTPError(FileHandle &, const duckdb_httplib_openssl::Response &response, const string &url) override;
 };
 
 // Helper class to do s3 ListObjectV2 api call https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjectsV2.html

extension/httpfs/s3fs.cpp

@@ -1076,6 +1076,27 @@ bool S3FileSystem::ListFiles(const string &directory, const std::function<void(c
 	return true;
 }
 
+HTTPException S3FileSystem::GetS3Error(S3AuthParams &s3_auth_params, const duckdb_httplib_openssl::Response &response, const string &url) {
+	string region = s3_auth_params.region;
+	string extra_text = "\n\nAuthentication Failure - this is usually caused by invalid or missing credentials.";
+	if (s3_auth_params.secret_access_key.empty() && s3_auth_params.access_key_id.empty()) {
+		extra_text += "\n* No credentials are provided.";
+	} else {
+		extra_text += "\n* Credentials are provided, but they did not work.";
+	}
+	extra_text += "\n* See https://duckdb.org/docs/stable/extensions/httpfs/s3api.html";
+	auto status_message = duckdb_httplib_openssl::status_message(response.status);
+	throw HTTPException(response, "HTTP GET error reading '%s' in region '%s' (HTTP %d %s)%s", url, s3_auth_params.region, response.status, status_message, extra_text);
+}
+
+HTTPException S3FileSystem::GetHTTPError(FileHandle &handle, const duckdb_httplib_openssl::Response &response, const string &url) {
+	if (response.status == 403) {
+		auto &s3_handle = handle.Cast<S3FileHandle>();
+		return GetS3Error(s3_handle.auth_params, response, url);
+	}
+	return HTTPFileSystem::GetHTTPError(handle, response, url);
+}
+
 string AWSListObjectV2::Request(string &path, HTTPParams &http_params, S3AuthParams &s3_auth_params,
                                 string &continuation_token, optional_ptr<HTTPState> state, bool use_delimiter) {
 	auto parsed_url = S3FileSystem::S3UrlParse(path, s3_auth_params);
@@ -1108,7 +1129,10 @@ string AWSListObjectV2::Request(string &path, HTTPParams &http_params, S3AuthPar
 	    listobjectv2_url.c_str(), *headers,
 	    [&](const duckdb_httplib_openssl::Response &response) {
 		    if (response.status >= 400) {
-			    throw HTTPException(response, "HTTP GET error on '%s' (HTTP %d)", listobjectv2_url, response.status);
+		    	string trimmed_path = path;
+		    	StringUtil::RTrim(trimmed_path, "/");
+		    	trimmed_path += listobjectv2_url;
+		    	throw S3FileSystem::GetS3Error(s3_auth_params, response, trimmed_path);
 		    }
 		    return true;
 	    },