diff --git a/.github/workflows/MainDistributionPipeline.yml b/.github/workflows/MainDistributionPipeline.yml index b06d2135..496d87c0 100644 --- a/.github/workflows/MainDistributionPipeline.yml +++ b/.github/workflows/MainDistributionPipeline.yml @@ -19,7 +19,6 @@ jobs: extension_name: httpfs duckdb_version: v1.3-ossivalis ci_tools_version: main - exclude_archs: 'wasm_mvp;wasm_eh;wasm_threads' duckdb-stable-deploy: @@ -32,4 +31,3 @@ jobs: duckdb_version: v1.3-ossivalis ci_tools_version: main deploy_latest: ${{ startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' }} - exclude_archs: 'wasm_mvp;wasm_eh;wasm_threads' diff --git a/CMakeLists.txt b/CMakeLists.txt index 92d45479..024cb72d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -9,16 +9,24 @@ add_extension_definitions() include_directories(extension/httpfs/include ${DUCKDB_MODULE_BASE_DIR}/third_party/httplib) +if (NOT EMSCRIPTEN) + set(EXTRA_SOURCES extension/httpfs/crypto.cpp extension/httpfs/httpfs_client.cpp) + add_definitions(-DOVERRIDE_ENCRYPTION_UTILS=1) +else() + set(EXTRA_SOURCES extension/httpfs/httpfs_client_wasm.cpp) +endif() + build_static_extension( httpfs extension/httpfs/hffs.cpp extension/httpfs/s3fs.cpp extension/httpfs/httpfs.cpp - extension/httpfs/httpfs_client.cpp extension/httpfs/http_state.cpp extension/httpfs/crypto.cpp + extension/httpfs/hash_functions.cpp extension/httpfs/create_secret_functions.cpp - extension/httpfs/httpfs_extension.cpp) + extension/httpfs/httpfs_extension.cpp + ${EXTRA_SOURCES} ) set(PARAMETERS "-warnings") build_loadable_extension( @@ -27,11 +35,12 @@ build_loadable_extension( extension/httpfs/hffs.cpp extension/httpfs/s3fs.cpp extension/httpfs/httpfs.cpp - extension/httpfs/httpfs_client.cpp extension/httpfs/http_state.cpp extension/httpfs/crypto.cpp + extension/httpfs/hash_functions.cpp extension/httpfs/create_secret_functions.cpp - extension/httpfs/httpfs_extension.cpp) + extension/httpfs/httpfs_extension.cpp + ${EXTRA_SOURCES} ) if(MINGW) set(OPENSSL_USE_STATIC_LIBS TRUE) diff --git a/extension/httpfs/crypto.cpp b/extension/httpfs/crypto.cpp index 04bd795e..3a89ca5f 100644 --- a/extension/httpfs/crypto.cpp +++ b/extension/httpfs/crypto.cpp @@ -1,4 +1,5 @@ #include "crypto.hpp" +#include "hash_functions.hpp" #include "mbedtls_wrapper.hpp" #include #include "duckdb/common/common.hpp" @@ -9,28 +10,6 @@ namespace duckdb { -void sha256(const char *in, size_t in_len, hash_bytes &out) { - duckdb_mbedtls::MbedTlsWrapper::ComputeSha256Hash(in, in_len, (char *)out); -} - -void hmac256(const std::string &message, const char *secret, size_t secret_len, hash_bytes &out) { - duckdb_mbedtls::MbedTlsWrapper::Hmac256(secret, secret_len, message.data(), message.size(), (char *)out); -} - -void hmac256(std::string message, hash_bytes secret, hash_bytes &out) { - hmac256(message, (char *)secret, sizeof(hash_bytes), out); -} - -void hex256(hash_bytes &in, hash_str &out) { - const char *hex = "0123456789abcdef"; - unsigned char *pin = in; - unsigned char *pout = out; - for (; pin < in + sizeof(in); pout += 2, pin++) { - pout[0] = hex[(*pin >> 4) & 0xF]; - pout[1] = hex[*pin & 0xF]; - } -} - AESStateSSL::AESStateSSL(const std::string *key) : context(EVP_CIPHER_CTX_new()) { if (!(context)) { throw InternalException("AES GCM failed with initializing context"); diff --git a/extension/httpfs/hash_functions.cpp b/extension/httpfs/hash_functions.cpp new file mode 100644 index 00000000..1e6bb8f1 --- /dev/null +++ b/extension/httpfs/hash_functions.cpp @@ -0,0 +1,28 @@ +#include "mbedtls_wrapper.hpp" +#include "hash_functions.hpp" + +namespace duckdb { + +void sha256(const char *in, size_t in_len, hash_bytes &out) { + duckdb_mbedtls::MbedTlsWrapper::ComputeSha256Hash(in, in_len, (char *)out); +} + +void hmac256(const std::string &message, const char *secret, size_t secret_len, hash_bytes &out) { + duckdb_mbedtls::MbedTlsWrapper::Hmac256(secret, secret_len, message.data(), message.size(), (char *)out); +} + +void hmac256(std::string message, hash_bytes secret, hash_bytes &out) { + hmac256(message, (char *)secret, sizeof(hash_bytes), out); +} + +void hex256(hash_bytes &in, hash_str &out) { + const char *hex = "0123456789abcdef"; + unsigned char *pin = in; + unsigned char *pout = out; + for (; pin < in + sizeof(in); pout += 2, pin++) { + pout[0] = hex[(*pin >> 4) & 0xF]; + pout[1] = hex[*pin & 0xF]; + } +} + +} // namespace duckdb diff --git a/extension/httpfs/httpfs.cpp b/extension/httpfs/httpfs.cpp index 56c8b8e3..20f85a69 100644 --- a/extension/httpfs/httpfs.cpp +++ b/extension/httpfs/httpfs.cpp @@ -727,4 +727,9 @@ void HTTPFileHandle::StoreClient(unique_ptr client) { HTTPFileHandle::~HTTPFileHandle() { DUCKDB_LOG_FILE_SYSTEM_CLOSE((*this)); }; + +string HTTPFSUtil::GetName() const { + return "HTTPFS"; +} + } // namespace duckdb diff --git a/extension/httpfs/httpfs_client.cpp b/extension/httpfs/httpfs_client.cpp index 84eb457b..3bf5a64f 100644 --- a/extension/httpfs/httpfs_client.cpp +++ b/extension/httpfs/httpfs_client.cpp @@ -160,8 +160,4 @@ unordered_map HTTPFSUtil::ParseGetParameters(const string &text) return result; } -string HTTPFSUtil::GetName() const { - return "HTTPFS"; -} - } // namespace duckdb diff --git a/extension/httpfs/httpfs_client_wasm.cpp b/extension/httpfs/httpfs_client_wasm.cpp new file mode 100644 index 00000000..aaa22bbe --- /dev/null +++ b/extension/httpfs/httpfs_client_wasm.cpp @@ -0,0 +1,16 @@ +#include "httpfs_client.hpp" +#include "http_state.hpp" + +namespace duckdb { + +unique_ptr HTTPFSUtil::InitializeClient(HTTPParams &http_params, const string &proto_host_port) { + throw InternalException("HTTPFSUtil::InitializeClient is not expected to be called"); +} + +unordered_map HTTPFSUtil::ParseGetParameters(const string &text) { + unordered_map result; + //TODO: HTTPFSUtil::ParseGetParameters is currently not implemented + return result; +} + +} // namespace duckdb diff --git a/extension/httpfs/httpfs_extension.cpp b/extension/httpfs/httpfs_extension.cpp index c9bc9853..945cdd48 100644 --- a/extension/httpfs/httpfs_extension.cpp +++ b/extension/httpfs/httpfs_extension.cpp @@ -6,7 +6,9 @@ #include "duckdb.hpp" #include "s3fs.hpp" #include "hffs.hpp" +#ifdef OVERRIDE_ENCRYPTION_UTILS #include "crypto.hpp" +#endif // OVERRIDE_ENCRYPTION_UTILS namespace duckdb { @@ -61,7 +63,12 @@ static void LoadInternal(DatabaseInstance &instance) { // HuggingFace options config.AddExtensionOption("hf_max_per_page", "Debug option to limit number of items returned in list requests", LogicalType::UBIGINT, Value::UBIGINT(0)); - config.http_util = make_shared_ptr(); + + if (config.http_util && config.http_util->GetName() == "WasmHTTPUtils") { + // Already handled, do not override + } else { + config.http_util = make_shared_ptr(); + } auto provider = make_uniq(config); provider->SetAll(); @@ -69,8 +76,10 @@ static void LoadInternal(DatabaseInstance &instance) { CreateS3SecretFunctions::Register(instance); CreateBearerTokenFunctions::Register(instance); +#ifdef OVERRIDE_ENCRYPTION_UTILS // set pointer to OpenSSL encryption state config.encryption_util = make_shared_ptr(); +#endif // OVERRIDE_ENCRYPTION_UTILS } void HttpfsExtension::Load(DuckDB &db) { LoadInternal(*db.instance); diff --git a/extension/httpfs/include/hash_functions.hpp b/extension/httpfs/include/hash_functions.hpp new file mode 100644 index 00000000..bfefe79e --- /dev/null +++ b/extension/httpfs/include/hash_functions.hpp @@ -0,0 +1,18 @@ +#pragma once + +#include "duckdb/common/helper.hpp" + +namespace duckdb { + +typedef unsigned char hash_bytes[32]; +typedef unsigned char hash_str[64]; + +void sha256(const char *in, size_t in_len, hash_bytes &out); + +void hmac256(const std::string &message, const char *secret, size_t secret_len, hash_bytes &out); + +void hmac256(std::string message, hash_bytes secret, hash_bytes &out); + +void hex256(hash_bytes &in, hash_str &out); + +} // namespace duckdb diff --git a/extension_config.cmake b/extension_config.cmake index 58810439..2664bc2a 100644 --- a/extension_config.cmake +++ b/extension_config.cmake @@ -15,4 +15,5 @@ duckdb_extension_load(httpfs SOURCE_DIR ${CMAKE_CURRENT_LIST_DIR} INCLUDE_DIR ${CMAKE_CURRENT_LIST_DIR}/extension/httpfs/include ${LOAD_HTTPFS_TESTS} + LINKED_LIBS "../../third_party/mbedtls/libduckdb_mbedtls.a" )