chore: support delta lake on linux x86-64 platform

Author: wudidapaopao

.github/workflows/build_linux_arm64_wheels-gh.yml

@@ -158,4 +158,3 @@ jobs:
         env:
           TWINE_USERNAME: __token__
           TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
-     

.github/workflows/build_linux_x86_wheels.yml

@@ -28,8 +28,8 @@ jobs:
       fail-fast: false
       matrix:
         os: [ ubuntu-22.04 ]
-        python-version: [ "3.8", "3.9", "3.10", "3.11", "3.12", "3.13" ]
-        # python-version: [ "3.7" ]
+        # python-version: [ "3.8", "3.9", "3.10", "3.11", "3.12", "3.13" ]
+        python-version: [ "3.13" ]
     env:
       RUNNER_OS: ${{ matrix.os }}
       PYTHON_VERSION: ${{ matrix.python-version }}

.github/workflows/build_macos_x86_wheels.yml

@@ -163,4 +163,3 @@ jobs:
         env:
           TWINE_USERNAME: __token__
           TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
-     

chdb/build.sh

@@ -13,6 +13,7 @@ BUILD_DIR=${PROJ_DIR}/buildlib
 
 HDFS="-DENABLE_HDFS=1 -DENABLE_GSASL_LIBRARY=1 -DENABLE_KRB5=1"
 MYSQL="-DENABLE_MYSQL=1"
+RUST_FEATURES="-DENABLE_RUST=0"
 # check current os type
 if [ "$(uname)" == "Darwin" ]; then
     export CXX=$(brew --prefix llvm@19)/bin/clang++
@@ -60,6 +61,18 @@ elif [ "$(uname)" == "Linux" ]; then
     if [ "$(uname -m)" == "x86_64" ]; then
         CPU_FEATURES="-DENABLE_AVX=1 -DENABLE_AVX2=0"
         LLVM="-DENABLE_EMBEDDED_COMPILER=1 -DENABLE_DWARF_PARSER=1"
+        RUST_FEATURES="-DENABLE_RUST=1 -DENABLE_DELTA_KERNEL_RS=1"
+        CORROSION_CMAKE_FILE="${PROJ_DIR}/contrib/corrosion-cmake/CMakeLists.txt"
+        if [ -f "${CORROSION_CMAKE_FILE}" ]; then
+            if ! grep -q 'OPENSSL_NO_DEPRECATED_3_0' "${CORROSION_CMAKE_FILE}"; then
+                echo "Modifying corrosion CMakeLists.txt for Linux x86_64..."
+                ${SED_INPLACE} 's/corrosion_set_env_vars(${target_name} "RUSTFLAGS=${RUSTFLAGS}")/corrosion_set_env_vars(${target_name} "RUSTFLAGS=${RUSTFLAGS} --cfg osslconf=\\\"OPENSSL_NO_DEPRECATED_3_0\\\"")/g' "${CORROSION_CMAKE_FILE}"
+            else
+                echo "corrosion CMakeLists.txt already modified, skipping..."
+            fi
+        else
+            echo "Warning: corrosion CMakeLists.txt not found at ${CORROSION_CMAKE_FILE}"
+        fi
     else
         CPU_FEATURES="-DENABLE_AVX=0 -DENABLE_AVX2=0 -DNO_ARMV81_OR_HIGHER=1"
         LLVM="-DENABLE_EMBEDDED_COMPILER=0 -DENABLE_DWARF_PARSER=0"
@@ -84,7 +97,7 @@ CMAKE_ARGS="-DCMAKE_BUILD_TYPE=${build_type} -DENABLE_THINLTO=0 -DENABLE_TESTS=0
     -DENABLE_LDAP=0 \
     ${MYSQL} \
     ${HDFS} \
-    -DENABLE_LIBRARIES=0 -DENABLE_RUST=0 \
+    -DENABLE_LIBRARIES=0 ${RUST_FEATURES} \
     ${GLIBC_COMPATIBILITY} \
     -DENABLE_UTILS=0 ${LLVM} ${UNWIND} \
     ${ICU} -DENABLE_UTF8PROC=1 ${JEMALLOC} \

rust/CMakeLists.txt

@@ -0,0 +1,2 @@
+add_subdirectory (workspace)
+add_subdirectory (chcache)

rust/README.md

@@ -0,0 +1,3 @@
+Read more about the Rust integration in the blog:
+
+## https://clickhouse.com/blog/rust

rust/VENDOR.md

@@ -0,0 +1,65 @@
+As we have multiple projects we use a workspace to manage them (it's way simpler and leads to less issues). In order
+to vendor all the dependencies we need to store both the registry and the packages themselves.
+
+Note that this includes the exact `std` dependencies for the rustc version used in CI (currently nightly-2024-12-01),
+so you need to install `rustup component add rust-src` for the specific version.
+
+* First step: (Re)-generate the Cargo.lock file (run under `workspace/`).
+
+```bash
+cargo generate-lockfile
+```
+
+* Generate the local registry:
+
+Note that we use both commands to vendor both registry and crates. No idea why both are necessary.
+
+  * First we need to install the tool if you don't already have it:
+```bash
+cargo install --version 0.2.7 cargo-local-registry
+```
+
+  * Now add the local packages:
+
+```bash
+export CH_TOP_DIR=$(git rev-parse --show-toplevel)
+export RUSTC_ROOT=$(rustc --print=sysroot)
+# Currently delta-lake is built outside the workspace (TODO)
+export DELTA_LAKE_DIR="$CH_TOP_DIR"/contrib/delta-kernel-rs
+
+# Clean the vendor repo
+rm -rf "$CH_TOP_DIR"/contrib/rust_vendor/*
+
+cd "$CH_TOP_DIR"/rust/workspace
+cargo local-registry --git --sync Cargo.lock "$CH_TOP_DIR"/contrib/rust_vendor
+
+# Now handle delta-lake
+cd "$DELTA_LAKE_DIR"
+cargo local-registry --no-delete --git --sync "$DELTA_LAKE_DIR/Cargo.lock" "$CH_TOP_DIR"/contrib/rust_vendor
+
+# Standard library deps
+cp "$RUSTC_ROOT"/lib/rustlib/src/rust/library/Cargo.lock "$RUSTC_ROOT"/lib/rustlib/src/rust/library/std/
+cargo local-registry --no-delete --git --sync "$RUSTC_ROOT"/lib/rustlib/src/rust/library/std/Cargo.lock "$CH_TOP_DIR"/contrib/rust_vendor
+cp "$RUSTC_ROOT"/lib/rustlib/src/rust/library/Cargo.lock "$RUSTC_ROOT"/lib/rustlib/src/rust/library/test/
+cargo local-registry --no-delete --git --sync "$RUSTC_ROOT"/lib/rustlib/src/rust/library/test/Cargo.lock "$CH_TOP_DIR"/contrib/rust_vendor
+
+# Now we vendor the modules themselves
+cd "$CH_TOP_DIR"/rust/workspace
+cargo vendor --no-delete --locked "$CH_TOP_DIR"/contrib/rust_vendor
+cd "$DELTA_LAKE_DIR"
+cargo vendor --no-delete --locked "$CH_TOP_DIR"/contrib/rust_vendor
+cd "$RUSTC_ROOT"/lib/rustlib/src/rust/library/std/
+cargo vendor --no-delete "$CH_TOP_DIR"/contrib/rust_vendor
+cd "$RUSTC_ROOT"/lib/rustlib/src/rust/library/test/
+cargo vendor --no-delete "$CH_TOP_DIR"/contrib/rust_vendor
+
+# Remove windows only dependencies (which are really heavy and we don't want in the repo)
+rm -rf "$CH_TOP_DIR"/contrib/rust_vendor/winapi* "$CH_TOP_DIR"/contrib/rust_vendor/windows*
+
+# Cleanup the lock files we copied
+rm "$RUSTC_ROOT"/lib/rustlib/src/rust/library/std/Cargo.lock "$RUSTC_ROOT"/lib/rustlib/src/rust/library/test/Cargo.lock
+cd "$CH_TOP_DIR"/rust/workspace
+```
+
+The `rustc --print=sysroot` part includes `std` dependencies, required to build with sanitizer flags. It must be kept
+in sync with the rustc version used in CI.

rust/chcache/.gitignore

@@ -0,0 +1 @@
+/target

rust/chcache/CMakeLists.txt

@@ -0,0 +1,18 @@
+if (COMPILER_CACHE STREQUAL "chcache")
+    corrosion_import_crate(
+        MANIFEST_PATH Cargo.toml
+        PROFILE release
+        LOCKED
+    )
+
+    corrosion_set_env_vars(
+        chcache
+        RUSTFLAGS=
+        RUST_CFLAGS=
+        RUST_CXXFLAGS=
+        CFLAGS=
+        CXXFLAGS=
+    )
+
+    corrosion_set_hostbuild(chcache)
+endif()