diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 0000000..92278f6 --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,171 @@ +name: build, test and release sqlite-vector +on: + push: + workflow_dispatch: + +permissions: + contents: write + +jobs: + build: + runs-on: ${{ matrix.os }} + name: ${{ matrix.name }}${{ matrix.arch && format('-{0}', matrix.arch) || '' }} build${{ matrix.arch != 'arm64-v8a' && matrix.name != 'isim' && matrix.name != 'ios' && ' + test' || ''}} + timeout-minutes: 20 + strategy: + fail-fast: false + matrix: + include: + - os: ubuntu-latest + arch: x86_64 + name: linux + - os: LinuxARM64 + arch: arm64 + name: linux + - os: macos-latest + name: macos + - os: windows-latest + arch: x86_64 + name: windows + - os: ubuntu-latest + arch: arm64-v8a + name: android + make: PLATFORM=android ARCH=arm64-v8a + - os: ubuntu-latest + arch: x86_64 + name: android + make: PLATFORM=android ARCH=x86_64 + sqlite-amalgamation-zip: https://sqlite.org/2025/sqlite-amalgamation-3490100.zip + - os: macos-latest + name: ios + make: PLATFORM=ios + - os: macos-latest + name: isim + make: PLATFORM=isim + + defaults: + run: + shell: bash + + steps: + + - uses: actions/checkout@v4.2.2 + + - name: build sqlite-vector + run: make extension ${{ matrix.make && matrix.make || ''}} + + - name: windows install sqlite3 + if: matrix.os == 'windows-latest' + run: choco install sqlite -y + + - name: macos install sqlite3 without SQLITE_OMIT_LOAD_EXTENSION + if: matrix.name == 'macos' + run: brew link sqlite --force + + - name: android setup test environment + if: matrix.name == 'android' && matrix.arch != 'arm64-v8a' + run: | + + echo "::group::enable kvm group perms" + echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules + sudo udevadm control --reload-rules + sudo udevadm trigger --name-match=kvm + echo "::endgroup::" + + echo "::group::download and build sqlite3 without SQLITE_OMIT_LOAD_EXTENSION" + curl -O ${{ matrix.sqlite-amalgamation-zip }} + unzip sqlite-amalgamation-*.zip + export ${{ matrix.make }} + $ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/${{ matrix.arch }}-linux-android26-clang sqlite-amalgamation-*/shell.c sqlite-amalgamation-*/sqlite3.c -o sqlite3 -ldl + # remove unused folders to save up space + rm -rf sqlite-amalgamation-*.zip sqlite-amalgamation-* openssl + echo "::endgroup::" + + echo "::group::prepare the test script" + make test PLATFORM=$PLATFORM ARCH=$ARCH || echo "It should fail. Running remaining commands in the emulator" + cat > commands.sh << EOF + mv -f /data/local/tmp/sqlite3 /system/xbin + cd /data/local/tmp + $(make test PLATFORM=$PLATFORM ARCH=$ARCH -n) + EOF + echo "::endgroup::" + + - name: android test sqlite-vector + if: matrix.name == 'android' && matrix.arch != 'arm64-v8a' + uses: reactivecircus/android-emulator-runner@v2.34.0 + with: + api-level: 26 + arch: ${{ matrix.arch }} + script: | + adb root + adb remount + adb push ${{ github.workspace }}/. /data/local/tmp/ + adb shell "sh /data/local/tmp/commands.sh" + + - name: test sqlite-vector + if: matrix.name == 'linux' || matrix.name == 'windows' || matrix.name == 'macos' + run: make test + + - uses: actions/upload-artifact@v4.6.2 + if: always() + with: + name: vector-${{ matrix.name }}${{ matrix.arch && format('-{0}', matrix.arch) || '' }} + path: dist/vector.* + if-no-files-found: error + + release: + runs-on: ubuntu-latest + name: release + needs: build + if: github.ref == 'refs/heads/main' + + env: + GH_TOKEN: ${{ github.token }} + + steps: + + - uses: actions/checkout@v4.2.2 + + - uses: actions/download-artifact@v4.2.1 + with: + path: artifacts + + - name: release tag version from sqlite-vector.h + id: tag + run: | + FILE="src/sqlite-vector.h" + VERSION=$(grep -oP '#define SQLITE_VECTOR_VERSION\s+"\K[^"]+' "$FILE") + if [[ "$VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + LATEST=$(curl -s -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" https://api.github.com/repos/${{ github.repository }}/releases/latest | jq -r '.name') + if [[ "$VERSION" != "$LATEST" || "$GITHUB_EVENT_NAME" == "workflow_dispatch" ]]; then + echo "version=$VERSION" >> $GITHUB_OUTPUT + else + echo "::warning file=src/sqlite-vector.h::To release a new version, please update the SQLITE_VECTOR_VERSION in src/sqlite-vector.h to be different than the latest $LATEST" + fi + exit 0 + fi + echo "❌ SQLITE_VECTOR_VERSION not found in sqlite-vector.h" + exit 1 + + - name: zip artifacts + run: | + for folder in "artifacts"/*; do + if [ -d "$folder" ]; then + name=$(basename "$folder") + if [[ "$name" != "github-pages" ]]; then + zip -jq "${name}-${{ steps.tag.outputs.version }}.zip" "$folder"/* + tar -cJf "${name}-${{ steps.tag.outputs.version }}.tar.xz" -C "$folder" . + tar -czf "${name}-${{ steps.tag.outputs.version }}.tar.gz" -C "$folder" . + fi + fi + done + + - uses: softprops/action-gh-release@v2.2.1 + if: steps.tag.outputs.version != '' + with: + generate_release_notes: true + tag_name: ${{ steps.tag.outputs.version }} + files: | + vector-*-${{ steps.tag.outputs.version }}.zip + vector-*-${{ steps.tag.outputs.version }}.tar.xz + vector-*-${{ steps.tag.outputs.version }}.tar.gz + make_latest: true \ No newline at end of file diff --git a/.gitignore b/.gitignore index e43b0f9..06667c1 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,10 @@ .DS_Store +*.xcworkspacedata +*.xcuserstate +*.xcbkptlist +*.plist +/build +/dist +*.sqlite +*.a +.vscode \ No newline at end of file diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..5de5ec9 --- /dev/null +++ b/Makefile @@ -0,0 +1,142 @@ +# Makefile for SQLite Vector Extension +# Supports compilation for Linux, macOS, Windows, Android and iOS + +# customize sqlite3 executable with +# make test SQLITE3=/opt/homebrew/Cellar/sqlite/3.49.1/bin/sqlite3 +SQLITE3 ?= sqlite3 + +# Set default platform if not specified +ifeq ($(OS),Windows_NT) + PLATFORM := windows + HOST := windows + CPUS := $(shell powershell -Command "[Environment]::ProcessorCount") +else + HOST = $(shell uname -s | tr '[:upper:]' '[:lower:]') + ifeq ($(HOST),darwin) + PLATFORM := macos + CPUS := $(shell sysctl -n hw.ncpu) + else + PLATFORM := $(HOST) + CPUS := $(shell nproc) + endif +endif + +# Speed up builds by using all available CPU cores +MAKEFLAGS += -j$(CPUS) + +# Compiler and flags +CC = gcc +CFLAGS = -Wall -Wextra -Wno-unused-parameter -I$(SRC_DIR) -I$(LIB_DIR) + +# Directories +SRC_DIR = src +DIST_DIR = dist +LIB_DIR = libs +VPATH = $(SRC_DIR):$(LIB_DIR) +BUILD_DIR = build + +# Files +SRC_FILES = $(wildcard $(SRC_DIR)/*.c) +OBJ_FILES = $(patsubst %.c, $(BUILD_DIR)/%.o, $(notdir $(SRC_FILES))) + +# Platform-specific settings +ifeq ($(PLATFORM),windows) + TARGET := $(DIST_DIR)/vector.dll + LDFLAGS += -shared + # Create .def file for Windows + DEF_FILE := $(BUILD_DIR)/vector.def +else ifeq ($(PLATFORM),macos) + TARGET := $(DIST_DIR)/vector.dylib + LDFLAGS += -arch x86_64 -arch arm64 -dynamiclib -undefined dynamic_lookup + CFLAGS += -arch x86_64 -arch arm64 +else ifeq ($(PLATFORM),android) + # Set ARCH to find Android NDK's Clang compiler, the user should set the ARCH + ifeq ($(filter %,$(ARCH)),) + $(error "Android ARCH must be set to ARCH=x86_64 or ARCH=arm64-v8a") + endif + # Set ANDROID_NDK path to find android build tools + # e.g. on MacOS: export ANDROID_NDK=/Users/username/Library/Android/sdk/ndk/25.2.9519653 + ifeq ($(filter %,$(ANDROID_NDK)),) + $(error "Android NDK must be set") + endif + + BIN = $(ANDROID_NDK)/toolchains/llvm/prebuilt/$(HOST)-x86_64/bin + PATH := $(BIN):$(PATH) + + ifneq (,$(filter $(ARCH),arm64 arm64-v8a)) + override ARCH := aarch64 + endif + + CC = $(BIN)/$(ARCH)-linux-android26-clang + TARGET := $(DIST_DIR)/vector.so + LDFLAGS += -shared +else ifeq ($(PLATFORM),ios) + TARGET := $(DIST_DIR)/vector.dylib + SDK := -isysroot $(shell xcrun --sdk iphoneos --show-sdk-path) -miphoneos-version-min=11.0 + LDFLAGS += -dynamiclib $(SDK) + CFLAGS += -arch arm64 $(SDK) +else ifeq ($(PLATFORM),isim) + TARGET := $(DIST_DIR)/vector.dylib + SDK := -isysroot $(shell xcrun --sdk iphonesimulator --show-sdk-path) -miphonesimulator-version-min=11.0 + LDFLAGS += -arch x86_64 -arch arm64 -dynamiclib $(SDK) + CFLAGS += -arch x86_64 -arch arm64 $(SDK) +else # linux + TARGET := $(DIST_DIR)/vector.so + LDFLAGS += -shared +endif + +# Windows .def file generation +$(DEF_FILE): +ifeq ($(PLATFORM),windows) + @echo "LIBRARY vector.dll" > $@ + @echo "EXPORTS" >> $@ + @echo " sqlite3_vector_init" >> $@ +endif + +# Make sure the build and dist directories exist +$(shell mkdir -p $(BUILD_DIR) $(DIST_DIR)) + +# Default target +extension: $(TARGET) +all: $(TARGET) + +# Loadable library +$(TARGET): $(OBJ_FILES) $(DEF_FILE) + $(CC) $(OBJ_FILES) $(DEF_FILE) -o $@ $(LDFLAGS) +ifeq ($(PLATFORM),windows) + # Generate import library for Windows + dlltool -D $@ -d $(DEF_FILE) -l $(DIST_DIR)/vector.lib +endif + +# Object files +$(BUILD_DIR)/%.o: %.c + $(CC) $(CFLAGS) -O3 -fPIC -c $< -o $@ + +test: $(TARGET) + $(SQLITE3) ":memory:" -cmd ".bail on" ".load ./$<" "SELECT vector_version();" + +# Clean up generated files +clean: + rm -rf $(BUILD_DIR)/* $(DIST_DIR)/* *.gcda *.gcno *.gcov *.sqlite + +# Help message +help: + @echo "SQLite Vector Extension Makefile" + @echo "Usage:" + @echo " make [PLATFORM=platform] [ARCH=arch] [ANDROID_NDK=\$$ANDROID_HOME/ndk/26.1.10909125] [target]" + @echo "" + @echo "Platforms:" + @echo " linux (default on Linux)" + @echo " macos (default on macOS)" + @echo " windows (default on Windows)" + @echo " android (needs ARCH to be set to x86_64 or arm64-v8a and ANDROID_NDK to be set)" + @echo " ios (only on macOS)" + @echo " isim (only on macOS)" + @echo "" + @echo "Targets:" + @echo " all - Build the extension (default)" + @echo " clean - Remove built files" + @echo " test - Test the extension" + @echo " help - Display this help message" + +.PHONY: all clean test extension help diff --git a/src/distance-neon.c b/src/distance-neon.c index 1510a3e..c1af56d 100644 --- a/src/distance-neon.c +++ b/src/distance-neon.c @@ -155,8 +155,8 @@ static inline float uint8_distance_l2_impl_neon(const void *v1, const void *v2, uint8x16_t vb = vld1q_u8(b + i); // compute 8-bit differences widened to signed 16-bit - int16x8_t diff_lo = vsubl_u8(vget_low_u8(va), vget_low_u8(vb)); - int16x8_t diff_hi = vsubl_u8(vget_high_u8(va), vget_high_u8(vb)); + int16x8_t diff_lo = (int16x8_t)vsubl_u8(vget_low_u8(va), vget_low_u8(vb)); + int16x8_t diff_hi = (int16x8_t)vsubl_u8(vget_high_u8(va), vget_high_u8(vb)); // widen to signed 32-bit and square int32x4_t diff_lo_0 = vmovl_s16(vget_low_s16(diff_lo)); diff --git a/src/sqlite-vector.c b/src/sqlite-vector.c index accb06a..ea1bf7a 100644 --- a/src/sqlite-vector.c +++ b/src/sqlite-vector.c @@ -18,6 +18,32 @@ #include #include #include +#include + +#ifdef _WIN32 +char *strcasestr(const char *haystack, const char *needle) { + if (!haystack || !needle) return NULL; + if (!*needle) return (char *)haystack; + for (; *haystack; ++haystack) { + const char *h = haystack; + const char *n = needle; + while (*h && *n && tolower((unsigned char)*h) == tolower((unsigned char)*n)) { + ++h; + ++n; + } + if (!*n) return (char *)haystack; + } + return NULL; +} +#endif + +#if defined(_WIN32) || defined(__linux__) +#include +#endif + +#ifndef SQLITE_CORE +SQLITE_EXTENSION_INIT1 +#endif #define DEBUG_VECTOR_ALWAYS(...) do {printf(__VA_ARGS__ );printf("\n");} while (0) @@ -852,8 +878,13 @@ static int vector_rebuild_quantization (sqlite3_context *context, const char *ta // STEP 1 // find global min/max across ALL vectors + #if defined(_WIN32) || defined(__linux__) + float min_val = FLT_MAX; + float max_val = -FLT_MAX; + #else float min_val = MAXFLOAT; float max_val = -MAXFLOAT; + #endif while (1) { rc = sqlite3_step(vm); @@ -1748,6 +1779,9 @@ static void vector_backend (sqlite3_context *context, int argc, sqlite3_value ** // MARK: - SQLITE_VECTOR_API int sqlite3_vector_init (sqlite3 *db, char **pzErrMsg, const sqlite3_api_routines *pApi) { + #ifndef SQLITE_CORE + SQLITE_EXTENSION_INIT2(pApi); + #endif int rc = SQLITE_OK; init_distance_functions(false);