feat: vector buckets

Author: fenos

.env.test.sample

@@ -22,4 +22,7 @@ AWS_DEFAULT_REGION=ap-southeast-1
 STORAGE_S3_ENDPOINT=http://127.0.0.1:9000
 STORAGE_S3_PROTOCOL=http
 STORAGE_S3_FORCE_PATH_STYLE=true
-REQUEST_X_FORWARDED_HOST_REGEXP=
+REQUEST_X_FORWARDED_HOST_REGEXP=
+
+VECTOR_ENABLED=true
+ICEBERG_ENABLED=true

.github/workflows/ci.yml

@@ -82,6 +82,8 @@ jobs:
           MULTI_TENANT: false
           S3_PROTOCOL_ACCESS_KEY_ID: ${{ secrets.TENANT_ID }}
           S3_PROTOCOL_ACCESS_KEY_SECRET: ${{ secrets.SERVICE_KEY }}
+          VECTOR_ENABLED: true
+          ICEBERG_ENABLED: true
 
       - name: Upload coverage results to Coveralls
         uses: coverallsapp/github-action@master

migrations/multitenant/0020-vector-buckets-feature.sql

@@ -0,0 +1,3 @@
+ALTER TABLE tenants ADD COLUMN IF NOT EXISTS feature_vector_buckets boolean NOT NULL DEFAULT false;
+ALTER TABLE tenants ADD COLUMN IF NOT EXISTS feature_vector_buckets_max_buckets int NOT NULL DEFAULT 10;
+ALTER TABLE tenants ADD COLUMN IF NOT EXISTS feature_vector_buckets_max_indexes int NOT NULL DEFAULT 5;

migrations/tenant/0044-vector-bucket-type.sql

@@ -0,0 +1,13 @@
+DO $$
+    DECLARE
+    BEGIN
+        IF NOT EXISTS (
+            SELECT 1
+            FROM pg_enum
+                     JOIN pg_type ON pg_enum.enumtypid = pg_type.oid
+            WHERE pg_type.typname = 'buckettype'
+              AND enumlabel = 'VECTOR'
+        ) THEN
+            ALTER TYPE storage.BucketType ADD VALUE 'VECTOR';
+        END IF;
+END$$;

migrations/tenant/0045-vector-buckets.sql

@@ -0,0 +1,35 @@
+DO $$
+    DECLARE
+        anon_role text = COALESCE(current_setting('storage.anon_role', true), 'anon');
+        authenticated_role text = COALESCE(current_setting('storage.authenticated_role', true), 'authenticated');
+        service_role text = COALESCE(current_setting('storage.service_role', true), 'service_role');
+    BEGIN
+        CREATE TABLE IF NOT EXISTS storage.buckets_vectors (
+            id text not null primary key,
+            type storage.BucketType NOT NULL default 'VECTOR',
+            created_at timestamptz NOT NULL default now(),
+            updated_at timestamptz NOT NULL default now()
+        );
+
+        CREATE TABLE IF NOT EXISTS storage.vector_indexes
+        (
+            id                     text             primary key default gen_random_uuid(),
+            name                   text COLLATE "C" NOT NULL,
+            bucket_id              text             NOT NULL references storage.buckets_vectors (id),
+            data_type              text             NOT NULL,
+            dimension              integer          NOT NULL,
+            distance_metric        text             NOT NULL,
+            metadata_configuration jsonb            NULL,
+            status                 text             NOT NULL default 'PENDING',
+            created_at             timestamptz      NOT NULL default now(),
+            updated_at             timestamptz      NOT NULL default now()
+        );
+
+        ALTER TABLE storage.buckets_vectors ENABLE ROW LEVEL SECURITY;
+        ALTER TABLE storage.vector_indexes ENABLE ROW LEVEL SECURITY;
+
+        EXECUTE 'GRANT SELECT ON TABLE storage.buckets_vectors TO ' || service_role || ', ' || authenticated_role || ', ' || anon_role;
+        EXECUTE 'GRANT SELECT ON TABLE storage.vector_indexes TO ' || service_role || ', ' || authenticated_role || ', ' || anon_role;
+
+        CREATE UNIQUE INDEX IF NOT EXISTS vector_indexes_name_bucket_id_idx ON storage.vector_indexes (name, bucket_id);
+END$$;