From f2ae953222fc390ceb417fc63c893ea8485b9a76 Mon Sep 17 00:00:00 2001 From: fenos Date: Mon, 15 Sep 2025 09:30:12 +0200 Subject: [PATCH 1/2] feat: vector buckets --- .env.test.sample | 6 +- .github/workflows/ci.yml | 3 + .../0020-vector-buckets-feature.sql | 3 + migrations/tenant/0044-vector-bucket-type.sql | 13 + migrations/tenant/0045-vector-buckets.sql | 35 + package-lock.json | 1925 +++++++++++++++- package.json | 4 +- src/app.ts | 3 +- src/config.ts | 14 + src/http/plugins/index.ts | 1 + src/http/plugins/jwt.ts | 5 + src/http/plugins/signature-v4.ts | 244 ++- src/http/plugins/vector.ts | 49 + src/http/routes/admin/tenants.ts | 47 +- src/http/routes/iceberg/bucket.ts | 122 ++ src/http/routes/iceberg/index.ts | 22 +- src/http/routes/index.ts | 1 + src/http/routes/operations.ts | 17 + src/http/routes/vector/create-bucket.ts | 45 + src/http/routes/vector/create-index.ts | 66 + src/http/routes/vector/delete-bucket.ts | 45 + src/http/routes/vector/delete-index.ts | 53 + src/http/routes/vector/delete-vectors.ts | 50 + src/http/routes/vector/get-bucket.ts | 45 + src/http/routes/vector/get-index.ts | 64 + src/http/routes/vector/get-vectors.ts | 49 + src/http/routes/vector/index.ts | 79 + src/http/routes/vector/list-buckets.ts | 46 + src/http/routes/vector/list-indexes.ts | 51 + src/http/routes/vector/list-vectors.ts | 60 + src/http/routes/vector/put-vectors.ts | 84 + src/http/routes/vector/query-vectors.ts | 159 ++ src/internal/auth/jwt.ts | 7 + src/internal/database/migrations/types.ts | 5 +- src/internal/database/tenant.ts | 26 +- src/internal/errors/codes.ts | 50 + src/internal/streams/byte-counter.ts | 10 + src/internal/streams/hash-stream.ts | 261 +++ src/internal/streams/index.ts | 1 + src/internal/streams/types.d.ts | 3 + src/storage/database/adapter.ts | 7 +- src/storage/database/knex.ts | 33 +- src/storage/events/index.ts | 1 + src/storage/events/vectors/reconcile.ts | 46 + .../iceberg/catalog/tenant-catalog.ts | 1 + src/storage/protocols/s3/signature-v4.ts | 46 +- .../protocols/vector/adapter/s3-vector.ts | 86 + src/storage/protocols/vector/index.ts | 3 + src/storage/protocols/vector/knex.ts | 250 +++ src/storage/protocols/vector/vector-store.ts | 339 +++ src/storage/schemas/bucket.ts | 1 + src/storage/schemas/vector.ts | 26 + src/storage/storage.ts | 4 + src/test/hash-stream.test.ts | 598 +++++ src/test/iceberg.test.ts | 70 +- src/test/tenant.test.ts | 14 +- src/test/vectors.test.ts | 1940 +++++++++++++++++ src/test/x-forwarded-host.test.ts | 5 + 58 files changed, 7079 insertions(+), 164 deletions(-) create mode 100644 migrations/multitenant/0020-vector-buckets-feature.sql create mode 100644 migrations/tenant/0044-vector-bucket-type.sql create mode 100644 migrations/tenant/0045-vector-buckets.sql create mode 100644 src/http/plugins/vector.ts create mode 100644 src/http/routes/iceberg/bucket.ts create mode 100644 src/http/routes/vector/create-bucket.ts create mode 100644 src/http/routes/vector/create-index.ts create mode 100644 src/http/routes/vector/delete-bucket.ts create mode 100644 src/http/routes/vector/delete-index.ts create mode 100644 src/http/routes/vector/delete-vectors.ts create mode 100644 src/http/routes/vector/get-bucket.ts create mode 100644 src/http/routes/vector/get-index.ts create mode 100644 src/http/routes/vector/get-vectors.ts create mode 100644 src/http/routes/vector/index.ts create mode 100644 src/http/routes/vector/list-buckets.ts create mode 100644 src/http/routes/vector/list-indexes.ts create mode 100644 src/http/routes/vector/list-vectors.ts create mode 100644 src/http/routes/vector/put-vectors.ts create mode 100644 src/http/routes/vector/query-vectors.ts create mode 100644 src/internal/streams/hash-stream.ts create mode 100644 src/internal/streams/types.d.ts create mode 100644 src/storage/events/vectors/reconcile.ts create mode 100644 src/storage/protocols/vector/adapter/s3-vector.ts create mode 100644 src/storage/protocols/vector/index.ts create mode 100644 src/storage/protocols/vector/knex.ts create mode 100644 src/storage/protocols/vector/vector-store.ts create mode 100644 src/storage/schemas/vector.ts create mode 100644 src/test/hash-stream.test.ts create mode 100644 src/test/vectors.test.ts diff --git a/.env.test.sample b/.env.test.sample index fee2ccbb0..790243d07 100644 --- a/.env.test.sample +++ b/.env.test.sample @@ -22,4 +22,8 @@ AWS_DEFAULT_REGION=ap-southeast-1 STORAGE_S3_ENDPOINT=http://127.0.0.1:9000 STORAGE_S3_PROTOCOL=http STORAGE_S3_FORCE_PATH_STYLE=true -REQUEST_X_FORWARDED_HOST_REGEXP= \ No newline at end of file +REQUEST_X_FORWARDED_HOST_REGEXP= + +VECTOR_ENABLED=true +ICEBERG_ENABLED=true +ICEBERG_BUCKET_DETECTION_MODE="BUCKET" \ No newline at end of file diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3a82c537b..98f00789c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -82,6 +82,9 @@ jobs: MULTI_TENANT: false S3_PROTOCOL_ACCESS_KEY_ID: ${{ secrets.TENANT_ID }} S3_PROTOCOL_ACCESS_KEY_SECRET: ${{ secrets.SERVICE_KEY }} + VECTOR_BUCKET_S3: supa-test-local-dev + VECTOR_ENABLED: true + ICEBERG_ENABLED: true - name: Upload coverage results to Coveralls uses: coverallsapp/github-action@master diff --git a/migrations/multitenant/0020-vector-buckets-feature.sql b/migrations/multitenant/0020-vector-buckets-feature.sql new file mode 100644 index 000000000..021f7d2e7 --- /dev/null +++ b/migrations/multitenant/0020-vector-buckets-feature.sql @@ -0,0 +1,3 @@ +ALTER TABLE tenants ADD COLUMN IF NOT EXISTS feature_vector_buckets boolean NOT NULL DEFAULT false; +ALTER TABLE tenants ADD COLUMN IF NOT EXISTS feature_vector_buckets_max_buckets int NOT NULL DEFAULT 10; +ALTER TABLE tenants ADD COLUMN IF NOT EXISTS feature_vector_buckets_max_indexes int NOT NULL DEFAULT 5; diff --git a/migrations/tenant/0044-vector-bucket-type.sql b/migrations/tenant/0044-vector-bucket-type.sql new file mode 100644 index 000000000..6ba4ec587 --- /dev/null +++ b/migrations/tenant/0044-vector-bucket-type.sql @@ -0,0 +1,13 @@ +DO $$ + DECLARE + BEGIN + IF NOT EXISTS ( + SELECT 1 + FROM pg_enum + JOIN pg_type ON pg_enum.enumtypid = pg_type.oid + WHERE pg_type.typname = 'buckettype' + AND enumlabel = 'VECTOR' + ) THEN + ALTER TYPE storage.BucketType ADD VALUE 'VECTOR'; + END IF; +END$$; \ No newline at end of file diff --git a/migrations/tenant/0045-vector-buckets.sql b/migrations/tenant/0045-vector-buckets.sql new file mode 100644 index 000000000..ff5108ca3 --- /dev/null +++ b/migrations/tenant/0045-vector-buckets.sql @@ -0,0 +1,35 @@ +DO $$ + DECLARE + anon_role text = COALESCE(current_setting('storage.anon_role', true), 'anon'); + authenticated_role text = COALESCE(current_setting('storage.authenticated_role', true), 'authenticated'); + service_role text = COALESCE(current_setting('storage.service_role', true), 'service_role'); + BEGIN + CREATE TABLE IF NOT EXISTS storage.buckets_vectors ( + id text not null primary key, + type storage.BucketType NOT NULL default 'VECTOR', + created_at timestamptz NOT NULL default now(), + updated_at timestamptz NOT NULL default now() + ); + + CREATE TABLE IF NOT EXISTS storage.vector_indexes + ( + id text primary key default gen_random_uuid(), + name text COLLATE "C" NOT NULL, + bucket_id text NOT NULL references storage.buckets_vectors (id), + data_type text NOT NULL, + dimension integer NOT NULL, + distance_metric text NOT NULL, + metadata_configuration jsonb NULL, + status text NOT NULL default 'PENDING', + created_at timestamptz NOT NULL default now(), + updated_at timestamptz NOT NULL default now() + ); + + ALTER TABLE storage.buckets_vectors ENABLE ROW LEVEL SECURITY; + ALTER TABLE storage.vector_indexes ENABLE ROW LEVEL SECURITY; + + EXECUTE 'GRANT SELECT ON TABLE storage.buckets_vectors TO ' || service_role || ', ' || authenticated_role || ', ' || anon_role; + EXECUTE 'GRANT SELECT ON TABLE storage.vector_indexes TO ' || service_role || ', ' || authenticated_role || ', ' || anon_role; + + CREATE UNIQUE INDEX IF NOT EXISTS vector_indexes_name_bucket_id_idx ON storage.vector_indexes (name, bucket_id); +END$$; \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index 086a0eb8d..67997c10d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11,6 +11,7 @@ "dependencies": { "@aws-sdk/client-ecs": "^3.795.0", "@aws-sdk/client-s3": "3.654.0", + "@aws-sdk/client-s3vectors": "^3.883.0", "@aws-sdk/lib-storage": "3.654.0", "@aws-sdk/s3-request-presigner": "3.654.0", "@fastify/accepts": "^4.3.0", @@ -75,6 +76,7 @@ "@babel/preset-typescript": "^7.27.0", "@types/async-retry": "^1.4.5", "@types/busboy": "^1.3.0", + "@types/cloneable-readable": "^2.0.3", "@types/crypto-js": "^4.1.1", "@types/fs-extra": "^9.0.13", "@types/glob": "^8.1.0", @@ -82,7 +84,7 @@ "@types/js-yaml": "^4.0.5", "@types/multistream": "^4.1.3", "@types/mustache": "^4.2.2", - "@types/node": "^20.11.5", + "@types/node": "^22.18.8", "@types/pg": "^8.6.4", "@types/stream-buffers": "^3.0.7", "@types/xml2js": "^0.4.14", @@ -1647,6 +1649,1029 @@ "node": ">=16.0.0" } }, + "node_modules/@aws-sdk/client-s3vectors": { + "version": "3.883.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-s3vectors/-/client-s3vectors-3.883.0.tgz", + "integrity": "sha512-QtGyHqvih3it25HFnxudHg6J6FvlpSQs9aSN2IgpofAQGLENGVtKFtVlFay2KOlUFPh5aL1OOl5ONR70O8q/Eg==", + "dependencies": { + "@aws-crypto/sha256-browser": "5.2.0", + "@aws-crypto/sha256-js": "5.2.0", + "@aws-sdk/core": "3.883.0", + "@aws-sdk/credential-provider-node": "3.883.0", + "@aws-sdk/middleware-host-header": "3.873.0", + "@aws-sdk/middleware-logger": "3.876.0", + "@aws-sdk/middleware-recursion-detection": "3.873.0", + "@aws-sdk/middleware-user-agent": "3.883.0", + "@aws-sdk/region-config-resolver": "3.873.0", + "@aws-sdk/types": "3.862.0", + "@aws-sdk/util-endpoints": "3.879.0", + "@aws-sdk/util-user-agent-browser": "3.873.0", + "@aws-sdk/util-user-agent-node": "3.883.0", + "@smithy/config-resolver": "^4.1.5", + "@smithy/core": "^3.9.2", + "@smithy/fetch-http-handler": "^5.1.1", + "@smithy/hash-node": "^4.0.5", + "@smithy/invalid-dependency": "^4.0.5", + "@smithy/middleware-content-length": "^4.0.5", + "@smithy/middleware-endpoint": "^4.1.21", + "@smithy/middleware-retry": "^4.1.22", + "@smithy/middleware-serde": "^4.0.9", + "@smithy/middleware-stack": "^4.0.5", + "@smithy/node-config-provider": "^4.1.4", + "@smithy/node-http-handler": "^4.1.1", + "@smithy/protocol-http": "^5.1.3", + "@smithy/smithy-client": "^4.5.2", + "@smithy/types": "^4.3.2", + "@smithy/url-parser": "^4.0.5", + "@smithy/util-base64": "^4.0.0", + "@smithy/util-body-length-browser": "^4.0.0", + "@smithy/util-body-length-node": "^4.0.0", + "@smithy/util-defaults-mode-browser": "^4.0.29", + "@smithy/util-defaults-mode-node": "^4.0.29", + "@smithy/util-endpoints": "^3.0.7", + "@smithy/util-middleware": "^4.0.5", + "@smithy/util-retry": "^4.0.7", + "@smithy/util-utf8": "^4.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-s3vectors/node_modules/@aws-sdk/client-sso": { + "version": "3.883.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-sso/-/client-sso-3.883.0.tgz", + "integrity": "sha512-Ybjw76yPceEBO7+VLjy5+/Gr0A1UNymSDHda5w8tfsS2iHZt/vuD6wrYpHdLoUx4H5la8ZhwcSfK/+kmE+QLPw==", + "dependencies": { + "@aws-crypto/sha256-browser": "5.2.0", + "@aws-crypto/sha256-js": "5.2.0", + "@aws-sdk/core": "3.883.0", + "@aws-sdk/middleware-host-header": "3.873.0", + "@aws-sdk/middleware-logger": "3.876.0", + "@aws-sdk/middleware-recursion-detection": "3.873.0", + "@aws-sdk/middleware-user-agent": "3.883.0", + "@aws-sdk/region-config-resolver": "3.873.0", + "@aws-sdk/types": "3.862.0", + "@aws-sdk/util-endpoints": "3.879.0", + "@aws-sdk/util-user-agent-browser": "3.873.0", + "@aws-sdk/util-user-agent-node": "3.883.0", + "@smithy/config-resolver": "^4.1.5", + "@smithy/core": "^3.9.2", + "@smithy/fetch-http-handler": "^5.1.1", + "@smithy/hash-node": "^4.0.5", + "@smithy/invalid-dependency": "^4.0.5", + "@smithy/middleware-content-length": "^4.0.5", + "@smithy/middleware-endpoint": "^4.1.21", + "@smithy/middleware-retry": "^4.1.22", + "@smithy/middleware-serde": "^4.0.9", + "@smithy/middleware-stack": "^4.0.5", + "@smithy/node-config-provider": "^4.1.4", + "@smithy/node-http-handler": "^4.1.1", + "@smithy/protocol-http": "^5.1.3", + "@smithy/smithy-client": "^4.5.2", + "@smithy/types": "^4.3.2", + "@smithy/url-parser": "^4.0.5", + "@smithy/util-base64": "^4.0.0", + "@smithy/util-body-length-browser": "^4.0.0", + "@smithy/util-body-length-node": "^4.0.0", + "@smithy/util-defaults-mode-browser": "^4.0.29", + "@smithy/util-defaults-mode-node": "^4.0.29", + "@smithy/util-endpoints": "^3.0.7", + "@smithy/util-middleware": "^4.0.5", + "@smithy/util-retry": "^4.0.7", + "@smithy/util-utf8": "^4.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-s3vectors/node_modules/@aws-sdk/core": { + "version": "3.883.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/core/-/core-3.883.0.tgz", + "integrity": "sha512-FmkqnqBLkXi4YsBPbF6vzPa0m4XKUuvgKDbamfw4DZX2CzfBZH6UU4IwmjNV3ZM38m0xraHarK8KIbGSadN3wg==", + "dependencies": { + "@aws-sdk/types": "3.862.0", + "@aws-sdk/xml-builder": "3.873.0", + "@smithy/core": "^3.9.2", + "@smithy/node-config-provider": "^4.1.4", + "@smithy/property-provider": "^4.0.5", + "@smithy/protocol-http": "^5.1.3", + "@smithy/signature-v4": "^5.1.3", + "@smithy/smithy-client": "^4.5.2", + "@smithy/types": "^4.3.2", + "@smithy/util-base64": "^4.0.0", + "@smithy/util-body-length-browser": "^4.0.0", + "@smithy/util-middleware": "^4.0.5", + "@smithy/util-utf8": "^4.0.0", + "fast-xml-parser": "5.2.5", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-s3vectors/node_modules/@aws-sdk/credential-provider-env": { + "version": "3.883.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-env/-/credential-provider-env-3.883.0.tgz", + "integrity": "sha512-Z6tPBXPCodfhIF1rvQKoeRGMkwL6TK0xdl1UoMIA1x4AfBpPICAF77JkFBExk/pdiFYq1d04Qzddd/IiujSlLg==", + "dependencies": { + "@aws-sdk/core": "3.883.0", + "@aws-sdk/types": "3.862.0", + "@smithy/property-provider": "^4.0.5", + "@smithy/types": "^4.3.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-s3vectors/node_modules/@aws-sdk/credential-provider-http": { + "version": "3.883.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-http/-/credential-provider-http-3.883.0.tgz", + "integrity": "sha512-P589ug1lMOOEYLTaQJjSP+Gee34za8Kk2LfteNQfO9SpByHFgGj++Sg8VyIe30eZL8Q+i4qTt24WDCz1c+dgYg==", + "dependencies": { + "@aws-sdk/core": "3.883.0", + "@aws-sdk/types": "3.862.0", + "@smithy/fetch-http-handler": "^5.1.1", + "@smithy/node-http-handler": "^4.1.1", + "@smithy/property-provider": "^4.0.5", + "@smithy/protocol-http": "^5.1.3", + "@smithy/smithy-client": "^4.5.2", + "@smithy/types": "^4.3.2", + "@smithy/util-stream": "^4.2.4", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-s3vectors/node_modules/@aws-sdk/credential-provider-ini": { + "version": "3.883.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-ini/-/credential-provider-ini-3.883.0.tgz", + "integrity": "sha512-n6z9HTzuDEdugXvPiE/95VJXbF4/gBffdV/SRHDJKtDHaRuvp/gggbfmfVSTFouGVnlKPb2pQWQsW3Nr/Y3Lrw==", + "dependencies": { + "@aws-sdk/core": "3.883.0", + "@aws-sdk/credential-provider-env": "3.883.0", + "@aws-sdk/credential-provider-http": "3.883.0", + "@aws-sdk/credential-provider-process": "3.883.0", + "@aws-sdk/credential-provider-sso": "3.883.0", + "@aws-sdk/credential-provider-web-identity": "3.883.0", + "@aws-sdk/nested-clients": "3.883.0", + "@aws-sdk/types": "3.862.0", + "@smithy/credential-provider-imds": "^4.0.7", + "@smithy/property-provider": "^4.0.5", + "@smithy/shared-ini-file-loader": "^4.0.5", + "@smithy/types": "^4.3.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-s3vectors/node_modules/@aws-sdk/credential-provider-node": { + "version": "3.883.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-node/-/credential-provider-node-3.883.0.tgz", + "integrity": "sha512-QIUhsatsrwfB9ZsKpmi0EySSfexVP61wgN7hr493DOileh2QsKW4XATEfsWNmx0dj9323Vg1Mix7bXtRfl9cGg==", + "dependencies": { + "@aws-sdk/credential-provider-env": "3.883.0", + "@aws-sdk/credential-provider-http": "3.883.0", + "@aws-sdk/credential-provider-ini": "3.883.0", + "@aws-sdk/credential-provider-process": "3.883.0", + "@aws-sdk/credential-provider-sso": "3.883.0", + "@aws-sdk/credential-provider-web-identity": "3.883.0", + "@aws-sdk/types": "3.862.0", + "@smithy/credential-provider-imds": "^4.0.7", + "@smithy/property-provider": "^4.0.5", + "@smithy/shared-ini-file-loader": "^4.0.5", + "@smithy/types": "^4.3.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-s3vectors/node_modules/@aws-sdk/credential-provider-process": { + "version": "3.883.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-process/-/credential-provider-process-3.883.0.tgz", + "integrity": "sha512-m1shbHY/Vppy4EdddG9r8x64TO/9FsCjokp5HbKcZvVoTOTgUJrdT8q2TAQJ89+zYIJDqsKbqfrmfwJ1zOdnGQ==", + "dependencies": { + "@aws-sdk/core": "3.883.0", + "@aws-sdk/types": "3.862.0", + "@smithy/property-provider": "^4.0.5", + "@smithy/shared-ini-file-loader": "^4.0.5", + "@smithy/types": "^4.3.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-s3vectors/node_modules/@aws-sdk/credential-provider-sso": { + "version": "3.883.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-sso/-/credential-provider-sso-3.883.0.tgz", + "integrity": "sha512-37ve9Tult08HLXrJFHJM/sGB/vO7wzI6v1RUUfeTiShqx8ZQ5fTzCTNY/duO96jCtCexmFNSycpQzh7lDIf0aA==", + "dependencies": { + "@aws-sdk/client-sso": "3.883.0", + "@aws-sdk/core": "3.883.0", + "@aws-sdk/token-providers": "3.883.0", + "@aws-sdk/types": "3.862.0", + "@smithy/property-provider": "^4.0.5", + "@smithy/shared-ini-file-loader": "^4.0.5", + "@smithy/types": "^4.3.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-s3vectors/node_modules/@aws-sdk/credential-provider-web-identity": { + "version": "3.883.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-web-identity/-/credential-provider-web-identity-3.883.0.tgz", + "integrity": "sha512-SL82K9Jb0vpuTadqTO4Fpdu7SKtebZ3Yo4LZvk/U0UauVMlJj5ZTos0mFx1QSMB9/4TpqifYrSZcdnxgYg8Eqw==", + "dependencies": { + "@aws-sdk/core": "3.883.0", + "@aws-sdk/nested-clients": "3.883.0", + "@aws-sdk/types": "3.862.0", + "@smithy/property-provider": "^4.0.5", + "@smithy/types": "^4.3.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-s3vectors/node_modules/@aws-sdk/middleware-host-header": { + "version": "3.873.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-host-header/-/middleware-host-header-3.873.0.tgz", + "integrity": "sha512-KZ/W1uruWtMOs7D5j3KquOxzCnV79KQW9MjJFZM/M0l6KI8J6V3718MXxFHsTjUE4fpdV6SeCNLV1lwGygsjJA==", + "dependencies": { + "@aws-sdk/types": "3.862.0", + "@smithy/protocol-http": "^5.1.3", + "@smithy/types": "^4.3.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-s3vectors/node_modules/@aws-sdk/middleware-logger": { + "version": "3.876.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-logger/-/middleware-logger-3.876.0.tgz", + "integrity": "sha512-cpWJhOuMSyz9oV25Z/CMHCBTgafDCbv7fHR80nlRrPdPZ8ETNsahwRgltXP1QJJ8r3X/c1kwpOR7tc+RabVzNA==", + "dependencies": { + "@aws-sdk/types": "3.862.0", + "@smithy/types": "^4.3.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-s3vectors/node_modules/@aws-sdk/middleware-recursion-detection": { + "version": "3.873.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-recursion-detection/-/middleware-recursion-detection-3.873.0.tgz", + "integrity": "sha512-OtgY8EXOzRdEWR//WfPkA/fXl0+WwE8hq0y9iw2caNyKPtca85dzrrZWnPqyBK/cpImosrpR1iKMYr41XshsCg==", + "dependencies": { + "@aws-sdk/types": "3.862.0", + "@smithy/protocol-http": "^5.1.3", + "@smithy/types": "^4.3.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-s3vectors/node_modules/@aws-sdk/middleware-user-agent": { + "version": "3.883.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-user-agent/-/middleware-user-agent-3.883.0.tgz", + "integrity": "sha512-q58uLYnGLg7hsnWpdj7Cd1Ulsq1/PUJOHvAfgcBuiDE/+Fwh0DZxZZyjrU+Cr+dbeowIdUaOO8BEDDJ0CUenJw==", + "dependencies": { + "@aws-sdk/core": "3.883.0", + "@aws-sdk/types": "3.862.0", + "@aws-sdk/util-endpoints": "3.879.0", + "@smithy/core": "^3.9.2", + "@smithy/protocol-http": "^5.1.3", + "@smithy/types": "^4.3.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-s3vectors/node_modules/@aws-sdk/nested-clients": { + "version": "3.883.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/nested-clients/-/nested-clients-3.883.0.tgz", + "integrity": "sha512-IhzDM+v0ga53GOOrZ9jmGNr7JU5OR6h6ZK9NgB7GXaa+gsDbqfUuXRwyKDYXldrTXf1sUR3vy1okWDXA7S2ejQ==", + "dependencies": { + "@aws-crypto/sha256-browser": "5.2.0", + "@aws-crypto/sha256-js": "5.2.0", + "@aws-sdk/core": "3.883.0", + "@aws-sdk/middleware-host-header": "3.873.0", + "@aws-sdk/middleware-logger": "3.876.0", + "@aws-sdk/middleware-recursion-detection": "3.873.0", + "@aws-sdk/middleware-user-agent": "3.883.0", + "@aws-sdk/region-config-resolver": "3.873.0", + "@aws-sdk/types": "3.862.0", + "@aws-sdk/util-endpoints": "3.879.0", + "@aws-sdk/util-user-agent-browser": "3.873.0", + "@aws-sdk/util-user-agent-node": "3.883.0", + "@smithy/config-resolver": "^4.1.5", + "@smithy/core": "^3.9.2", + "@smithy/fetch-http-handler": "^5.1.1", + "@smithy/hash-node": "^4.0.5", + "@smithy/invalid-dependency": "^4.0.5", + "@smithy/middleware-content-length": "^4.0.5", + "@smithy/middleware-endpoint": "^4.1.21", + "@smithy/middleware-retry": "^4.1.22", + "@smithy/middleware-serde": "^4.0.9", + "@smithy/middleware-stack": "^4.0.5", + "@smithy/node-config-provider": "^4.1.4", + "@smithy/node-http-handler": "^4.1.1", + "@smithy/protocol-http": "^5.1.3", + "@smithy/smithy-client": "^4.5.2", + "@smithy/types": "^4.3.2", + "@smithy/url-parser": "^4.0.5", + "@smithy/util-base64": "^4.0.0", + "@smithy/util-body-length-browser": "^4.0.0", + "@smithy/util-body-length-node": "^4.0.0", + "@smithy/util-defaults-mode-browser": "^4.0.29", + "@smithy/util-defaults-mode-node": "^4.0.29", + "@smithy/util-endpoints": "^3.0.7", + "@smithy/util-middleware": "^4.0.5", + "@smithy/util-retry": "^4.0.7", + "@smithy/util-utf8": "^4.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-s3vectors/node_modules/@aws-sdk/region-config-resolver": { + "version": "3.873.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/region-config-resolver/-/region-config-resolver-3.873.0.tgz", + "integrity": "sha512-q9sPoef+BBG6PJnc4x60vK/bfVwvRWsPgcoQyIra057S/QGjq5VkjvNk6H8xedf6vnKlXNBwq9BaANBXnldUJg==", + "dependencies": { + "@aws-sdk/types": "3.862.0", + "@smithy/node-config-provider": "^4.1.4", + "@smithy/types": "^4.3.2", + "@smithy/util-config-provider": "^4.0.0", + "@smithy/util-middleware": "^4.0.5", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-s3vectors/node_modules/@aws-sdk/token-providers": { + "version": "3.883.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/token-providers/-/token-providers-3.883.0.tgz", + "integrity": "sha512-tcj/Z5paGn9esxhmmkEW7gt39uNoIRbXG1UwJrfKu4zcTr89h86PDiIE2nxUO3CMQf1KgncPpr5WouPGzkh/QQ==", + "dependencies": { + "@aws-sdk/core": "3.883.0", + "@aws-sdk/nested-clients": "3.883.0", + "@aws-sdk/types": "3.862.0", + "@smithy/property-provider": "^4.0.5", + "@smithy/shared-ini-file-loader": "^4.0.5", + "@smithy/types": "^4.3.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-s3vectors/node_modules/@aws-sdk/types": { + "version": "3.862.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.862.0.tgz", + "integrity": "sha512-Bei+RL0cDxxV+lW2UezLbCYYNeJm6Nzee0TpW0FfyTRBhH9C1XQh4+x+IClriXvgBnRquTMMYsmJfvx8iyLKrg==", + "dependencies": { + "@smithy/types": "^4.3.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-s3vectors/node_modules/@aws-sdk/util-endpoints": { + "version": "3.879.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-endpoints/-/util-endpoints-3.879.0.tgz", + "integrity": "sha512-aVAJwGecYoEmbEFju3127TyJDF9qJsKDUUTRMDuS8tGn+QiWQFnfInmbt+el9GU1gEJupNTXV+E3e74y51fb7A==", + "dependencies": { + "@aws-sdk/types": "3.862.0", + "@smithy/types": "^4.3.2", + "@smithy/url-parser": "^4.0.5", + "@smithy/util-endpoints": "^3.0.7", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-s3vectors/node_modules/@aws-sdk/util-user-agent-browser": { + "version": "3.873.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-user-agent-browser/-/util-user-agent-browser-3.873.0.tgz", + "integrity": "sha512-AcRdbK6o19yehEcywI43blIBhOCSo6UgyWcuOJX5CFF8k39xm1ILCjQlRRjchLAxWrm0lU0Q7XV90RiMMFMZtA==", + "dependencies": { + "@aws-sdk/types": "3.862.0", + "@smithy/types": "^4.3.2", + "bowser": "^2.11.0", + "tslib": "^2.6.2" + } + }, + "node_modules/@aws-sdk/client-s3vectors/node_modules/@aws-sdk/util-user-agent-node": { + "version": "3.883.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-user-agent-node/-/util-user-agent-node-3.883.0.tgz", + "integrity": "sha512-28cQZqC+wsKUHGpTBr+afoIdjS6IoEJkMqcZsmo2Ag8LzmTa6BUWQenFYB0/9BmDy4PZFPUn+uX+rJgWKB+jzA==", + "dependencies": { + "@aws-sdk/middleware-user-agent": "3.883.0", + "@aws-sdk/types": "3.862.0", + "@smithy/node-config-provider": "^4.1.4", + "@smithy/types": "^4.3.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + }, + "peerDependencies": { + "aws-crt": ">=1.0.0" + }, + "peerDependenciesMeta": { + "aws-crt": { + "optional": true + } + } + }, + "node_modules/@aws-sdk/client-s3vectors/node_modules/@aws-sdk/xml-builder": { + "version": "3.873.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/xml-builder/-/xml-builder-3.873.0.tgz", + "integrity": "sha512-kLO7k7cGJ6KaHiExSJWojZurF7SnGMDHXRuQunFnEoD0n1yB6Lqy/S/zHiQ7oJnBhPr9q0TW9qFkrsZb1Uc54w==", + "dependencies": { + "@smithy/types": "^4.3.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-s3vectors/node_modules/@smithy/abort-controller": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@smithy/abort-controller/-/abort-controller-4.1.0.tgz", + "integrity": "sha512-wEhSYznxOmx7EdwK1tYEWJF5+/wmSFsff9BfTOn8oO/+KPl3gsmThrb6MJlWbOC391+Ya31s5JuHiC2RlT80Zg==", + "dependencies": { + "@smithy/types": "^4.4.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-s3vectors/node_modules/@smithy/config-resolver": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/@smithy/config-resolver/-/config-resolver-4.2.0.tgz", + "integrity": "sha512-FA10YhPFLy23uxeWu7pOM2ctlw+gzbPMTZQwrZ8FRIfyJ/p8YIVz7AVTB5jjLD+QIerydyKcVMZur8qzzDILAQ==", + "dependencies": { + "@smithy/node-config-provider": "^4.2.0", + "@smithy/types": "^4.4.0", + "@smithy/util-config-provider": "^4.1.0", + "@smithy/util-middleware": "^4.1.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-s3vectors/node_modules/@smithy/core": { + "version": "3.10.0", + "resolved": "https://registry.npmjs.org/@smithy/core/-/core-3.10.0.tgz", + "integrity": "sha512-bXyD3Ij6b1qDymEYlEcF+QIjwb9gObwZNaRjETJsUEvSIzxFdynSQ3E4ysY7lUFSBzeWBNaFvX+5A0smbC2q6A==", + "dependencies": { + "@smithy/middleware-serde": "^4.1.0", + "@smithy/protocol-http": "^5.2.0", + "@smithy/types": "^4.4.0", + "@smithy/util-base64": "^4.1.0", + "@smithy/util-body-length-browser": "^4.1.0", + "@smithy/util-middleware": "^4.1.0", + "@smithy/util-stream": "^4.3.0", + "@smithy/util-utf8": "^4.1.0", + "@types/uuid": "^9.0.1", + "tslib": "^2.6.2", + "uuid": "^9.0.1" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-s3vectors/node_modules/@smithy/credential-provider-imds": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@smithy/credential-provider-imds/-/credential-provider-imds-4.1.0.tgz", + "integrity": "sha512-iVwNhxTsCQTPdp++4C/d9xvaDmuEWhXi55qJobMp9QMaEHRGH3kErU4F8gohtdsawRqnUy/ANylCjKuhcR2mPw==", + "dependencies": { + "@smithy/node-config-provider": "^4.2.0", + "@smithy/property-provider": "^4.1.0", + "@smithy/types": "^4.4.0", + "@smithy/url-parser": "^4.1.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-s3vectors/node_modules/@smithy/fetch-http-handler": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@smithy/fetch-http-handler/-/fetch-http-handler-5.2.0.tgz", + "integrity": "sha512-VZenjDdVaUGiy3hwQtxm75nhXZrhFG+3xyL93qCQAlYDyhT/jeDWM8/3r5uCFMlTmmyrIjiDyiOynVFchb0BSg==", + "dependencies": { + "@smithy/protocol-http": "^5.2.0", + "@smithy/querystring-builder": "^4.1.0", + "@smithy/types": "^4.4.0", + "@smithy/util-base64": "^4.1.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-s3vectors/node_modules/@smithy/hash-node": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@smithy/hash-node/-/hash-node-4.1.0.tgz", + "integrity": "sha512-mXkJQ/6lAXTuoSsEH+d/fHa4ms4qV5LqYoPLYhmhCRTNcMMdg+4Ya8cMgU1W8+OR40eX0kzsExT7fAILqtTl2w==", + "dependencies": { + "@smithy/types": "^4.4.0", + "@smithy/util-buffer-from": "^4.1.0", + "@smithy/util-utf8": "^4.1.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-s3vectors/node_modules/@smithy/invalid-dependency": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@smithy/invalid-dependency/-/invalid-dependency-4.1.0.tgz", + "integrity": "sha512-4/FcV6aCMzgpM4YyA/GRzTtG28G0RQJcWK722MmpIgzOyfSceWcI9T9c8matpHU9qYYLaWtk8pSGNCLn5kzDRw==", + "dependencies": { + "@smithy/types": "^4.4.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-s3vectors/node_modules/@smithy/is-array-buffer": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-4.1.0.tgz", + "integrity": "sha512-ePTYUOV54wMogio+he4pBybe8fwg4sDvEVDBU8ZlHOZXbXK3/C0XfJgUCu6qAZcawv05ZhZzODGUerFBPsPUDQ==", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-s3vectors/node_modules/@smithy/middleware-content-length": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@smithy/middleware-content-length/-/middleware-content-length-4.1.0.tgz", + "integrity": "sha512-x3dgLFubk/ClKVniJu+ELeZGk4mq7Iv0HgCRUlxNUIcerHTLVmq7Q5eGJL0tOnUltY6KFw5YOKaYxwdcMwox/w==", + "dependencies": { + "@smithy/protocol-http": "^5.2.0", + "@smithy/types": "^4.4.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-s3vectors/node_modules/@smithy/middleware-endpoint": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/@smithy/middleware-endpoint/-/middleware-endpoint-4.2.0.tgz", + "integrity": "sha512-J1eCF7pPDwgv7fGwRd2+Y+H9hlIolF3OZ2PjptonzzyOXXGh/1KGJAHpEcY1EX+WLlclKu2yC5k+9jWXdUG4YQ==", + "dependencies": { + "@smithy/core": "^3.10.0", + "@smithy/middleware-serde": "^4.1.0", + "@smithy/node-config-provider": "^4.2.0", + "@smithy/shared-ini-file-loader": "^4.1.0", + "@smithy/types": "^4.4.0", + "@smithy/url-parser": "^4.1.0", + "@smithy/util-middleware": "^4.1.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-s3vectors/node_modules/@smithy/middleware-retry": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/@smithy/middleware-retry/-/middleware-retry-4.2.0.tgz", + "integrity": "sha512-raL5oWYf5ALl3jCJrajE8enKJEnV/2wZkKS6mb3ZRY2tg3nj66ssdWy5Ps8E6Yu8Wqh3Tt+Sb9LozjvwZupq+A==", + "dependencies": { + "@smithy/node-config-provider": "^4.2.0", + "@smithy/protocol-http": "^5.2.0", + "@smithy/service-error-classification": "^4.1.0", + "@smithy/smithy-client": "^4.6.0", + "@smithy/types": "^4.4.0", + "@smithy/util-middleware": "^4.1.0", + "@smithy/util-retry": "^4.1.0", + "@types/uuid": "^9.0.1", + "tslib": "^2.6.2", + "uuid": "^9.0.1" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-s3vectors/node_modules/@smithy/middleware-serde": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@smithy/middleware-serde/-/middleware-serde-4.1.0.tgz", + "integrity": "sha512-CtLFYlHt7c2VcztyVRc+25JLV4aGpmaSv9F1sPB0AGFL6S+RPythkqpGDa2XBQLJQooKkjLA1g7Xe4450knShg==", + "dependencies": { + "@smithy/protocol-http": "^5.2.0", + "@smithy/types": "^4.4.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-s3vectors/node_modules/@smithy/middleware-stack": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@smithy/middleware-stack/-/middleware-stack-4.1.0.tgz", + "integrity": "sha512-91Fuw4IKp0eK8PNhMXrHRcYA1jvbZ9BJGT91wwPy3bTQT8mHTcQNius/EhSQTlT9QUI3Ki1wjHeNXbWK0tO8YQ==", + "dependencies": { + "@smithy/types": "^4.4.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-s3vectors/node_modules/@smithy/node-config-provider": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/@smithy/node-config-provider/-/node-config-provider-4.2.0.tgz", + "integrity": "sha512-8/fpilqKurQ+f8nFvoFkJ0lrymoMJ+5/CQV5IcTv/MyKhk2Q/EFYCAgTSWHD4nMi9ux9NyBBynkyE9SLg2uSLA==", + "dependencies": { + "@smithy/property-provider": "^4.1.0", + "@smithy/shared-ini-file-loader": "^4.1.0", + "@smithy/types": "^4.4.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-s3vectors/node_modules/@smithy/node-http-handler": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-4.2.0.tgz", + "integrity": "sha512-G4NV70B4hF9vBrUkkvNfWO6+QR4jYjeO4tc+4XrKCb4nPYj49V9Hu8Ftio7Mb0/0IlFyEOORudHrm+isY29nCA==", + "dependencies": { + "@smithy/abort-controller": "^4.1.0", + "@smithy/protocol-http": "^5.2.0", + "@smithy/querystring-builder": "^4.1.0", + "@smithy/types": "^4.4.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-s3vectors/node_modules/@smithy/property-provider": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@smithy/property-provider/-/property-provider-4.1.0.tgz", + "integrity": "sha512-eksMjMHUlG5PwOUWO3k+rfLNOPVPJ70mUzyYNKb5lvyIuAwS4zpWGsxGiuT74DFWonW0xRNy+jgzGauUzX7SyA==", + "dependencies": { + "@smithy/types": "^4.4.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-s3vectors/node_modules/@smithy/protocol-http": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@smithy/protocol-http/-/protocol-http-5.2.0.tgz", + "integrity": "sha512-bwjlh5JwdOQnA01be+5UvHK4HQz4iaRKlVG46hHSJuqi0Ribt3K06Z1oQ29i35Np4G9MCDgkOGcHVyLMreMcbg==", + "dependencies": { + "@smithy/types": "^4.4.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-s3vectors/node_modules/@smithy/querystring-builder": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@smithy/querystring-builder/-/querystring-builder-4.1.0.tgz", + "integrity": "sha512-JqTWmVIq4AF8R8OK/2cCCiQo5ZJ0SRPsDkDgLO5/3z8xxuUp1oMIBBjfuueEe+11hGTZ6rRebzYikpKc6yQV9Q==", + "dependencies": { + "@smithy/types": "^4.4.0", + "@smithy/util-uri-escape": "^4.1.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-s3vectors/node_modules/@smithy/querystring-parser": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@smithy/querystring-parser/-/querystring-parser-4.1.0.tgz", + "integrity": "sha512-VgdHhr8YTRsjOl4hnKFm7xEMOCRTnKw3FJ1nU+dlWNhdt/7eEtxtkdrJdx7PlRTabdANTmvyjE4umUl9cK4awg==", + "dependencies": { + "@smithy/types": "^4.4.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-s3vectors/node_modules/@smithy/service-error-classification": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@smithy/service-error-classification/-/service-error-classification-4.1.0.tgz", + "integrity": "sha512-UBpNFzBNmS20jJomuYn++Y+soF8rOK9AvIGjS9yGP6uRXF5rP18h4FDUsoNpWTlSsmiJ87e2DpZo9ywzSMH7PQ==", + "dependencies": { + "@smithy/types": "^4.4.0" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-s3vectors/node_modules/@smithy/shared-ini-file-loader": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@smithy/shared-ini-file-loader/-/shared-ini-file-loader-4.1.0.tgz", + "integrity": "sha512-W0VMlz9yGdQ/0ZAgWICFjFHTVU0YSfGoCVpKaExRM/FDkTeP/yz8OKvjtGjs6oFokCRm0srgj/g4Cg0xuHu8Rw==", + "dependencies": { + "@smithy/types": "^4.4.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-s3vectors/node_modules/@smithy/signature-v4": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@smithy/signature-v4/-/signature-v4-5.2.0.tgz", + "integrity": "sha512-ObX1ZqG2DdZQlXx9mLD7yAR8AGb7yXurGm+iWx9x4l1fBZ8CZN2BRT09aSbcXVPZXWGdn5VtMuupjxhOTI2EjA==", + "dependencies": { + "@smithy/is-array-buffer": "^4.1.0", + "@smithy/protocol-http": "^5.2.0", + "@smithy/types": "^4.4.0", + "@smithy/util-hex-encoding": "^4.1.0", + "@smithy/util-middleware": "^4.1.0", + "@smithy/util-uri-escape": "^4.1.0", + "@smithy/util-utf8": "^4.1.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-s3vectors/node_modules/@smithy/smithy-client": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/@smithy/smithy-client/-/smithy-client-4.6.0.tgz", + "integrity": "sha512-TvlIshqx5PIi0I0AiR+PluCpJ8olVG++xbYkAIGCUkByaMUlfOXLgjQTmYbr46k4wuDe8eHiTIlUflnjK2drPQ==", + "dependencies": { + "@smithy/core": "^3.10.0", + "@smithy/middleware-endpoint": "^4.2.0", + "@smithy/middleware-stack": "^4.1.0", + "@smithy/protocol-http": "^5.2.0", + "@smithy/types": "^4.4.0", + "@smithy/util-stream": "^4.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-s3vectors/node_modules/@smithy/types": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/@smithy/types/-/types-4.4.0.tgz", + "integrity": "sha512-4jY91NgZz+ZnSFcVzWwngOW6VuK3gR/ihTwSU1R/0NENe9Jd8SfWgbhDCAGUWL3bI7DiDSW7XF6Ui6bBBjrqXw==", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-s3vectors/node_modules/@smithy/url-parser": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@smithy/url-parser/-/url-parser-4.1.0.tgz", + "integrity": "sha512-/LYEIOuO5B2u++tKr1NxNxhZTrr3A63jW8N73YTwVeUyAlbB/YM+hkftsvtKAcMt3ADYo0FsF1GY3anehffSVQ==", + "dependencies": { + "@smithy/querystring-parser": "^4.1.0", + "@smithy/types": "^4.4.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-s3vectors/node_modules/@smithy/util-base64": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@smithy/util-base64/-/util-base64-4.1.0.tgz", + "integrity": "sha512-RUGd4wNb8GeW7xk+AY5ghGnIwM96V0l2uzvs/uVHf+tIuVX2WSvynk5CxNoBCsM2rQRSZElAo9rt3G5mJ/gktQ==", + "dependencies": { + "@smithy/util-buffer-from": "^4.1.0", + "@smithy/util-utf8": "^4.1.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-s3vectors/node_modules/@smithy/util-body-length-browser": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@smithy/util-body-length-browser/-/util-body-length-browser-4.1.0.tgz", + "integrity": "sha512-V2E2Iez+bo6bUMOTENPr6eEmepdY8Hbs+Uc1vkDKgKNA/brTJqOW/ai3JO1BGj9GbCeLqw90pbbH7HFQyFotGQ==", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-s3vectors/node_modules/@smithy/util-body-length-node": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@smithy/util-body-length-node/-/util-body-length-node-4.1.0.tgz", + "integrity": "sha512-BOI5dYjheZdgR9XiEM3HJcEMCXSoqbzu7CzIgYrx0UtmvtC3tC2iDGpJLsSRFffUpy8ymsg2ARMP5fR8mtuUQQ==", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-s3vectors/node_modules/@smithy/util-buffer-from": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-4.1.0.tgz", + "integrity": "sha512-N6yXcjfe/E+xKEccWEKzK6M+crMrlwaCepKja0pNnlSkm6SjAeLKKA++er5Ba0I17gvKfN/ThV+ZOx/CntKTVw==", + "dependencies": { + "@smithy/is-array-buffer": "^4.1.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-s3vectors/node_modules/@smithy/util-config-provider": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@smithy/util-config-provider/-/util-config-provider-4.1.0.tgz", + "integrity": "sha512-swXz2vMjrP1ZusZWVTB/ai5gK+J8U0BWvP10v9fpcFvg+Xi/87LHvHfst2IgCs1i0v4qFZfGwCmeD/KNCdJZbQ==", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-s3vectors/node_modules/@smithy/util-defaults-mode-browser": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@smithy/util-defaults-mode-browser/-/util-defaults-mode-browser-4.1.0.tgz", + "integrity": "sha512-D27cLtJtC4EEeERJXS+JPoogz2tE5zeE3zhWSSu6ER5/wJ5gihUxIzoarDX6K1U27IFTHit5YfHqU4Y9RSGE0w==", + "dependencies": { + "@smithy/property-provider": "^4.1.0", + "@smithy/smithy-client": "^4.6.0", + "@smithy/types": "^4.4.0", + "bowser": "^2.11.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-s3vectors/node_modules/@smithy/util-defaults-mode-node": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@smithy/util-defaults-mode-node/-/util-defaults-mode-node-4.1.0.tgz", + "integrity": "sha512-gnZo3u5dP1o87plKupg39alsbeIY1oFFnCyV2nI/++pL19vTtBLgOyftLEjPjuXmoKn2B2rskX8b7wtC/+3Okg==", + "dependencies": { + "@smithy/config-resolver": "^4.2.0", + "@smithy/credential-provider-imds": "^4.1.0", + "@smithy/node-config-provider": "^4.2.0", + "@smithy/property-provider": "^4.1.0", + "@smithy/smithy-client": "^4.6.0", + "@smithy/types": "^4.4.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-s3vectors/node_modules/@smithy/util-endpoints": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/@smithy/util-endpoints/-/util-endpoints-3.1.0.tgz", + "integrity": "sha512-5LFg48KkunBVGrNs3dnQgLlMXJLVo7k9sdZV5su3rjO3c3DmQ2LwUZI0Zr49p89JWK6sB7KmzyI2fVcDsZkwuw==", + "dependencies": { + "@smithy/node-config-provider": "^4.2.0", + "@smithy/types": "^4.4.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-s3vectors/node_modules/@smithy/util-hex-encoding": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@smithy/util-hex-encoding/-/util-hex-encoding-4.1.0.tgz", + "integrity": "sha512-1LcueNN5GYC4tr8mo14yVYbh/Ur8jHhWOxniZXii+1+ePiIbsLZ5fEI0QQGtbRRP5mOhmooos+rLmVASGGoq5w==", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-s3vectors/node_modules/@smithy/util-middleware": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@smithy/util-middleware/-/util-middleware-4.1.0.tgz", + "integrity": "sha512-612onNcKyxhP7/YOTKFTb2F6sPYtMRddlT5mZvYf1zduzaGzkYhpYIPxIeeEwBZFjnvEqe53Ijl2cYEfJ9d6/Q==", + "dependencies": { + "@smithy/types": "^4.4.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-s3vectors/node_modules/@smithy/util-retry": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@smithy/util-retry/-/util-retry-4.1.0.tgz", + "integrity": "sha512-5AGoBHb207xAKSVwaUnaER+L55WFY8o2RhlafELZR3mB0J91fpL+Qn+zgRkPzns3kccGaF2vy0HmNVBMWmN6dA==", + "dependencies": { + "@smithy/service-error-classification": "^4.1.0", + "@smithy/types": "^4.4.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-s3vectors/node_modules/@smithy/util-stream": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/@smithy/util-stream/-/util-stream-4.3.0.tgz", + "integrity": "sha512-ZOYS94jksDwvsCJtppHprUhsIscRnCKGr6FXCo3SxgQ31ECbza3wqDBqSy6IsAak+h/oAXb1+UYEBmDdseAjUQ==", + "dependencies": { + "@smithy/fetch-http-handler": "^5.2.0", + "@smithy/node-http-handler": "^4.2.0", + "@smithy/types": "^4.4.0", + "@smithy/util-base64": "^4.1.0", + "@smithy/util-buffer-from": "^4.1.0", + "@smithy/util-hex-encoding": "^4.1.0", + "@smithy/util-utf8": "^4.1.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-s3vectors/node_modules/@smithy/util-uri-escape": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@smithy/util-uri-escape/-/util-uri-escape-4.1.0.tgz", + "integrity": "sha512-b0EFQkq35K5NHUYxU72JuoheM6+pytEVUGlTwiFxWFpmddA+Bpz3LgsPRIpBk8lnPE47yT7AF2Egc3jVnKLuPg==", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-s3vectors/node_modules/@smithy/util-utf8": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-4.1.0.tgz", + "integrity": "sha512-mEu1/UIXAdNYuBcyEPbjScKi/+MQVXNIuY/7Cm5XLIWe319kDrT5SizBE95jqtmEXoDbGoZxKLCMttdZdqTZKQ==", + "dependencies": { + "@smithy/util-buffer-from": "^4.1.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-s3vectors/node_modules/fast-xml-parser": { + "version": "5.2.5", + "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-5.2.5.tgz", + "integrity": "sha512-pfX9uG9Ki0yekDHx2SiuRIyFdyAr1kMIMitPvb0YBo8SUfKvia7w7FIyd/l6av85pFYRhZscS75MwMnbvY+hcQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "dependencies": { + "strnum": "^2.1.0" + }, + "bin": { + "fxparser": "src/cli/cli.js" + } + }, + "node_modules/@aws-sdk/client-s3vectors/node_modules/strnum": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/strnum/-/strnum-2.1.1.tgz", + "integrity": "sha512-7ZvoFTiCnGxBtDqJ//Cu6fWtZtc7Y3x+QOirG15wztbdngGSkht27o2pyGWrVy0b4WAy3jbKmnoK6g5VlVNUUw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ] + }, "node_modules/@aws-sdk/client-sso": { "version": "3.848.0", "resolved": "https://registry.npmjs.org/@aws-sdk/client-sso/-/client-sso-3.848.0.tgz", @@ -12118,21 +13143,6 @@ "ws": "^8.18.2" } }, - "node_modules/@kubernetes/client-node/node_modules/@types/node": { - "version": "22.18.6", - "resolved": "https://registry.npmjs.org/@types/node/-/node-22.18.6.tgz", - "integrity": "sha512-r8uszLPpeIWbNKtvWRt/DbVi5zbqZyj1PTmhRMqBMvDnaz1QpmSKujUtJLrqGZeoM8v72MfYggDceY4K1itzWQ==", - "license": "MIT", - "dependencies": { - "undici-types": "~6.21.0" - } - }, - "node_modules/@kubernetes/client-node/node_modules/undici-types": { - "version": "6.21.0", - "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", - "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", - "license": "MIT" - }, "node_modules/@lukeed/ms": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/@lukeed/ms/-/ms-2.0.2.tgz", @@ -15892,6 +16902,15 @@ "@types/node": "*" } }, + "node_modules/@types/cloneable-readable": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/@types/cloneable-readable/-/cloneable-readable-2.0.3.tgz", + "integrity": "sha512-+Ihof4L4iu9k4WTzYbJSkzUxt6f1wzXn6u48fZYxgST+BsC9bBHTOJ59Buy1/4sC9j7ZWF7bxDf/n/mrtk/nzw==", + "dev": true, + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/connect": { "version": "3.4.36", "resolved": "https://registry.npmjs.org/@types/connect/-/connect-3.4.36.tgz", @@ -16017,11 +17036,11 @@ } }, "node_modules/@types/node": { - "version": "20.11.18", - "resolved": "https://registry.npmjs.org/@types/node/-/node-20.11.18.tgz", - "integrity": "sha512-ABT5VWnnYneSBcNWYSCuR05M826RoMyMSGiFivXGx6ZUIsXb9vn4643IEwkg2zbEOSgAiSogtapN2fgc4mAPlw==", + "version": "22.18.8", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.18.8.tgz", + "integrity": "sha512-pAZSHMiagDR7cARo/cch1f3rXy0AEXwsVsVH09FcyeJVAzCnGgmYis7P3JidtTUjyadhTeSo8TgRPswstghDaw==", "dependencies": { - "undici-types": "~5.26.4" + "undici-types": "~6.21.0" } }, "node_modules/@types/node-fetch": { @@ -22836,9 +23855,9 @@ } }, "node_modules/undici-types": { - "version": "5.26.5", - "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", - "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==" + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", + "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==" }, "node_modules/unicode-canonical-property-names-ecmascript": { "version": "2.0.1", @@ -24460,6 +25479,828 @@ } } }, + "@aws-sdk/client-s3vectors": { + "version": "3.883.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-s3vectors/-/client-s3vectors-3.883.0.tgz", + "integrity": "sha512-QtGyHqvih3it25HFnxudHg6J6FvlpSQs9aSN2IgpofAQGLENGVtKFtVlFay2KOlUFPh5aL1OOl5ONR70O8q/Eg==", + "requires": { + "@aws-crypto/sha256-browser": "5.2.0", + "@aws-crypto/sha256-js": "5.2.0", + "@aws-sdk/core": "3.883.0", + "@aws-sdk/credential-provider-node": "3.883.0", + "@aws-sdk/middleware-host-header": "3.873.0", + "@aws-sdk/middleware-logger": "3.876.0", + "@aws-sdk/middleware-recursion-detection": "3.873.0", + "@aws-sdk/middleware-user-agent": "3.883.0", + "@aws-sdk/region-config-resolver": "3.873.0", + "@aws-sdk/types": "3.862.0", + "@aws-sdk/util-endpoints": "3.879.0", + "@aws-sdk/util-user-agent-browser": "3.873.0", + "@aws-sdk/util-user-agent-node": "3.883.0", + "@smithy/config-resolver": "^4.1.5", + "@smithy/core": "^3.9.2", + "@smithy/fetch-http-handler": "^5.1.1", + "@smithy/hash-node": "^4.0.5", + "@smithy/invalid-dependency": "^4.0.5", + "@smithy/middleware-content-length": "^4.0.5", + "@smithy/middleware-endpoint": "^4.1.21", + "@smithy/middleware-retry": "^4.1.22", + "@smithy/middleware-serde": "^4.0.9", + "@smithy/middleware-stack": "^4.0.5", + "@smithy/node-config-provider": "^4.1.4", + "@smithy/node-http-handler": "^4.1.1", + "@smithy/protocol-http": "^5.1.3", + "@smithy/smithy-client": "^4.5.2", + "@smithy/types": "^4.3.2", + "@smithy/url-parser": "^4.0.5", + "@smithy/util-base64": "^4.0.0", + "@smithy/util-body-length-browser": "^4.0.0", + "@smithy/util-body-length-node": "^4.0.0", + "@smithy/util-defaults-mode-browser": "^4.0.29", + "@smithy/util-defaults-mode-node": "^4.0.29", + "@smithy/util-endpoints": "^3.0.7", + "@smithy/util-middleware": "^4.0.5", + "@smithy/util-retry": "^4.0.7", + "@smithy/util-utf8": "^4.0.0", + "tslib": "^2.6.2" + }, + "dependencies": { + "@aws-sdk/client-sso": { + "version": "3.883.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-sso/-/client-sso-3.883.0.tgz", + "integrity": "sha512-Ybjw76yPceEBO7+VLjy5+/Gr0A1UNymSDHda5w8tfsS2iHZt/vuD6wrYpHdLoUx4H5la8ZhwcSfK/+kmE+QLPw==", + "requires": { + "@aws-crypto/sha256-browser": "5.2.0", + "@aws-crypto/sha256-js": "5.2.0", + "@aws-sdk/core": "3.883.0", + "@aws-sdk/middleware-host-header": "3.873.0", + "@aws-sdk/middleware-logger": "3.876.0", + "@aws-sdk/middleware-recursion-detection": "3.873.0", + "@aws-sdk/middleware-user-agent": "3.883.0", + "@aws-sdk/region-config-resolver": "3.873.0", + "@aws-sdk/types": "3.862.0", + "@aws-sdk/util-endpoints": "3.879.0", + "@aws-sdk/util-user-agent-browser": "3.873.0", + "@aws-sdk/util-user-agent-node": "3.883.0", + "@smithy/config-resolver": "^4.1.5", + "@smithy/core": "^3.9.2", + "@smithy/fetch-http-handler": "^5.1.1", + "@smithy/hash-node": "^4.0.5", + "@smithy/invalid-dependency": "^4.0.5", + "@smithy/middleware-content-length": "^4.0.5", + "@smithy/middleware-endpoint": "^4.1.21", + "@smithy/middleware-retry": "^4.1.22", + "@smithy/middleware-serde": "^4.0.9", + "@smithy/middleware-stack": "^4.0.5", + "@smithy/node-config-provider": "^4.1.4", + "@smithy/node-http-handler": "^4.1.1", + "@smithy/protocol-http": "^5.1.3", + "@smithy/smithy-client": "^4.5.2", + "@smithy/types": "^4.3.2", + "@smithy/url-parser": "^4.0.5", + "@smithy/util-base64": "^4.0.0", + "@smithy/util-body-length-browser": "^4.0.0", + "@smithy/util-body-length-node": "^4.0.0", + "@smithy/util-defaults-mode-browser": "^4.0.29", + "@smithy/util-defaults-mode-node": "^4.0.29", + "@smithy/util-endpoints": "^3.0.7", + "@smithy/util-middleware": "^4.0.5", + "@smithy/util-retry": "^4.0.7", + "@smithy/util-utf8": "^4.0.0", + "tslib": "^2.6.2" + } + }, + "@aws-sdk/core": { + "version": "3.883.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/core/-/core-3.883.0.tgz", + "integrity": "sha512-FmkqnqBLkXi4YsBPbF6vzPa0m4XKUuvgKDbamfw4DZX2CzfBZH6UU4IwmjNV3ZM38m0xraHarK8KIbGSadN3wg==", + "requires": { + "@aws-sdk/types": "3.862.0", + "@aws-sdk/xml-builder": "3.873.0", + "@smithy/core": "^3.9.2", + "@smithy/node-config-provider": "^4.1.4", + "@smithy/property-provider": "^4.0.5", + "@smithy/protocol-http": "^5.1.3", + "@smithy/signature-v4": "^5.1.3", + "@smithy/smithy-client": "^4.5.2", + "@smithy/types": "^4.3.2", + "@smithy/util-base64": "^4.0.0", + "@smithy/util-body-length-browser": "^4.0.0", + "@smithy/util-middleware": "^4.0.5", + "@smithy/util-utf8": "^4.0.0", + "fast-xml-parser": "5.2.5", + "tslib": "^2.6.2" + } + }, + "@aws-sdk/credential-provider-env": { + "version": "3.883.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-env/-/credential-provider-env-3.883.0.tgz", + "integrity": "sha512-Z6tPBXPCodfhIF1rvQKoeRGMkwL6TK0xdl1UoMIA1x4AfBpPICAF77JkFBExk/pdiFYq1d04Qzddd/IiujSlLg==", + "requires": { + "@aws-sdk/core": "3.883.0", + "@aws-sdk/types": "3.862.0", + "@smithy/property-provider": "^4.0.5", + "@smithy/types": "^4.3.2", + "tslib": "^2.6.2" + } + }, + "@aws-sdk/credential-provider-http": { + "version": "3.883.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-http/-/credential-provider-http-3.883.0.tgz", + "integrity": "sha512-P589ug1lMOOEYLTaQJjSP+Gee34za8Kk2LfteNQfO9SpByHFgGj++Sg8VyIe30eZL8Q+i4qTt24WDCz1c+dgYg==", + "requires": { + "@aws-sdk/core": "3.883.0", + "@aws-sdk/types": "3.862.0", + "@smithy/fetch-http-handler": "^5.1.1", + "@smithy/node-http-handler": "^4.1.1", + "@smithy/property-provider": "^4.0.5", + "@smithy/protocol-http": "^5.1.3", + "@smithy/smithy-client": "^4.5.2", + "@smithy/types": "^4.3.2", + "@smithy/util-stream": "^4.2.4", + "tslib": "^2.6.2" + } + }, + "@aws-sdk/credential-provider-ini": { + "version": "3.883.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-ini/-/credential-provider-ini-3.883.0.tgz", + "integrity": "sha512-n6z9HTzuDEdugXvPiE/95VJXbF4/gBffdV/SRHDJKtDHaRuvp/gggbfmfVSTFouGVnlKPb2pQWQsW3Nr/Y3Lrw==", + "requires": { + "@aws-sdk/core": "3.883.0", + "@aws-sdk/credential-provider-env": "3.883.0", + "@aws-sdk/credential-provider-http": "3.883.0", + "@aws-sdk/credential-provider-process": "3.883.0", + "@aws-sdk/credential-provider-sso": "3.883.0", + "@aws-sdk/credential-provider-web-identity": "3.883.0", + "@aws-sdk/nested-clients": "3.883.0", + "@aws-sdk/types": "3.862.0", + "@smithy/credential-provider-imds": "^4.0.7", + "@smithy/property-provider": "^4.0.5", + "@smithy/shared-ini-file-loader": "^4.0.5", + "@smithy/types": "^4.3.2", + "tslib": "^2.6.2" + } + }, + "@aws-sdk/credential-provider-node": { + "version": "3.883.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-node/-/credential-provider-node-3.883.0.tgz", + "integrity": "sha512-QIUhsatsrwfB9ZsKpmi0EySSfexVP61wgN7hr493DOileh2QsKW4XATEfsWNmx0dj9323Vg1Mix7bXtRfl9cGg==", + "requires": { + "@aws-sdk/credential-provider-env": "3.883.0", + "@aws-sdk/credential-provider-http": "3.883.0", + "@aws-sdk/credential-provider-ini": "3.883.0", + "@aws-sdk/credential-provider-process": "3.883.0", + "@aws-sdk/credential-provider-sso": "3.883.0", + "@aws-sdk/credential-provider-web-identity": "3.883.0", + "@aws-sdk/types": "3.862.0", + "@smithy/credential-provider-imds": "^4.0.7", + "@smithy/property-provider": "^4.0.5", + "@smithy/shared-ini-file-loader": "^4.0.5", + "@smithy/types": "^4.3.2", + "tslib": "^2.6.2" + } + }, + "@aws-sdk/credential-provider-process": { + "version": "3.883.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-process/-/credential-provider-process-3.883.0.tgz", + "integrity": "sha512-m1shbHY/Vppy4EdddG9r8x64TO/9FsCjokp5HbKcZvVoTOTgUJrdT8q2TAQJ89+zYIJDqsKbqfrmfwJ1zOdnGQ==", + "requires": { + "@aws-sdk/core": "3.883.0", + "@aws-sdk/types": "3.862.0", + "@smithy/property-provider": "^4.0.5", + "@smithy/shared-ini-file-loader": "^4.0.5", + "@smithy/types": "^4.3.2", + "tslib": "^2.6.2" + } + }, + "@aws-sdk/credential-provider-sso": { + "version": "3.883.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-sso/-/credential-provider-sso-3.883.0.tgz", + "integrity": "sha512-37ve9Tult08HLXrJFHJM/sGB/vO7wzI6v1RUUfeTiShqx8ZQ5fTzCTNY/duO96jCtCexmFNSycpQzh7lDIf0aA==", + "requires": { + "@aws-sdk/client-sso": "3.883.0", + "@aws-sdk/core": "3.883.0", + "@aws-sdk/token-providers": "3.883.0", + "@aws-sdk/types": "3.862.0", + "@smithy/property-provider": "^4.0.5", + "@smithy/shared-ini-file-loader": "^4.0.5", + "@smithy/types": "^4.3.2", + "tslib": "^2.6.2" + } + }, + "@aws-sdk/credential-provider-web-identity": { + "version": "3.883.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-web-identity/-/credential-provider-web-identity-3.883.0.tgz", + "integrity": "sha512-SL82K9Jb0vpuTadqTO4Fpdu7SKtebZ3Yo4LZvk/U0UauVMlJj5ZTos0mFx1QSMB9/4TpqifYrSZcdnxgYg8Eqw==", + "requires": { + "@aws-sdk/core": "3.883.0", + "@aws-sdk/nested-clients": "3.883.0", + "@aws-sdk/types": "3.862.0", + "@smithy/property-provider": "^4.0.5", + "@smithy/types": "^4.3.2", + "tslib": "^2.6.2" + } + }, + "@aws-sdk/middleware-host-header": { + "version": "3.873.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-host-header/-/middleware-host-header-3.873.0.tgz", + "integrity": "sha512-KZ/W1uruWtMOs7D5j3KquOxzCnV79KQW9MjJFZM/M0l6KI8J6V3718MXxFHsTjUE4fpdV6SeCNLV1lwGygsjJA==", + "requires": { + "@aws-sdk/types": "3.862.0", + "@smithy/protocol-http": "^5.1.3", + "@smithy/types": "^4.3.2", + "tslib": "^2.6.2" + } + }, + "@aws-sdk/middleware-logger": { + "version": "3.876.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-logger/-/middleware-logger-3.876.0.tgz", + "integrity": "sha512-cpWJhOuMSyz9oV25Z/CMHCBTgafDCbv7fHR80nlRrPdPZ8ETNsahwRgltXP1QJJ8r3X/c1kwpOR7tc+RabVzNA==", + "requires": { + "@aws-sdk/types": "3.862.0", + "@smithy/types": "^4.3.2", + "tslib": "^2.6.2" + } + }, + "@aws-sdk/middleware-recursion-detection": { + "version": "3.873.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-recursion-detection/-/middleware-recursion-detection-3.873.0.tgz", + "integrity": "sha512-OtgY8EXOzRdEWR//WfPkA/fXl0+WwE8hq0y9iw2caNyKPtca85dzrrZWnPqyBK/cpImosrpR1iKMYr41XshsCg==", + "requires": { + "@aws-sdk/types": "3.862.0", + "@smithy/protocol-http": "^5.1.3", + "@smithy/types": "^4.3.2", + "tslib": "^2.6.2" + } + }, + "@aws-sdk/middleware-user-agent": { + "version": "3.883.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-user-agent/-/middleware-user-agent-3.883.0.tgz", + "integrity": "sha512-q58uLYnGLg7hsnWpdj7Cd1Ulsq1/PUJOHvAfgcBuiDE/+Fwh0DZxZZyjrU+Cr+dbeowIdUaOO8BEDDJ0CUenJw==", + "requires": { + "@aws-sdk/core": "3.883.0", + "@aws-sdk/types": "3.862.0", + "@aws-sdk/util-endpoints": "3.879.0", + "@smithy/core": "^3.9.2", + "@smithy/protocol-http": "^5.1.3", + "@smithy/types": "^4.3.2", + "tslib": "^2.6.2" + } + }, + "@aws-sdk/nested-clients": { + "version": "3.883.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/nested-clients/-/nested-clients-3.883.0.tgz", + "integrity": "sha512-IhzDM+v0ga53GOOrZ9jmGNr7JU5OR6h6ZK9NgB7GXaa+gsDbqfUuXRwyKDYXldrTXf1sUR3vy1okWDXA7S2ejQ==", + "requires": { + "@aws-crypto/sha256-browser": "5.2.0", + "@aws-crypto/sha256-js": "5.2.0", + "@aws-sdk/core": "3.883.0", + "@aws-sdk/middleware-host-header": "3.873.0", + "@aws-sdk/middleware-logger": "3.876.0", + "@aws-sdk/middleware-recursion-detection": "3.873.0", + "@aws-sdk/middleware-user-agent": "3.883.0", + "@aws-sdk/region-config-resolver": "3.873.0", + "@aws-sdk/types": "3.862.0", + "@aws-sdk/util-endpoints": "3.879.0", + "@aws-sdk/util-user-agent-browser": "3.873.0", + "@aws-sdk/util-user-agent-node": "3.883.0", + "@smithy/config-resolver": "^4.1.5", + "@smithy/core": "^3.9.2", + "@smithy/fetch-http-handler": "^5.1.1", + "@smithy/hash-node": "^4.0.5", + "@smithy/invalid-dependency": "^4.0.5", + "@smithy/middleware-content-length": "^4.0.5", + "@smithy/middleware-endpoint": "^4.1.21", + "@smithy/middleware-retry": "^4.1.22", + "@smithy/middleware-serde": "^4.0.9", + "@smithy/middleware-stack": "^4.0.5", + "@smithy/node-config-provider": "^4.1.4", + "@smithy/node-http-handler": "^4.1.1", + "@smithy/protocol-http": "^5.1.3", + "@smithy/smithy-client": "^4.5.2", + "@smithy/types": "^4.3.2", + "@smithy/url-parser": "^4.0.5", + "@smithy/util-base64": "^4.0.0", + "@smithy/util-body-length-browser": "^4.0.0", + "@smithy/util-body-length-node": "^4.0.0", + "@smithy/util-defaults-mode-browser": "^4.0.29", + "@smithy/util-defaults-mode-node": "^4.0.29", + "@smithy/util-endpoints": "^3.0.7", + "@smithy/util-middleware": "^4.0.5", + "@smithy/util-retry": "^4.0.7", + "@smithy/util-utf8": "^4.0.0", + "tslib": "^2.6.2" + } + }, + "@aws-sdk/region-config-resolver": { + "version": "3.873.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/region-config-resolver/-/region-config-resolver-3.873.0.tgz", + "integrity": "sha512-q9sPoef+BBG6PJnc4x60vK/bfVwvRWsPgcoQyIra057S/QGjq5VkjvNk6H8xedf6vnKlXNBwq9BaANBXnldUJg==", + "requires": { + "@aws-sdk/types": "3.862.0", + "@smithy/node-config-provider": "^4.1.4", + "@smithy/types": "^4.3.2", + "@smithy/util-config-provider": "^4.0.0", + "@smithy/util-middleware": "^4.0.5", + "tslib": "^2.6.2" + } + }, + "@aws-sdk/token-providers": { + "version": "3.883.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/token-providers/-/token-providers-3.883.0.tgz", + "integrity": "sha512-tcj/Z5paGn9esxhmmkEW7gt39uNoIRbXG1UwJrfKu4zcTr89h86PDiIE2nxUO3CMQf1KgncPpr5WouPGzkh/QQ==", + "requires": { + "@aws-sdk/core": "3.883.0", + "@aws-sdk/nested-clients": "3.883.0", + "@aws-sdk/types": "3.862.0", + "@smithy/property-provider": "^4.0.5", + "@smithy/shared-ini-file-loader": "^4.0.5", + "@smithy/types": "^4.3.2", + "tslib": "^2.6.2" + } + }, + "@aws-sdk/types": { + "version": "3.862.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.862.0.tgz", + "integrity": "sha512-Bei+RL0cDxxV+lW2UezLbCYYNeJm6Nzee0TpW0FfyTRBhH9C1XQh4+x+IClriXvgBnRquTMMYsmJfvx8iyLKrg==", + "requires": { + "@smithy/types": "^4.3.2", + "tslib": "^2.6.2" + } + }, + "@aws-sdk/util-endpoints": { + "version": "3.879.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-endpoints/-/util-endpoints-3.879.0.tgz", + "integrity": "sha512-aVAJwGecYoEmbEFju3127TyJDF9qJsKDUUTRMDuS8tGn+QiWQFnfInmbt+el9GU1gEJupNTXV+E3e74y51fb7A==", + "requires": { + "@aws-sdk/types": "3.862.0", + "@smithy/types": "^4.3.2", + "@smithy/url-parser": "^4.0.5", + "@smithy/util-endpoints": "^3.0.7", + "tslib": "^2.6.2" + } + }, + "@aws-sdk/util-user-agent-browser": { + "version": "3.873.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-user-agent-browser/-/util-user-agent-browser-3.873.0.tgz", + "integrity": "sha512-AcRdbK6o19yehEcywI43blIBhOCSo6UgyWcuOJX5CFF8k39xm1ILCjQlRRjchLAxWrm0lU0Q7XV90RiMMFMZtA==", + "requires": { + "@aws-sdk/types": "3.862.0", + "@smithy/types": "^4.3.2", + "bowser": "^2.11.0", + "tslib": "^2.6.2" + } + }, + "@aws-sdk/util-user-agent-node": { + "version": "3.883.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-user-agent-node/-/util-user-agent-node-3.883.0.tgz", + "integrity": "sha512-28cQZqC+wsKUHGpTBr+afoIdjS6IoEJkMqcZsmo2Ag8LzmTa6BUWQenFYB0/9BmDy4PZFPUn+uX+rJgWKB+jzA==", + "requires": { + "@aws-sdk/middleware-user-agent": "3.883.0", + "@aws-sdk/types": "3.862.0", + "@smithy/node-config-provider": "^4.1.4", + "@smithy/types": "^4.3.2", + "tslib": "^2.6.2" + } + }, + "@aws-sdk/xml-builder": { + "version": "3.873.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/xml-builder/-/xml-builder-3.873.0.tgz", + "integrity": "sha512-kLO7k7cGJ6KaHiExSJWojZurF7SnGMDHXRuQunFnEoD0n1yB6Lqy/S/zHiQ7oJnBhPr9q0TW9qFkrsZb1Uc54w==", + "requires": { + "@smithy/types": "^4.3.2", + "tslib": "^2.6.2" + } + }, + "@smithy/abort-controller": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@smithy/abort-controller/-/abort-controller-4.1.0.tgz", + "integrity": "sha512-wEhSYznxOmx7EdwK1tYEWJF5+/wmSFsff9BfTOn8oO/+KPl3gsmThrb6MJlWbOC391+Ya31s5JuHiC2RlT80Zg==", + "requires": { + "@smithy/types": "^4.4.0", + "tslib": "^2.6.2" + } + }, + "@smithy/config-resolver": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/@smithy/config-resolver/-/config-resolver-4.2.0.tgz", + "integrity": "sha512-FA10YhPFLy23uxeWu7pOM2ctlw+gzbPMTZQwrZ8FRIfyJ/p8YIVz7AVTB5jjLD+QIerydyKcVMZur8qzzDILAQ==", + "requires": { + "@smithy/node-config-provider": "^4.2.0", + "@smithy/types": "^4.4.0", + "@smithy/util-config-provider": "^4.1.0", + "@smithy/util-middleware": "^4.1.0", + "tslib": "^2.6.2" + } + }, + "@smithy/core": { + "version": "3.10.0", + "resolved": "https://registry.npmjs.org/@smithy/core/-/core-3.10.0.tgz", + "integrity": "sha512-bXyD3Ij6b1qDymEYlEcF+QIjwb9gObwZNaRjETJsUEvSIzxFdynSQ3E4ysY7lUFSBzeWBNaFvX+5A0smbC2q6A==", + "requires": { + "@smithy/middleware-serde": "^4.1.0", + "@smithy/protocol-http": "^5.2.0", + "@smithy/types": "^4.4.0", + "@smithy/util-base64": "^4.1.0", + "@smithy/util-body-length-browser": "^4.1.0", + "@smithy/util-middleware": "^4.1.0", + "@smithy/util-stream": "^4.3.0", + "@smithy/util-utf8": "^4.1.0", + "@types/uuid": "^9.0.1", + "tslib": "^2.6.2", + "uuid": "^9.0.1" + } + }, + "@smithy/credential-provider-imds": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@smithy/credential-provider-imds/-/credential-provider-imds-4.1.0.tgz", + "integrity": "sha512-iVwNhxTsCQTPdp++4C/d9xvaDmuEWhXi55qJobMp9QMaEHRGH3kErU4F8gohtdsawRqnUy/ANylCjKuhcR2mPw==", + "requires": { + "@smithy/node-config-provider": "^4.2.0", + "@smithy/property-provider": "^4.1.0", + "@smithy/types": "^4.4.0", + "@smithy/url-parser": "^4.1.0", + "tslib": "^2.6.2" + } + }, + "@smithy/fetch-http-handler": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@smithy/fetch-http-handler/-/fetch-http-handler-5.2.0.tgz", + "integrity": "sha512-VZenjDdVaUGiy3hwQtxm75nhXZrhFG+3xyL93qCQAlYDyhT/jeDWM8/3r5uCFMlTmmyrIjiDyiOynVFchb0BSg==", + "requires": { + "@smithy/protocol-http": "^5.2.0", + "@smithy/querystring-builder": "^4.1.0", + "@smithy/types": "^4.4.0", + "@smithy/util-base64": "^4.1.0", + "tslib": "^2.6.2" + } + }, + "@smithy/hash-node": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@smithy/hash-node/-/hash-node-4.1.0.tgz", + "integrity": "sha512-mXkJQ/6lAXTuoSsEH+d/fHa4ms4qV5LqYoPLYhmhCRTNcMMdg+4Ya8cMgU1W8+OR40eX0kzsExT7fAILqtTl2w==", + "requires": { + "@smithy/types": "^4.4.0", + "@smithy/util-buffer-from": "^4.1.0", + "@smithy/util-utf8": "^4.1.0", + "tslib": "^2.6.2" + } + }, + "@smithy/invalid-dependency": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@smithy/invalid-dependency/-/invalid-dependency-4.1.0.tgz", + "integrity": "sha512-4/FcV6aCMzgpM4YyA/GRzTtG28G0RQJcWK722MmpIgzOyfSceWcI9T9c8matpHU9qYYLaWtk8pSGNCLn5kzDRw==", + "requires": { + "@smithy/types": "^4.4.0", + "tslib": "^2.6.2" + } + }, + "@smithy/is-array-buffer": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-4.1.0.tgz", + "integrity": "sha512-ePTYUOV54wMogio+he4pBybe8fwg4sDvEVDBU8ZlHOZXbXK3/C0XfJgUCu6qAZcawv05ZhZzODGUerFBPsPUDQ==", + "requires": { + "tslib": "^2.6.2" + } + }, + "@smithy/middleware-content-length": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@smithy/middleware-content-length/-/middleware-content-length-4.1.0.tgz", + "integrity": "sha512-x3dgLFubk/ClKVniJu+ELeZGk4mq7Iv0HgCRUlxNUIcerHTLVmq7Q5eGJL0tOnUltY6KFw5YOKaYxwdcMwox/w==", + "requires": { + "@smithy/protocol-http": "^5.2.0", + "@smithy/types": "^4.4.0", + "tslib": "^2.6.2" + } + }, + "@smithy/middleware-endpoint": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/@smithy/middleware-endpoint/-/middleware-endpoint-4.2.0.tgz", + "integrity": "sha512-J1eCF7pPDwgv7fGwRd2+Y+H9hlIolF3OZ2PjptonzzyOXXGh/1KGJAHpEcY1EX+WLlclKu2yC5k+9jWXdUG4YQ==", + "requires": { + "@smithy/core": "^3.10.0", + "@smithy/middleware-serde": "^4.1.0", + "@smithy/node-config-provider": "^4.2.0", + "@smithy/shared-ini-file-loader": "^4.1.0", + "@smithy/types": "^4.4.0", + "@smithy/url-parser": "^4.1.0", + "@smithy/util-middleware": "^4.1.0", + "tslib": "^2.6.2" + } + }, + "@smithy/middleware-retry": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/@smithy/middleware-retry/-/middleware-retry-4.2.0.tgz", + "integrity": "sha512-raL5oWYf5ALl3jCJrajE8enKJEnV/2wZkKS6mb3ZRY2tg3nj66ssdWy5Ps8E6Yu8Wqh3Tt+Sb9LozjvwZupq+A==", + "requires": { + "@smithy/node-config-provider": "^4.2.0", + "@smithy/protocol-http": "^5.2.0", + "@smithy/service-error-classification": "^4.1.0", + "@smithy/smithy-client": "^4.6.0", + "@smithy/types": "^4.4.0", + "@smithy/util-middleware": "^4.1.0", + "@smithy/util-retry": "^4.1.0", + "@types/uuid": "^9.0.1", + "tslib": "^2.6.2", + "uuid": "^9.0.1" + } + }, + "@smithy/middleware-serde": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@smithy/middleware-serde/-/middleware-serde-4.1.0.tgz", + "integrity": "sha512-CtLFYlHt7c2VcztyVRc+25JLV4aGpmaSv9F1sPB0AGFL6S+RPythkqpGDa2XBQLJQooKkjLA1g7Xe4450knShg==", + "requires": { + "@smithy/protocol-http": "^5.2.0", + "@smithy/types": "^4.4.0", + "tslib": "^2.6.2" + } + }, + "@smithy/middleware-stack": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@smithy/middleware-stack/-/middleware-stack-4.1.0.tgz", + "integrity": "sha512-91Fuw4IKp0eK8PNhMXrHRcYA1jvbZ9BJGT91wwPy3bTQT8mHTcQNius/EhSQTlT9QUI3Ki1wjHeNXbWK0tO8YQ==", + "requires": { + "@smithy/types": "^4.4.0", + "tslib": "^2.6.2" + } + }, + "@smithy/node-config-provider": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/@smithy/node-config-provider/-/node-config-provider-4.2.0.tgz", + "integrity": "sha512-8/fpilqKurQ+f8nFvoFkJ0lrymoMJ+5/CQV5IcTv/MyKhk2Q/EFYCAgTSWHD4nMi9ux9NyBBynkyE9SLg2uSLA==", + "requires": { + "@smithy/property-provider": "^4.1.0", + "@smithy/shared-ini-file-loader": "^4.1.0", + "@smithy/types": "^4.4.0", + "tslib": "^2.6.2" + } + }, + "@smithy/node-http-handler": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-4.2.0.tgz", + "integrity": "sha512-G4NV70B4hF9vBrUkkvNfWO6+QR4jYjeO4tc+4XrKCb4nPYj49V9Hu8Ftio7Mb0/0IlFyEOORudHrm+isY29nCA==", + "requires": { + "@smithy/abort-controller": "^4.1.0", + "@smithy/protocol-http": "^5.2.0", + "@smithy/querystring-builder": "^4.1.0", + "@smithy/types": "^4.4.0", + "tslib": "^2.6.2" + } + }, + "@smithy/property-provider": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@smithy/property-provider/-/property-provider-4.1.0.tgz", + "integrity": "sha512-eksMjMHUlG5PwOUWO3k+rfLNOPVPJ70mUzyYNKb5lvyIuAwS4zpWGsxGiuT74DFWonW0xRNy+jgzGauUzX7SyA==", + "requires": { + "@smithy/types": "^4.4.0", + "tslib": "^2.6.2" + } + }, + "@smithy/protocol-http": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@smithy/protocol-http/-/protocol-http-5.2.0.tgz", + "integrity": "sha512-bwjlh5JwdOQnA01be+5UvHK4HQz4iaRKlVG46hHSJuqi0Ribt3K06Z1oQ29i35Np4G9MCDgkOGcHVyLMreMcbg==", + "requires": { + "@smithy/types": "^4.4.0", + "tslib": "^2.6.2" + } + }, + "@smithy/querystring-builder": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@smithy/querystring-builder/-/querystring-builder-4.1.0.tgz", + "integrity": "sha512-JqTWmVIq4AF8R8OK/2cCCiQo5ZJ0SRPsDkDgLO5/3z8xxuUp1oMIBBjfuueEe+11hGTZ6rRebzYikpKc6yQV9Q==", + "requires": { + "@smithy/types": "^4.4.0", + "@smithy/util-uri-escape": "^4.1.0", + "tslib": "^2.6.2" + } + }, + "@smithy/querystring-parser": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@smithy/querystring-parser/-/querystring-parser-4.1.0.tgz", + "integrity": "sha512-VgdHhr8YTRsjOl4hnKFm7xEMOCRTnKw3FJ1nU+dlWNhdt/7eEtxtkdrJdx7PlRTabdANTmvyjE4umUl9cK4awg==", + "requires": { + "@smithy/types": "^4.4.0", + "tslib": "^2.6.2" + } + }, + "@smithy/service-error-classification": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@smithy/service-error-classification/-/service-error-classification-4.1.0.tgz", + "integrity": "sha512-UBpNFzBNmS20jJomuYn++Y+soF8rOK9AvIGjS9yGP6uRXF5rP18h4FDUsoNpWTlSsmiJ87e2DpZo9ywzSMH7PQ==", + "requires": { + "@smithy/types": "^4.4.0" + } + }, + "@smithy/shared-ini-file-loader": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@smithy/shared-ini-file-loader/-/shared-ini-file-loader-4.1.0.tgz", + "integrity": "sha512-W0VMlz9yGdQ/0ZAgWICFjFHTVU0YSfGoCVpKaExRM/FDkTeP/yz8OKvjtGjs6oFokCRm0srgj/g4Cg0xuHu8Rw==", + "requires": { + "@smithy/types": "^4.4.0", + "tslib": "^2.6.2" + } + }, + "@smithy/signature-v4": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@smithy/signature-v4/-/signature-v4-5.2.0.tgz", + "integrity": "sha512-ObX1ZqG2DdZQlXx9mLD7yAR8AGb7yXurGm+iWx9x4l1fBZ8CZN2BRT09aSbcXVPZXWGdn5VtMuupjxhOTI2EjA==", + "requires": { + "@smithy/is-array-buffer": "^4.1.0", + "@smithy/protocol-http": "^5.2.0", + "@smithy/types": "^4.4.0", + "@smithy/util-hex-encoding": "^4.1.0", + "@smithy/util-middleware": "^4.1.0", + "@smithy/util-uri-escape": "^4.1.0", + "@smithy/util-utf8": "^4.1.0", + "tslib": "^2.6.2" + } + }, + "@smithy/smithy-client": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/@smithy/smithy-client/-/smithy-client-4.6.0.tgz", + "integrity": "sha512-TvlIshqx5PIi0I0AiR+PluCpJ8olVG++xbYkAIGCUkByaMUlfOXLgjQTmYbr46k4wuDe8eHiTIlUflnjK2drPQ==", + "requires": { + "@smithy/core": "^3.10.0", + "@smithy/middleware-endpoint": "^4.2.0", + "@smithy/middleware-stack": "^4.1.0", + "@smithy/protocol-http": "^5.2.0", + "@smithy/types": "^4.4.0", + "@smithy/util-stream": "^4.3.0", + "tslib": "^2.6.2" + } + }, + "@smithy/types": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/@smithy/types/-/types-4.4.0.tgz", + "integrity": "sha512-4jY91NgZz+ZnSFcVzWwngOW6VuK3gR/ihTwSU1R/0NENe9Jd8SfWgbhDCAGUWL3bI7DiDSW7XF6Ui6bBBjrqXw==", + "requires": { + "tslib": "^2.6.2" + } + }, + "@smithy/url-parser": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@smithy/url-parser/-/url-parser-4.1.0.tgz", + "integrity": "sha512-/LYEIOuO5B2u++tKr1NxNxhZTrr3A63jW8N73YTwVeUyAlbB/YM+hkftsvtKAcMt3ADYo0FsF1GY3anehffSVQ==", + "requires": { + "@smithy/querystring-parser": "^4.1.0", + "@smithy/types": "^4.4.0", + "tslib": "^2.6.2" + } + }, + "@smithy/util-base64": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@smithy/util-base64/-/util-base64-4.1.0.tgz", + "integrity": "sha512-RUGd4wNb8GeW7xk+AY5ghGnIwM96V0l2uzvs/uVHf+tIuVX2WSvynk5CxNoBCsM2rQRSZElAo9rt3G5mJ/gktQ==", + "requires": { + "@smithy/util-buffer-from": "^4.1.0", + "@smithy/util-utf8": "^4.1.0", + "tslib": "^2.6.2" + } + }, + "@smithy/util-body-length-browser": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@smithy/util-body-length-browser/-/util-body-length-browser-4.1.0.tgz", + "integrity": "sha512-V2E2Iez+bo6bUMOTENPr6eEmepdY8Hbs+Uc1vkDKgKNA/brTJqOW/ai3JO1BGj9GbCeLqw90pbbH7HFQyFotGQ==", + "requires": { + "tslib": "^2.6.2" + } + }, + "@smithy/util-body-length-node": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@smithy/util-body-length-node/-/util-body-length-node-4.1.0.tgz", + "integrity": "sha512-BOI5dYjheZdgR9XiEM3HJcEMCXSoqbzu7CzIgYrx0UtmvtC3tC2iDGpJLsSRFffUpy8ymsg2ARMP5fR8mtuUQQ==", + "requires": { + "tslib": "^2.6.2" + } + }, + "@smithy/util-buffer-from": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-4.1.0.tgz", + "integrity": "sha512-N6yXcjfe/E+xKEccWEKzK6M+crMrlwaCepKja0pNnlSkm6SjAeLKKA++er5Ba0I17gvKfN/ThV+ZOx/CntKTVw==", + "requires": { + "@smithy/is-array-buffer": "^4.1.0", + "tslib": "^2.6.2" + } + }, + "@smithy/util-config-provider": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@smithy/util-config-provider/-/util-config-provider-4.1.0.tgz", + "integrity": "sha512-swXz2vMjrP1ZusZWVTB/ai5gK+J8U0BWvP10v9fpcFvg+Xi/87LHvHfst2IgCs1i0v4qFZfGwCmeD/KNCdJZbQ==", + "requires": { + "tslib": "^2.6.2" + } + }, + "@smithy/util-defaults-mode-browser": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@smithy/util-defaults-mode-browser/-/util-defaults-mode-browser-4.1.0.tgz", + "integrity": "sha512-D27cLtJtC4EEeERJXS+JPoogz2tE5zeE3zhWSSu6ER5/wJ5gihUxIzoarDX6K1U27IFTHit5YfHqU4Y9RSGE0w==", + "requires": { + "@smithy/property-provider": "^4.1.0", + "@smithy/smithy-client": "^4.6.0", + "@smithy/types": "^4.4.0", + "bowser": "^2.11.0", + "tslib": "^2.6.2" + } + }, + "@smithy/util-defaults-mode-node": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@smithy/util-defaults-mode-node/-/util-defaults-mode-node-4.1.0.tgz", + "integrity": "sha512-gnZo3u5dP1o87plKupg39alsbeIY1oFFnCyV2nI/++pL19vTtBLgOyftLEjPjuXmoKn2B2rskX8b7wtC/+3Okg==", + "requires": { + "@smithy/config-resolver": "^4.2.0", + "@smithy/credential-provider-imds": "^4.1.0", + "@smithy/node-config-provider": "^4.2.0", + "@smithy/property-provider": "^4.1.0", + "@smithy/smithy-client": "^4.6.0", + "@smithy/types": "^4.4.0", + "tslib": "^2.6.2" + } + }, + "@smithy/util-endpoints": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/@smithy/util-endpoints/-/util-endpoints-3.1.0.tgz", + "integrity": "sha512-5LFg48KkunBVGrNs3dnQgLlMXJLVo7k9sdZV5su3rjO3c3DmQ2LwUZI0Zr49p89JWK6sB7KmzyI2fVcDsZkwuw==", + "requires": { + "@smithy/node-config-provider": "^4.2.0", + "@smithy/types": "^4.4.0", + "tslib": "^2.6.2" + } + }, + "@smithy/util-hex-encoding": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@smithy/util-hex-encoding/-/util-hex-encoding-4.1.0.tgz", + "integrity": "sha512-1LcueNN5GYC4tr8mo14yVYbh/Ur8jHhWOxniZXii+1+ePiIbsLZ5fEI0QQGtbRRP5mOhmooos+rLmVASGGoq5w==", + "requires": { + "tslib": "^2.6.2" + } + }, + "@smithy/util-middleware": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@smithy/util-middleware/-/util-middleware-4.1.0.tgz", + "integrity": "sha512-612onNcKyxhP7/YOTKFTb2F6sPYtMRddlT5mZvYf1zduzaGzkYhpYIPxIeeEwBZFjnvEqe53Ijl2cYEfJ9d6/Q==", + "requires": { + "@smithy/types": "^4.4.0", + "tslib": "^2.6.2" + } + }, + "@smithy/util-retry": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@smithy/util-retry/-/util-retry-4.1.0.tgz", + "integrity": "sha512-5AGoBHb207xAKSVwaUnaER+L55WFY8o2RhlafELZR3mB0J91fpL+Qn+zgRkPzns3kccGaF2vy0HmNVBMWmN6dA==", + "requires": { + "@smithy/service-error-classification": "^4.1.0", + "@smithy/types": "^4.4.0", + "tslib": "^2.6.2" + } + }, + "@smithy/util-stream": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/@smithy/util-stream/-/util-stream-4.3.0.tgz", + "integrity": "sha512-ZOYS94jksDwvsCJtppHprUhsIscRnCKGr6FXCo3SxgQ31ECbza3wqDBqSy6IsAak+h/oAXb1+UYEBmDdseAjUQ==", + "requires": { + "@smithy/fetch-http-handler": "^5.2.0", + "@smithy/node-http-handler": "^4.2.0", + "@smithy/types": "^4.4.0", + "@smithy/util-base64": "^4.1.0", + "@smithy/util-buffer-from": "^4.1.0", + "@smithy/util-hex-encoding": "^4.1.0", + "@smithy/util-utf8": "^4.1.0", + "tslib": "^2.6.2" + } + }, + "@smithy/util-uri-escape": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@smithy/util-uri-escape/-/util-uri-escape-4.1.0.tgz", + "integrity": "sha512-b0EFQkq35K5NHUYxU72JuoheM6+pytEVUGlTwiFxWFpmddA+Bpz3LgsPRIpBk8lnPE47yT7AF2Egc3jVnKLuPg==", + "requires": { + "tslib": "^2.6.2" + } + }, + "@smithy/util-utf8": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-4.1.0.tgz", + "integrity": "sha512-mEu1/UIXAdNYuBcyEPbjScKi/+MQVXNIuY/7Cm5XLIWe319kDrT5SizBE95jqtmEXoDbGoZxKLCMttdZdqTZKQ==", + "requires": { + "@smithy/util-buffer-from": "^4.1.0", + "tslib": "^2.6.2" + } + }, + "fast-xml-parser": { + "version": "5.2.5", + "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-5.2.5.tgz", + "integrity": "sha512-pfX9uG9Ki0yekDHx2SiuRIyFdyAr1kMIMitPvb0YBo8SUfKvia7w7FIyd/l6av85pFYRhZscS75MwMnbvY+hcQ==", + "requires": { + "strnum": "^2.1.0" + } + }, + "strnum": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/strnum/-/strnum-2.1.1.tgz", + "integrity": "sha512-7ZvoFTiCnGxBtDqJ//Cu6fWtZtc7Y3x+QOirG15wztbdngGSkht27o2pyGWrVy0b4WAy3jbKmnoK6g5VlVNUUw==" + } + } + }, "@aws-sdk/client-sso": { "version": "3.848.0", "resolved": "https://registry.npmjs.org/@aws-sdk/client-sso/-/client-sso-3.848.0.tgz", @@ -32218,21 +34059,6 @@ "stream-buffers": "^3.0.2", "tar-fs": "^3.0.8", "ws": "^8.18.2" - }, - "dependencies": { - "@types/node": { - "version": "22.18.6", - "resolved": "https://registry.npmjs.org/@types/node/-/node-22.18.6.tgz", - "integrity": "sha512-r8uszLPpeIWbNKtvWRt/DbVi5zbqZyj1PTmhRMqBMvDnaz1QpmSKujUtJLrqGZeoM8v72MfYggDceY4K1itzWQ==", - "requires": { - "undici-types": "~6.21.0" - } - }, - "undici-types": { - "version": "6.21.0", - "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", - "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==" - } } }, "@lukeed/ms": { @@ -35098,6 +36924,15 @@ "@types/node": "*" } }, + "@types/cloneable-readable": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/@types/cloneable-readable/-/cloneable-readable-2.0.3.tgz", + "integrity": "sha512-+Ihof4L4iu9k4WTzYbJSkzUxt6f1wzXn6u48fZYxgST+BsC9bBHTOJ59Buy1/4sC9j7ZWF7bxDf/n/mrtk/nzw==", + "dev": true, + "requires": { + "@types/node": "*" + } + }, "@types/connect": { "version": "3.4.36", "resolved": "https://registry.npmjs.org/@types/connect/-/connect-3.4.36.tgz", @@ -35223,11 +37058,11 @@ } }, "@types/node": { - "version": "20.11.18", - "resolved": "https://registry.npmjs.org/@types/node/-/node-20.11.18.tgz", - "integrity": "sha512-ABT5VWnnYneSBcNWYSCuR05M826RoMyMSGiFivXGx6ZUIsXb9vn4643IEwkg2zbEOSgAiSogtapN2fgc4mAPlw==", + "version": "22.18.8", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.18.8.tgz", + "integrity": "sha512-pAZSHMiagDR7cARo/cch1f3rXy0AEXwsVsVH09FcyeJVAzCnGgmYis7P3JidtTUjyadhTeSo8TgRPswstghDaw==", "requires": { - "undici-types": "~5.26.4" + "undici-types": "~6.21.0" } }, "@types/node-fetch": { @@ -40097,9 +41932,9 @@ "dev": true }, "undici-types": { - "version": "5.26.5", - "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", - "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==" + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", + "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==" }, "unicode-canonical-property-names-ecmascript": { "version": "2.0.1", diff --git a/package.json b/package.json index efe5e5e16..d61461628 100644 --- a/package.json +++ b/package.json @@ -30,6 +30,7 @@ "dependencies": { "@aws-sdk/client-ecs": "^3.795.0", "@aws-sdk/client-s3": "3.654.0", + "@aws-sdk/client-s3vectors": "^3.883.0", "@aws-sdk/lib-storage": "3.654.0", "@aws-sdk/s3-request-presigner": "3.654.0", "@fastify/accepts": "^4.3.0", @@ -91,6 +92,7 @@ "@babel/preset-typescript": "^7.27.0", "@types/async-retry": "^1.4.5", "@types/busboy": "^1.3.0", + "@types/cloneable-readable": "^2.0.3", "@types/crypto-js": "^4.1.1", "@types/fs-extra": "^9.0.13", "@types/glob": "^8.1.0", @@ -98,7 +100,7 @@ "@types/js-yaml": "^4.0.5", "@types/multistream": "^4.1.3", "@types/mustache": "^4.2.2", - "@types/node": "^20.11.5", + "@types/node": "^22.18.8", "@types/pg": "^8.6.4", "@types/stream-buffers": "^3.0.7", "@types/xml2js": "^0.4.14", diff --git a/src/app.ts b/src/app.ts index dc99fe7eb..9b97c6699 100644 --- a/src/app.ts +++ b/src/app.ts @@ -62,7 +62,8 @@ const build = (opts: buildOpts = {}): FastifyInstance => { app.register(routes.s3, { prefix: 's3' }) app.register(routes.cdn, { prefix: 'cdn' }) app.register(routes.healthcheck, { prefix: 'health' }) - app.register(routes.iceberg, { prefix: 'iceberg/v1' }) + app.register(routes.iceberg, { prefix: 'iceberg' }) + app.register(routes.vector, { prefix: 'vector' }) setErrorHandler(app) diff --git a/src/config.ts b/src/config.ts index c05d207cc..1c817e80b 100644 --- a/src/config.ts +++ b/src/config.ts @@ -176,6 +176,7 @@ type StorageConfigType = { cdnPurgeEndpointURL?: string cdnPurgeEndpointKey?: string + icebergEnabled: boolean icebergWarehouse: string icebergCatalogUrl: string icebergCatalogAuthType: IcebergCatalogAuthType @@ -186,6 +187,12 @@ type StorageConfigType = { icebergBucketDetectionSuffix: string icebergBucketDetectionMode: 'BUCKET' | 'FULL_PATH' icebergS3DeleteEnabled: boolean + + vectorEnabled: boolean + vectorBucketS3?: string + vectorBucketRegion?: string + vectorMaxBucketsCount: number + vectorMaxIndexesCount: number } function getOptionalConfigFromEnv(key: string, fallback?: string): string | undefined { @@ -506,6 +513,7 @@ export function getConfig(options?: { reload?: boolean }): StorageConfigType { 10 ), + icebergEnabled: getOptionalConfigFromEnv('ICEBERG_ENABLED') === 'true', icebergWarehouse: getOptionalConfigFromEnv('ICEBERG_WAREHOUSE') || '', icebergCatalogUrl: getOptionalConfigFromEnv('ICEBERG_CATALOG_URL') || @@ -524,6 +532,12 @@ export function getConfig(options?: { reload?: boolean }): StorageConfigType { ), icebergMaxTableCount: parseInt(getOptionalConfigFromEnv('ICEBERG_MAX_TABLES') || '10', 10), icebergS3DeleteEnabled: getOptionalConfigFromEnv('ICEBERG_S3_DELETE_ENABLED') === 'true', + + vectorEnabled: getOptionalConfigFromEnv('VECTOR_ENABLED') === 'true', + vectorBucketS3: getOptionalConfigFromEnv('VECTOR_BUCKET_S3') || undefined, + vectorBucketRegion: getOptionalConfigFromEnv('VECTOR_BUCKET_REGION') || undefined, + vectorMaxBucketsCount: parseInt(getOptionalConfigFromEnv('VECTOR_MAX_BUCKETS') || '10', 10), + vectorMaxIndexesCount: parseInt(getOptionalConfigFromEnv('VECTOR_MAX_INDEXES') || '20', 10), } as StorageConfigType const serviceKey = getOptionalConfigFromEnv('SERVICE_KEY') || '' diff --git a/src/http/plugins/index.ts b/src/http/plugins/index.ts index d8e02b9bf..2c6132027 100644 --- a/src/http/plugins/index.ts +++ b/src/http/plugins/index.ts @@ -11,3 +11,4 @@ export * from './signature-v4' export * from './tracing' export * from './signals' export * from './iceberg' +export * from './vector' diff --git a/src/http/plugins/jwt.ts b/src/http/plugins/jwt.ts index 0ef11a402..16a2aa2ab 100644 --- a/src/http/plugins/jwt.ts +++ b/src/http/plugins/jwt.ts @@ -21,6 +21,7 @@ declare module 'fastify' { interface JWTPluginOptions { enforceJwtRoles?: string[] + skipIfAlreadyAuthenticated?: boolean } const { jwtCachingEnabled } = getConfig() @@ -33,6 +34,10 @@ export const jwt = fastifyPlugin( fastify.decorateRequest('jwtPayload', undefined) fastify.addHook('preHandler', async (request) => { + if (opts.skipIfAlreadyAuthenticated && request.isAuthenticated && request.jwtPayload) { + return + } + request.jwt = (request.headers.authorization || '').replace(BEARER, '') if (!request.jwt && request.routeOptions.config.allowInvalidJwt) { diff --git a/src/http/plugins/signature-v4.ts b/src/http/plugins/signature-v4.ts index 6c7a7674a..c2ec8e067 100644 --- a/src/http/plugins/signature-v4.ts +++ b/src/http/plugins/signature-v4.ts @@ -1,8 +1,8 @@ import { FastifyInstance, FastifyRequest } from 'fastify' import fastifyPlugin from 'fastify-plugin' import { getJwtSecret, getTenantConfig, s3CredentialsManager } from '@internal/database' -import { ClientSignature, SignatureV4 } from '@storage/protocols/s3' -import { signJWT, verifyJWT } from '@internal/auth' +import { ClientSignature, SignatureV4, SignatureV4Service } from '@storage/protocols/s3' +import { isJwtToken, signJWT, verifyJWT } from '@internal/auth' import { ERRORS } from '@internal/errors' import { getConfig } from '../../config' @@ -11,6 +11,12 @@ import { ChunkSignatureV4Parser, V4StreamingAlgorithm, } from '@storage/protocols/s3/signature-v4-stream' +import { compose, Readable } from 'stream' +import { HashSpillWritable } from '@internal/streams/hash-stream' +import { RequestByteCounterStream } from '@internal/streams' +import { ByteLimitTransformStream } from '@storage/protocols/s3/byte-limit-stream' +import { Writable } from 'node:stream' +import { enforceJwtRole } from './jwt' const { anonKeyAsync, @@ -30,95 +36,170 @@ type AWSRequest = FastifyRequest<{ Querystring: { 'X-Amz-Credential'?: string } declare module 'fastify' { interface FastifyRequest { - multiPartFileStream?: MultipartFile streamingSignatureV4?: ChunkSignatureV4Parser + multiPartFileStream?: MultipartFile + bodySha256: string } } export const signatureV4 = fastifyPlugin( - async function (fastify: FastifyInstance) { - fastify.addHook('preHandler', async (request: AWSRequest) => { - const clientSignature = await extractSignature(request) - - const sessionToken = clientSignature.sessionToken - - const { - signature: signatureV4, - claims, - token, - } = await createServerSignature(request.tenantId, clientSignature) - - let storagePrefix = s3ProtocolPrefix - if ( - requestAllowXForwardedPrefix && - typeof request.headers['x-forwarded-prefix'] === 'string' - ) { - storagePrefix = request.headers['x-forwarded-prefix'] - } + async function ( + fastify: FastifyInstance, + opts: { + service?: SignatureV4Service + allowBodyHash?: boolean + skipIfJwtToken?: boolean + enforceJwtRoles?: string[] + } = { + service: SignatureV4Service.S3, + allowBodyHash: false, + skipIfJwtToken: false, + enforceJwtRoles: [], + } + ) { + // Use preParsing when allowing to pre-calculate the sha256 of the body + if (opts.allowBodyHash) { + fastify.addHook('preParsing', async (request: AWSRequest, reply, bodyPayload) => { + if (opts.skipIfJwtToken && isJwtToken(request.headers.authorization || '')) { + return bodyPayload + } - const isVerified = signatureV4.verify(clientSignature, { - url: request.url, - body: request.body as string | ReadableStream | Buffer, - headers: request.headers as Record, - method: request.method, - query: request.query as Record, - prefix: storagePrefix, + return await authorizeRequestSignV4( + request, + bodyPayload as Readable, + SignatureV4Service.S3VECTORS, + opts.allowBodyHash + ) }) + } - if (!isVerified && !sessionToken) { - throw ERRORS.SignatureDoesNotMatch( - 'The request signature we calculated does not match the signature you provided. Check your key and signing method.' - ) - } + // Use preHandler when not allowing to pre-calculate the sha256 of the body + if (!opts.allowBodyHash) { + fastify.addHook('preHandler', async (request: AWSRequest) => { + await authorizeRequestSignV4(request, request.raw as Readable, SignatureV4Service.S3) + }) + } - if (!isVerified && sessionToken) { - throw ERRORS.SignatureDoesNotMatch( - 'The request signature we calculated does not match the signature you provided, Check your credentials. ' + - 'The session token should be a valid JWT token' - ) - } + if (opts.enforceJwtRoles) { + fastify.register(enforceJwtRole, { + roles: opts.enforceJwtRoles, + }) + } + }, + { name: 'auth-signature-v4' } +) - const { secret: jwtSecret, jwks } = await getJwtSecret(request.tenantId) - - if (token) { - const payload = await verifyJWT(token, jwtSecret, jwks) - request.jwt = token - request.jwtPayload = payload - request.owner = payload.sub - - if (SignatureV4.isChunkedUpload(request.headers)) { - request.streamingSignatureV4 = createStreamingSignatureV4Parser({ - signatureV4, - streamAlgorithm: request.headers['x-amz-content-sha256'] as V4StreamingAlgorithm, - clientSignature, - trailers: request.headers['x-amz-trailer'] as string, - }) - } - return - } +/** + * Authorize incoming request with Signature V4 + * + * @param request + * @param body + * @param service + * @param allowBodyHash + */ +async function authorizeRequestSignV4( + request: AWSRequest, + body: string | Buffer | Readable, + service: SignatureV4Service, + allowBodyHash = false +) { + const clientSignature = await extractSignature(request) + + const sessionToken = clientSignature.sessionToken + + const { + signature: signatureV4, + claims, + token, + } = await createServerSignature(request.tenantId, clientSignature, service, allowBodyHash) + + let storagePrefix = s3ProtocolPrefix + if (requestAllowXForwardedPrefix && typeof request.headers['x-forwarded-prefix'] === 'string') { + storagePrefix = request.headers['x-forwarded-prefix'] + } - if (!claims) { - throw ERRORS.AccessDenied('Missing claims') - } + let hashStreamComposer: (Writable & { digestHex: () => string }) | undefined + let byteHasherStream: + | (Writable & { + digestHex: () => string + toReadable: (opts: { autoCleanup: boolean }) => Readable + }) + | undefined - const jwt = await signJWT(claims, jwtSecret, '5m') + if (allowBodyHash) { + byteHasherStream = new HashSpillWritable({ + alg: 'sha256', + limitInMemoryBytes: 1024 * 1024 * 5, // 5MB + }) + hashStreamComposer = compose(new ByteLimitTransformStream(1024 * 1024 * 20), byteHasherStream) + hashStreamComposer!.digestHex = byteHasherStream.digestHex.bind(byteHasherStream) + } - request.jwt = jwt - request.jwtPayload = claims - request.owner = claims.sub + const isVerified = await signatureV4.verify(clientSignature, { + url: request.url, + body: body, + headers: request.headers as Record, + method: request.method, + query: request.query as Record, + prefix: storagePrefix, + payloadHasher: hashStreamComposer, + }) - if (SignatureV4.isChunkedUpload(request.headers)) { - request.streamingSignatureV4 = createStreamingSignatureV4Parser({ - signatureV4, - streamAlgorithm: request.headers['x-amz-content-sha256'] as V4StreamingAlgorithm, - clientSignature, - trailers: request.headers['x-amz-trailer'] as string, - }) - } + if (!isVerified && !sessionToken) { + throw ERRORS.SignatureDoesNotMatch( + 'The request signature we calculated does not match the signature you provided. Check your key and signing method.' + ) + } + + if (!isVerified && sessionToken) { + throw ERRORS.SignatureDoesNotMatch( + 'The request signature we calculated does not match the signature you provided, Check your credentials. ' + + 'The session token should be a valid JWT token' + ) + } + + const wasBodyHashed = allowBodyHash && byteHasherStream && byteHasherStream.writableEnded + + const returnStream = wasBodyHashed + ? byteHasherStream!.toReadable({ autoCleanup: true }) + : (body as Readable) + + const { secret: jwtSecret, jwks } = await getJwtSecret(request.tenantId) + + if (!token) { + if (!claims) { + throw ERRORS.AccessDenied('Missing claims') + } + + const jwt = await signJWT(claims, jwtSecret, '5m') + + request.isAuthenticated = true + request.jwt = jwt + request.jwtPayload = claims + request.owner = claims.sub + } else { + const payload = await verifyJWT(token, jwtSecret, jwks) + request.isAuthenticated = true + request.jwt = token + request.jwtPayload = payload + request.owner = payload.sub + } + + if (SignatureV4.isChunkedUpload(request.headers)) { + request.streamingSignatureV4 = createStreamingSignatureV4Parser({ + signatureV4, + streamAlgorithm: request.headers['x-amz-content-sha256'] as V4StreamingAlgorithm, + clientSignature, + trailers: request.headers['x-amz-trailer'] as string, }) - }, - { name: 'auth-signature-v4' } -) + } + + if (wasBodyHashed) { + return compose(returnStream, new RequestByteCounterStream()) + } + + return returnStream +} async function extractSignature(req: AWSRequest) { if (typeof req.headers.authorization === 'string') { @@ -156,9 +237,13 @@ async function extractSignature(req: AWSRequest) { throw ERRORS.AccessDenied('Missing signature') } -async function createServerSignature(tenantId: string, clientSignature: ClientSignature) { +async function createServerSignature( + tenantId: string, + clientSignature: ClientSignature, + awsService = SignatureV4Service.S3, + allowBodyHash = false +) { const awsRegion = storageS3Region - const awsService = 's3' if (clientSignature?.sessionToken) { const tenantAnonKey = isMultitenant @@ -172,6 +257,7 @@ async function createServerSignature(tenantId: string, clientSignature: ClientSi const signature = new SignatureV4({ enforceRegion: s3ProtocolEnforceRegion, allowForwardedHeader: s3ProtocolAllowForwardedHeader, + allowBodyHashing: allowBodyHash, nonCanonicalForwardedHost: s3ProtocolNonCanonicalHostHeader, credentials: { accessKey: tenantId, @@ -193,6 +279,7 @@ async function createServerSignature(tenantId: string, clientSignature: ClientSi const signature = new SignatureV4({ enforceRegion: s3ProtocolEnforceRegion, allowForwardedHeader: s3ProtocolAllowForwardedHeader, + allowBodyHashing: allowBodyHash, nonCanonicalForwardedHost: s3ProtocolNonCanonicalHostHeader, credentials: { accessKey: credential.accessKey, @@ -214,6 +301,7 @@ async function createServerSignature(tenantId: string, clientSignature: ClientSi const signature = new SignatureV4({ enforceRegion: s3ProtocolEnforceRegion, allowForwardedHeader: s3ProtocolAllowForwardedHeader, + allowBodyHashing: allowBodyHash, nonCanonicalForwardedHost: s3ProtocolNonCanonicalHostHeader, credentials: { accessKey: s3ProtocolAccessKeyId, diff --git a/src/http/plugins/vector.ts b/src/http/plugins/vector.ts new file mode 100644 index 000000000..a46fa09f3 --- /dev/null +++ b/src/http/plugins/vector.ts @@ -0,0 +1,49 @@ +import fastifyPlugin from 'fastify-plugin' +import { FastifyInstance } from 'fastify' +import { getTenantConfig } from '@internal/database' +import { + createS3VectorClient, + KnexVectorMetadataDB, + VectorStoreManager, + S3Vector, +} from '@storage/protocols/vector' +import { getConfig } from '../../config' +import { ERRORS } from '@internal/errors' + +declare module 'fastify' { + interface FastifyRequest { + s3Vector: VectorStoreManager + } +} + +const s3VectorClient = createS3VectorClient() +const s3VectorAdapter = new S3Vector(s3VectorClient) + +export const s3vector = fastifyPlugin(async function (fastify: FastifyInstance) { + fastify.addHook('preHandler', async (req) => { + const { isMultitenant, vectorBucketS3, vectorMaxBucketsCount, vectorMaxIndexesCount } = + getConfig() + + if (!vectorBucketS3) { + throw ERRORS.FeatureNotEnabled('vector', 'Vector service not configured') + } + + let maxBucketCount = vectorMaxBucketsCount + let maxIndexCount = vectorMaxIndexesCount + + if (isMultitenant) { + const { features } = await getTenantConfig(req.tenantId) + maxBucketCount = features?.vectorBuckets?.maxBuckets || vectorMaxBucketsCount + maxIndexCount = features?.vectorBuckets?.maxIndexes || vectorMaxIndexesCount + } + + const db = req.db.pool.acquire() + const store = new KnexVectorMetadataDB(db) + req.s3Vector = new VectorStoreManager(s3VectorAdapter, store, { + tenantId: req.tenantId, + vectorBucketName: vectorBucketS3, + maxBucketCount: maxBucketCount, + maxIndexCount: maxIndexCount, + }) + }) +}) diff --git a/src/http/routes/admin/tenants.ts b/src/http/routes/admin/tenants.ts index 2493388aa..5200e580e 100644 --- a/src/http/routes/admin/tenants.ts +++ b/src/http/routes/admin/tenants.ts @@ -66,6 +66,14 @@ const patchSchema = { maxCatalogs: { type: 'number' }, }, }, + vectorBuckets: { + type: 'object', + properties: { + enabled: { type: 'boolean' }, + maxBuckets: { type: 'number' }, + maxIndexes: { type: 'number' }, + }, + }, }, }, }, @@ -113,9 +121,12 @@ interface tenantDBInterface { feature_iceberg_catalog_max_tables?: number | null feature_iceberg_catalog_max_catalogs?: number | null image_transformation_max_resolution?: number + feature_vector_buckets?: boolean + feature_vector_buckets_max_buckets?: number + feature_vector_buckets_max_indexes?: number } -const { dbMigrationFreezeAt } = getConfig() +const { dbMigrationFreezeAt, icebergEnabled, vectorEnabled } = getConfig() export default async function routes(fastify: FastifyInstance) { fastify.register(apiKey) @@ -141,6 +152,9 @@ export default async function routes(fastify: FastifyInstance) { feature_iceberg_catalog_max_catalogs, feature_iceberg_catalog_max_namespaces, feature_iceberg_catalog_max_tables, + feature_vector_buckets, + feature_vector_buckets_max_buckets, + feature_vector_buckets_max_indexes, image_transformation_max_resolution, migrations_version, migrations_status, @@ -172,11 +186,16 @@ export default async function routes(fastify: FastifyInstance) { enabled: feature_s3_protocol, }, icebergCatalog: { - enabled: feature_iceberg_catalog, + enabled: icebergEnabled || feature_iceberg_catalog, maxNamespaces: feature_iceberg_catalog_max_namespaces, maxTables: feature_iceberg_catalog_max_tables, maxCatalogs: feature_iceberg_catalog_max_catalogs, }, + vectorBuckets: { + enabled: vectorEnabled || feature_vector_buckets, + maxBuckets: feature_vector_buckets_max_buckets, + maxIndexes: feature_vector_buckets_max_indexes, + }, }, disableEvents: disable_events, }) @@ -205,6 +224,9 @@ export default async function routes(fastify: FastifyInstance) { feature_iceberg_catalog_max_catalogs, feature_iceberg_catalog_max_namespaces, feature_iceberg_catalog_max_tables, + feature_vector_buckets, + feature_vector_buckets_max_buckets, + feature_vector_buckets_max_indexes, image_transformation_max_resolution, migrations_version, migrations_status, @@ -242,11 +264,16 @@ export default async function routes(fastify: FastifyInstance) { enabled: feature_s3_protocol, }, icebergCatalog: { - enabled: feature_iceberg_catalog, + enabled: icebergEnabled || feature_iceberg_catalog, maxNamespaces: feature_iceberg_catalog_max_namespaces, maxTables: feature_iceberg_catalog_max_tables, maxCatalogs: feature_iceberg_catalog_max_catalogs, }, + vectorBuckets: { + enabled: vectorEnabled || feature_vector_buckets, + maxBuckets: feature_vector_buckets_max_buckets, + maxIndexes: feature_vector_buckets_max_indexes, + }, }, migrationVersion: migrations_version, migrationStatus: migrations_status, @@ -290,6 +317,9 @@ export default async function routes(fastify: FastifyInstance) { feature_iceberg_catalog_max_catalogs: features?.icebergCatalog?.maxCatalogs, feature_iceberg_catalog_max_namespaces: features?.icebergCatalog?.maxNamespaces, feature_iceberg_catalog_max_tables: features?.icebergCatalog?.maxTables, + feature_vector_buckets: features?.vectorBuckets?.enabled ?? false, + feature_vector_buckets_max_buckets: features?.vectorBuckets?.maxBuckets, + feature_vector_buckets_max_indexes: features?.vectorBuckets?.maxIndexes, migrations_version: null, migrations_status: null, tracing_mode: tracingMode, @@ -358,6 +388,9 @@ export default async function routes(fastify: FastifyInstance) { feature_iceberg_catalog_max_catalogs: features?.icebergCatalog?.maxCatalogs, feature_iceberg_catalog_max_namespaces: features?.icebergCatalog?.maxNamespaces, feature_iceberg_catalog_max_tables: features?.icebergCatalog?.maxTables, + feature_vector_buckets: features?.vectorBuckets?.enabled, + feature_vector_buckets_max_buckets: features?.vectorBuckets?.maxBuckets, + feature_vector_buckets_max_indexes: features?.vectorBuckets?.maxIndexes, image_transformation_max_resolution: features?.imageTransformation?.maxResolution === null ? null @@ -456,13 +489,19 @@ export default async function routes(fastify: FastifyInstance) { tenantInfo.tracing_mode = tracingMode } - if (features?.icebergCatalog?.enabled) { + if (typeof features?.icebergCatalog?.enabled) { tenantInfo.feature_iceberg_catalog = features?.icebergCatalog?.enabled tenantInfo.feature_iceberg_catalog_max_namespaces = features?.icebergCatalog?.maxNamespaces tenantInfo.feature_iceberg_catalog_max_tables = features?.icebergCatalog?.maxTables tenantInfo.feature_iceberg_catalog_max_catalogs = features?.icebergCatalog?.maxCatalogs } + if (typeof features?.vectorBuckets?.enabled) { + tenantInfo.feature_vector_buckets = features?.vectorBuckets?.enabled + tenantInfo.feature_vector_buckets_max_buckets = features?.vectorBuckets?.maxBuckets + tenantInfo.feature_vector_buckets_max_indexes = features?.vectorBuckets?.maxIndexes + } + await multitenantKnex.transaction(async (trx) => { await trx('tenants').insert(tenantInfo).onConflict('id').merge() await jwksManager.generateUrlSigningJwk(tenantId, trx) diff --git a/src/http/routes/iceberg/bucket.ts b/src/http/routes/iceberg/bucket.ts new file mode 100644 index 000000000..6c9bdfb93 --- /dev/null +++ b/src/http/routes/iceberg/bucket.ts @@ -0,0 +1,122 @@ +import { FastifyInstance } from 'fastify' +import { FromSchema } from 'json-schema-to-ts' +import { createDefaultSchema, createResponse } from '../../routes-helper' +import { AuthenticatedRequest } from '../../types' +import { ROUTE_OPERATIONS } from '../operations' + +const deleteBucketParamsSchema = { + type: 'object', + properties: { + bucketId: { type: 'string', examples: ['avatars'] }, + }, + required: ['bucketId'], +} as const + +const createBucketBodySchema = { + type: 'object', + properties: { + name: { type: 'string', examples: ['avatars'] }, + }, + required: ['name'], +} as const + +const listBucketsQuerySchema = { + type: 'object', + properties: { + limit: { type: 'integer', minimum: 1, examples: [10] }, + offset: { type: 'integer', minimum: 0, examples: [0] }, + sortColumn: { type: 'string', enum: ['id', 'name', 'created_at', 'updated_at'] }, + sortOrder: { type: 'string', enum: ['asc', 'desc'] }, + search: { type: 'string', examples: ['my-bucket'] }, + }, +} as const + +const successResponseSchema = { + type: 'object', + properties: { + message: { type: 'string', examples: ['Successfully deleted'] }, + }, +} + +interface deleteBucketRequestInterface extends AuthenticatedRequest { + Params: FromSchema +} + +interface createBucketRequestInterface extends AuthenticatedRequest { + Body: FromSchema +} + +interface listBucketRequestInterface extends AuthenticatedRequest { + Querystring: FromSchema +} + +export default async function routes(fastify: FastifyInstance) { + fastify.delete( + '/bucket/:bucketId', + { + schema: createDefaultSchema(successResponseSchema, { + params: deleteBucketParamsSchema, + summary: 'Delete an analytics bucket', + tags: ['bucket'], + }), + config: { + operation: { type: ROUTE_OPERATIONS.DELETE_BUCKET }, + }, + }, + async (request, response) => { + const { bucketId } = request.params + await request.storage.deleteIcebergBucket(bucketId) + + return response.status(200).send(createResponse('Successfully deleted')) + } + ) + + fastify.post( + '/bucket', + { + schema: { + body: createBucketBodySchema, + summary: 'Create an analytics bucket', + tags: ['bucket'], + }, + config: { + operation: { type: ROUTE_OPERATIONS.CREATE_BUCKET }, + }, + }, + async (request, response) => { + const { name } = request.body + const bucket = await request.storage.createIcebergBucket({ + id: name, + }) + + return response.status(200).send(bucket) + } + ) + + fastify.get( + '/bucket', + { + schema: { + querystring: listBucketsQuerySchema, + summary: 'List analytics buckets', + tags: ['bucket'], + }, + config: { + operation: { type: ROUTE_OPERATIONS.LIST_BUCKET }, + }, + }, + async (request, response) => { + const query = request.query + + const bucket = await request.storage.listIcebergBuckets('id,created_at,updated_at', { + limit: query.limit, + offset: query.offset, + sortColumn: query.sortColumn, + sortOrder: query.sortOrder, + search: query.search, + }) + + return response.status(200).send(bucket) + } + ) +} diff --git a/src/http/routes/iceberg/index.ts b/src/http/routes/iceberg/index.ts index 41a288954..9ef0ca316 100644 --- a/src/http/routes/iceberg/index.ts +++ b/src/http/routes/iceberg/index.ts @@ -3,24 +3,34 @@ import { db, icebergRestCatalog, jwt, requireTenantFeature, storage } from '../. import catalogue from './catalog' import namespace from './namespace' import table from './table' +import bucket from './bucket' import { setErrorHandler } from '../../error-handler' import { getConfig } from '../../../config' -const { dbServiceRole } = getConfig() +const { dbServiceRole, icebergEnabled, isMultitenant } = getConfig() export default async function routes(fastify: FastifyInstance) { + // Disable iceberg routes if the feature is not enabled + if (!icebergEnabled && !isMultitenant) { + return + } + fastify.register(async function authenticated(fastify) { fastify.register(jwt, { enforceJwtRoles: [dbServiceRole], }) - fastify.register(requireTenantFeature('icebergCatalog')) + + if (!icebergEnabled && isMultitenant) { + fastify.register(requireTenantFeature('icebergCatalog')) + } fastify.register(db) fastify.register(storage) - fastify.register(icebergRestCatalog) - fastify.register(catalogue) - fastify.register(namespace) - fastify.register(table) + fastify.register(bucket) + fastify.register(icebergRestCatalog, { prefix: 'v1' }) + fastify.register(catalogue, { prefix: 'v1' }) + fastify.register(namespace, { prefix: 'v1' }) + fastify.register(table, { prefix: 'v1' }) setErrorHandler(fastify, { respectStatusCode: true, diff --git a/src/http/routes/index.ts b/src/http/routes/index.ts index d8527091d..26d0ac2f6 100644 --- a/src/http/routes/index.ts +++ b/src/http/routes/index.ts @@ -6,4 +6,5 @@ export { default as healthcheck } from './health' export { default as s3 } from './s3' export { default as iceberg } from './iceberg' export { default as cdn } from './cdn' +export { default as vector } from './vector' export * from './admin' diff --git a/src/http/routes/operations.ts b/src/http/routes/operations.ts index e14571cf6..150b58aa1 100644 --- a/src/http/routes/operations.ts +++ b/src/http/routes/operations.ts @@ -80,4 +80,21 @@ export const ROUTE_OPERATIONS = { ICEBERG_CREATE_TABLE: 'storage.iceberg.table.create', ICEBERG_DROP_TABLE: 'storage.iceberg.table.drop', ICEBERG_COMMIT_TABLE: 'storage.iceberg.table.commit', + + // Vector + CREATE_VECTOR_BUCKET: 'storage.vector.bucket.create', + DELETE_VECTOR_BUCKET: 'storage.vector.bucket.delete', + LIST_VECTOR_BUCKETS: 'storage.vector.bucket.list', + GET_VECTOR_BUCKET: 'storage.vector.bucket.get', + + CREATE_VECTOR_INDEX: 'storage.vector.index.create', + DELETE_VECTOR_INDEX: 'storage.vector.index.delete', + LIST_VECTOR_INDEXES: 'storage.vector.index.list', + GET_VECTOR_INDEX: 'storage.vector.index.get', + + GET_VECTORS: 'storage.vector.vectors.get', + PUT_VECTORS: 'storage.vector.vectors.put', + LIST_VECTORS: 'storage.vector.vectors.list', + QUERY_VECTORS: 'storage.vector.vectors.query', + DELETE_VECTORS: 'storage.vector.vectors.delete', } diff --git a/src/http/routes/vector/create-bucket.ts b/src/http/routes/vector/create-bucket.ts new file mode 100644 index 000000000..bf22a0eb5 --- /dev/null +++ b/src/http/routes/vector/create-bucket.ts @@ -0,0 +1,45 @@ +import { FastifyInstance } from 'fastify' +import { AuthenticatedRequest } from '../../types' +import { FromSchema } from 'json-schema-to-ts' +import { ERRORS } from '@internal/errors' +import { ROUTE_OPERATIONS } from '../operations' + +const createVectorBucket = { + type: 'object', + body: { + type: 'object', + properties: { + vectorBucketName: { type: 'string' }, + }, + required: ['vectorBucketName'], + }, + summary: 'Create a vector bucket', +} as const + +interface createVectorIndexRequest extends AuthenticatedRequest { + Body: FromSchema<(typeof createVectorBucket)['body']> +} + +export default async function routes(fastify: FastifyInstance) { + fastify.post( + '/CreateVectorBucket', + { + config: { + operation: { type: ROUTE_OPERATIONS.CREATE_VECTOR_BUCKET }, + }, + schema: { + ...createVectorBucket, + tags: ['vector'], + }, + }, + async (request, response) => { + if (!request.s3Vector) { + throw ERRORS.FeatureNotEnabled('vectorStore', 'Vector service not configured') + } + + await request.s3Vector.createBucket(request.body.vectorBucketName) + + return response.send() + } + ) +} diff --git a/src/http/routes/vector/create-index.ts b/src/http/routes/vector/create-index.ts new file mode 100644 index 000000000..e98c87d02 --- /dev/null +++ b/src/http/routes/vector/create-index.ts @@ -0,0 +1,66 @@ +import { FastifyInstance } from 'fastify' +import { AuthenticatedRequest } from '../../types' +import { FromSchema } from 'json-schema-to-ts' +import { ERRORS } from '@internal/errors' +import { ROUTE_OPERATIONS } from '../operations' + +const createVectorIndex = { + type: 'object', + body: { + type: 'object', + properties: { + dataType: { type: 'string', enum: ['float32'] }, + dimension: { type: 'number', minimum: 1, maximum: 4096 }, + distanceMetric: { type: 'string', enum: ['cosine', 'euclidean'] }, + indexName: { + type: 'string', + minLength: 3, + maxLength: 45, + pattern: '^[a-z0-9](?:[a-z0-9.-]{1,61})?[a-z0-9]$', + description: + '3-63 chars, lowercase letters, numbers, hyphens, dots; must start/end with letter or number. Must be unique within the vector bucket.', + }, + metadataConfiguration: { + type: 'object', + required: ['nonFilterableMetadataKeys'], + properties: { + nonFilterableMetadataKeys: { + type: 'array', + items: { type: 'string' }, + }, + }, + }, + vectorBucketName: { type: 'string' }, + }, + required: ['dataType', 'dimension', 'distanceMetric', 'indexName', 'vectorBucketName'], + }, + summary: 'Create a vector index', +} as const + +interface createVectorIndexRequest extends AuthenticatedRequest { + Body: FromSchema<(typeof createVectorIndex)['body']> +} + +export default async function routes(fastify: FastifyInstance) { + fastify.post( + '/CreateIndex', + { + config: { + operation: { type: ROUTE_OPERATIONS.CREATE_VECTOR_INDEX }, + }, + schema: { + ...createVectorIndex, + tags: ['vector'], + }, + }, + async (request, response) => { + if (!request.s3Vector) { + throw ERRORS.FeatureNotEnabled('vectorStore', 'Vector service not configured') + } + + await request.s3Vector.createVectorIndex(request.body) + + return response.send() + } + ) +} diff --git a/src/http/routes/vector/delete-bucket.ts b/src/http/routes/vector/delete-bucket.ts new file mode 100644 index 000000000..22db7098a --- /dev/null +++ b/src/http/routes/vector/delete-bucket.ts @@ -0,0 +1,45 @@ +import { FastifyInstance } from 'fastify' +import { AuthenticatedRequest } from '../../types' +import { FromSchema } from 'json-schema-to-ts' +import { ERRORS } from '@internal/errors' +import { ROUTE_OPERATIONS } from '../operations' + +const deleteVectorBucket = { + type: 'object', + body: { + type: 'object', + properties: { + vectorBucketName: { type: 'string' }, + }, + required: ['vectorBucketName'], + }, + summary: 'Create a vector bucket', +} as const + +interface deleteVectorIndexRequest extends AuthenticatedRequest { + Body: FromSchema<(typeof deleteVectorBucket)['body']> +} + +export default async function routes(fastify: FastifyInstance) { + fastify.post( + '/DeleteVectorBucket', + { + config: { + operation: { type: ROUTE_OPERATIONS.DELETE_VECTOR_BUCKET }, + }, + schema: { + ...deleteVectorBucket, + tags: ['vector'], + }, + }, + async (request, response) => { + if (!request.s3Vector) { + throw ERRORS.FeatureNotEnabled('vectorStore', 'Vector service not configured') + } + + await request.s3Vector.deleteBucket(request.body.vectorBucketName) + + return response.send() + } + ) +} diff --git a/src/http/routes/vector/delete-index.ts b/src/http/routes/vector/delete-index.ts new file mode 100644 index 000000000..a556b8f26 --- /dev/null +++ b/src/http/routes/vector/delete-index.ts @@ -0,0 +1,53 @@ +import { FastifyInstance } from 'fastify' +import { AuthenticatedRequest } from '../../types' +import { FromSchema } from 'json-schema-to-ts' +import { ERRORS } from '@internal/errors' +import { ROUTE_OPERATIONS } from '../operations' + +const deleteVectorIndex = { + type: 'object', + body: { + type: 'object', + properties: { + indexName: { + type: 'string', + minLength: 3, + maxLength: 45, + pattern: '^[a-z0-9](?:[a-z0-9.-]{1,61})?[a-z0-9]$', + description: + '3-63 chars, lowercase letters, numbers, hyphens, dots; must start/end with letter or number. Must be unique within the vector bucket.', + }, + vectorBucketName: { type: 'string' }, + }, + required: ['indexName', 'vectorBucketName'], + }, + summary: 'Delete a vector index', +} as const + +interface deleteVectorIndexRequest extends AuthenticatedRequest { + Body: FromSchema<(typeof deleteVectorIndex)['body']> +} + +export default async function routes(fastify: FastifyInstance) { + fastify.post( + '/DeleteIndex', + { + config: { + operation: { type: ROUTE_OPERATIONS.DELETE_VECTOR_INDEX }, + }, + schema: { + ...deleteVectorIndex, + tags: ['vector'], + }, + }, + async (request, response) => { + if (!request.s3Vector) { + throw ERRORS.FeatureNotEnabled('vectorStore', 'Vector service not configured') + } + + await request.s3Vector.deleteIndex(request.body) + + return response.send() + } + ) +} diff --git a/src/http/routes/vector/delete-vectors.ts b/src/http/routes/vector/delete-vectors.ts new file mode 100644 index 000000000..94bb6a681 --- /dev/null +++ b/src/http/routes/vector/delete-vectors.ts @@ -0,0 +1,50 @@ +import { FastifyInstance } from 'fastify' +import { AuthenticatedRequest } from '../../types' +import { FromSchema } from 'json-schema-to-ts' +import { ERRORS } from '@internal/errors' +import { ROUTE_OPERATIONS } from '../operations' + +const deleteVector = { + body: { + type: 'object', + properties: { + vectorBucketName: { type: 'string' }, + indexName: { type: 'string' }, + keys: { type: 'array', items: { type: 'string' } }, + }, + required: ['vectorBucketName', 'indexName', 'keys'], + }, + summary: 'Delete vectors from an index', +} as const + +interface deleteVectorRequest extends AuthenticatedRequest { + Body: FromSchema<(typeof deleteVector)['body']> +} + +export default async function routes(fastify: FastifyInstance) { + fastify.post( + '/DeleteVectors', + { + config: { + operation: { type: ROUTE_OPERATIONS.DELETE_VECTORS }, + }, + schema: { + ...deleteVector, + tags: ['vector'], + }, + }, + async (request, response) => { + if (!request.s3Vector) { + throw ERRORS.FeatureNotEnabled('vectorStore', 'Vector service not configured') + } + + await request.s3Vector.deleteVectors({ + vectorBucketName: request.body.vectorBucketName, + indexName: request.body.indexName, + keys: request.body.keys, + }) + + return response.send() + } + ) +} diff --git a/src/http/routes/vector/get-bucket.ts b/src/http/routes/vector/get-bucket.ts new file mode 100644 index 000000000..28e6b5585 --- /dev/null +++ b/src/http/routes/vector/get-bucket.ts @@ -0,0 +1,45 @@ +import { FastifyInstance } from 'fastify' +import { AuthenticatedRequest } from '../../types' +import { FromSchema } from 'json-schema-to-ts' +import { ERRORS } from '@internal/errors' +import { ROUTE_OPERATIONS } from '../operations' + +const getVectorBucket = { + type: 'object', + body: { + type: 'object', + properties: { + vectorBucketName: { type: 'string' }, + }, + required: ['vectorBucketName'], + }, + summary: 'Create a vector bucket', +} as const + +interface getVectorBucketRequest extends AuthenticatedRequest { + Body: FromSchema<(typeof getVectorBucket)['body']> +} + +export default async function routes(fastify: FastifyInstance) { + fastify.post( + '/GetVectorBucket', + { + config: { + operation: { type: ROUTE_OPERATIONS.GET_VECTOR_BUCKET }, + }, + schema: { + ...getVectorBucket, + tags: ['vector'], + }, + }, + async (request, response) => { + if (!request.s3Vector) { + throw ERRORS.FeatureNotEnabled('vectorStore', 'Vector service not configured') + } + + const bucketResult = await request.s3Vector.getBucket(request.body) + + return response.send(bucketResult) + } + ) +} diff --git a/src/http/routes/vector/get-index.ts b/src/http/routes/vector/get-index.ts new file mode 100644 index 000000000..bff41d1ac --- /dev/null +++ b/src/http/routes/vector/get-index.ts @@ -0,0 +1,64 @@ +import { FastifyInstance } from 'fastify' +import { AuthenticatedRequest } from '../../types' +import { FromSchema } from 'json-schema-to-ts' +import { ERRORS } from '@internal/errors' +import { ROUTE_OPERATIONS } from '../operations' + +const getVectorIndex = { + type: 'object', + body: { + type: 'object', + properties: { + vectorBucketName: { type: 'string' }, + indexName: { + type: 'string', + minLength: 3, + maxLength: 45, + pattern: '^[a-z0-9](?:[a-z0-9.-]{1,61})?[a-z0-9]$', + description: + '3-63 chars, lowercase letters, numbers, hyphens, dots; must start/end with letter or number. Must be unique within the vector bucket.', + }, + }, + required: ['vectorBucketName', 'indexName'], + }, + summary: 'Get a vector index', +} as const + +interface getVectorIndexRequest extends AuthenticatedRequest { + Body: FromSchema<(typeof getVectorIndex)['body']> +} + +export default async function routes(fastify: FastifyInstance) { + fastify.post( + '/GetIndex', + { + config: { + operation: { type: ROUTE_OPERATIONS.GET_VECTOR_INDEX }, + }, + schema: { + ...getVectorIndex, + tags: ['vector'], + }, + }, + async (request, response) => { + if (!request.s3Vector) { + throw ERRORS.FeatureNotEnabled('vectorStore', 'Vector service not configured') + } + + const indexResult = await request.s3Vector.getIndex({ + vectorBucketName: request.body.vectorBucketName, + indexName: request.body.indexName, + }) + + return response.send({ + ...indexResult, + index: { + ...indexResult.index, + creationTime: indexResult.index?.creationTime + ? Math.floor(indexResult.index?.creationTime?.getTime() / 1000) + : undefined, + }, + }) + } + ) +} diff --git a/src/http/routes/vector/get-vectors.ts b/src/http/routes/vector/get-vectors.ts new file mode 100644 index 000000000..7a8691d9e --- /dev/null +++ b/src/http/routes/vector/get-vectors.ts @@ -0,0 +1,49 @@ +import { FastifyInstance } from 'fastify' +import { AuthenticatedRequest } from '../../types' +import { FromSchema } from 'json-schema-to-ts' +import { ERRORS } from '@internal/errors' +import { ROUTE_OPERATIONS } from '../operations' + +const getVectors = { + type: 'object', + body: { + type: 'object', + properties: { + indexName: { type: 'string' }, + keys: { type: 'array', items: { type: 'string' } }, + returnData: { type: 'boolean', default: false }, + returnMetadata: { type: 'boolean', default: false }, + vectorBucketName: { type: 'string' }, + }, + required: ['indexName', 'keys', 'vectorBucketName'], + }, + summary: 'Returns vector attributes', +} as const + +interface getVectorsRequest extends AuthenticatedRequest { + Body: FromSchema<(typeof getVectors)['body']> +} + +export default async function routes(fastify: FastifyInstance) { + fastify.post( + '/GetVectors', + { + config: { + operation: { type: ROUTE_OPERATIONS.GET_VECTORS }, + }, + schema: { + ...getVectors, + tags: ['vector'], + }, + }, + async (request, response) => { + if (!request.s3Vector) { + throw ERRORS.FeatureNotEnabled('vectorStore', 'Vector service not configured') + } + + const indexResult = await request.s3Vector.getVectors(request.body) + + return response.send(indexResult) + } + ) +} diff --git a/src/http/routes/vector/index.ts b/src/http/routes/vector/index.ts new file mode 100644 index 000000000..a93300d51 --- /dev/null +++ b/src/http/routes/vector/index.ts @@ -0,0 +1,79 @@ +import { FastifyInstance } from 'fastify' +import { + db, + dbSuperUser, + enforceJwtRole, + jwt, + requireTenantFeature, + s3vector, + signatureV4, +} from '../../plugins' +import { getConfig } from '../../../config' + +import createVectorBucket from './create-bucket' +import deleteVectorBucket from './delete-bucket' +import listVectorBuckets from './list-buckets' +import getVectorBucket from './get-bucket' + +import createVectorIndex from './create-index' +import deleteVectorIndex from './delete-index' +import listIndexes from './list-indexes' +import getIndex from './get-index' + +import getVectors from './get-vectors' +import putVectors from './put-vectors' +import listVectors from './list-vectors' +import queryVectors from './query-vectors' +import deleteVectors from './delete-vectors' +import { SignatureV4Service } from '@storage/protocols/s3' +import { setErrorHandler } from '../../error-handler' + +export default async function routes(fastify: FastifyInstance) { + const { dbServiceRole, vectorEnabled, isMultitenant } = getConfig() + + if (!vectorEnabled && !isMultitenant) { + return + } + + fastify.register(async function authenticated(fastify) { + if (!vectorEnabled && isMultitenant) { + fastify.register(requireTenantFeature('vectorBuckets')) + } + + fastify.register(signatureV4, { + service: SignatureV4Service.S3VECTORS, + allowBodyHash: true, + skipIfJwtToken: true, + }) + + fastify.register(jwt, { + skipIfAlreadyAuthenticated: true, + }) + + fastify.register(enforceJwtRole, { + roles: [dbServiceRole], + }) + + fastify.register(dbSuperUser) + fastify.register(s3vector) + + fastify.register(createVectorIndex) + fastify.register(deleteVectorIndex) + fastify.register(listIndexes) + fastify.register(getIndex) + + fastify.register(createVectorBucket) + fastify.register(deleteVectorBucket) + fastify.register(listVectorBuckets) + fastify.register(getVectorBucket) + + fastify.register(putVectors) + fastify.register(queryVectors) + fastify.register(deleteVectors) + fastify.register(listVectors) + fastify.register(getVectors) + setErrorHandler(fastify, { + respectStatusCode: true, + }) + }) +} diff --git a/src/http/routes/vector/list-buckets.ts b/src/http/routes/vector/list-buckets.ts new file mode 100644 index 000000000..d5256c6cf --- /dev/null +++ b/src/http/routes/vector/list-buckets.ts @@ -0,0 +1,46 @@ +import { FastifyInstance } from 'fastify' +import { AuthenticatedRequest } from '../../types' +import { FromSchema } from 'json-schema-to-ts' +import { ERRORS } from '@internal/errors' +import { ROUTE_OPERATIONS } from '../operations' + +const listBucket = { + type: 'object', + body: { + type: 'object', + properties: { + maxResults: { type: 'number', minimum: 1, maximum: 500, default: 500 }, + nextToken: { type: 'string' }, + prefix: { type: 'string' }, + }, + }, + summary: 'List vector buckets', +} as const + +interface listBucketRequest extends AuthenticatedRequest { + Body: FromSchema<(typeof listBucket)['body']> +} + +export default async function routes(fastify: FastifyInstance) { + fastify.post( + '/ListVectorBuckets', + { + config: { + operation: { type: ROUTE_OPERATIONS.LIST_VECTOR_BUCKETS }, + }, + schema: { + ...listBucket, + tags: ['vector'], + }, + }, + async (request, response) => { + if (!request.s3Vector) { + throw ERRORS.FeatureNotEnabled('vectorStore', 'Vector service not configured') + } + + const listBucketsResult = await request.s3Vector.listBuckets(request.body) + + return response.send(listBucketsResult) + } + ) +} diff --git a/src/http/routes/vector/list-indexes.ts b/src/http/routes/vector/list-indexes.ts new file mode 100644 index 000000000..33c488a01 --- /dev/null +++ b/src/http/routes/vector/list-indexes.ts @@ -0,0 +1,51 @@ +import { FastifyInstance } from 'fastify' +import { AuthenticatedRequest } from '../../types' +import { FromSchema } from 'json-schema-to-ts' +import { ERRORS } from '@internal/errors' +import { ROUTE_OPERATIONS } from '../operations' + +const listIndex = { + type: 'object', + body: { + type: 'object', + properties: { + vectorBucketName: { type: 'string' }, + maxResults: { type: 'number', minimum: 1, maximum: 500, default: 500 }, + nextToken: { type: 'string' }, + prefix: { type: 'string' }, + }, + required: ['vectorBucketName'], + }, + summary: 'List indexes in a vector bucket', +} as const + +interface listIndexRequest extends AuthenticatedRequest { + Body: FromSchema<(typeof listIndex)['body']> +} + +export default async function routes(fastify: FastifyInstance) { + fastify.post( + '/ListIndexes', + { + config: { + operation: { type: ROUTE_OPERATIONS.LIST_VECTOR_INDEXES }, + }, + schema: { + ...listIndex, + tags: ['vector'], + }, + }, + async (request, response) => { + if (!request.s3Vector) { + throw ERRORS.FeatureNotEnabled('vectorStore', 'Vector service not configured') + } + + const indexResult = await request.s3Vector.listIndexes({ + ...request.body, + vectorBucketName: request.body.vectorBucketName, + }) + + return response.send(indexResult) + } + ) +} diff --git a/src/http/routes/vector/list-vectors.ts b/src/http/routes/vector/list-vectors.ts new file mode 100644 index 000000000..e7e906360 --- /dev/null +++ b/src/http/routes/vector/list-vectors.ts @@ -0,0 +1,60 @@ +import { FastifyInstance } from 'fastify' +import { AuthenticatedRequest } from '../../types' +import { FromSchema } from 'json-schema-to-ts' +import { ERRORS } from '@internal/errors' +import { ROUTE_OPERATIONS } from '../operations' + +const listVectors = { + type: 'object', + body: { + type: 'object', + properties: { + vectorBucketName: { type: 'string' }, + indexArn: { type: 'string' }, + indexName: { + type: 'string', + minLength: 3, + maxLength: 45, + pattern: '^[a-z0-9](?:[a-z0-9.-]{1,61})?[a-z0-9]$', + description: + '3-63 chars, lowercase letters, numbers, hyphens, dots; must start/end with letter or number. Must be unique within the vector bucket.', + }, + maxResults: { type: 'number', minimum: 1, maximum: 500 }, + nextToken: { type: 'string' }, + returnData: { type: 'boolean' }, + returnMetadata: { type: 'boolean' }, + segmentCount: { type: 'number', minimum: 1, maximum: 16 }, + segmentIndex: { type: 'number', minimum: 0, maximum: 15 }, + }, + required: ['vectorBucketName', 'indexName'], + }, + summary: 'List vectors in a vector index', +} as const + +interface listVectorsRequest extends AuthenticatedRequest { + Body: FromSchema<(typeof listVectors)['body']> +} + +export default async function routes(fastify: FastifyInstance) { + fastify.post( + '/ListVectors', + { + config: { + operation: { type: ROUTE_OPERATIONS.LIST_VECTORS }, + }, + schema: { + ...listVectors, + tags: ['vector'], + }, + }, + async (request, response) => { + if (!request.s3Vector) { + throw ERRORS.FeatureNotEnabled('vectorStore', 'Vector service not configured') + } + + const indexResult = await request.s3Vector.listVectors(request.body) + + return response.send(indexResult) + } + ) +} diff --git a/src/http/routes/vector/put-vectors.ts b/src/http/routes/vector/put-vectors.ts new file mode 100644 index 000000000..786a69b25 --- /dev/null +++ b/src/http/routes/vector/put-vectors.ts @@ -0,0 +1,84 @@ +import { FastifyInstance } from 'fastify' +import { AuthenticatedRequest } from '../../types' +import { FromSchema } from 'json-schema-to-ts' +import { ERRORS } from '@internal/errors' +import { ROUTE_OPERATIONS } from '../operations' + +const putVector = { + body: { + type: 'object', + properties: { + vectorBucketName: { type: 'string' }, + indexName: { + type: 'string', + minLength: 3, + maxLength: 45, + pattern: '^[a-z0-9](?:[a-z0-9.-]{1,61})?[a-z0-9]$', + description: + '3-63 chars, lowercase letters, numbers, hyphens, dots; must start/end with letter or number. Must be unique within the vector bucket.', + }, + vectors: { + type: 'array', + minItems: 1, + maxItems: 500, + items: { + type: 'object', + properties: { + data: { + type: 'object', + properties: { + float32: { type: 'array', items: { type: 'number' } }, + }, + required: ['float32'], + }, + metadata: { + type: 'object', + additionalProperties: { type: 'string' }, + }, + key: { type: 'string' }, + }, + required: ['data'], + }, + }, + }, + required: ['vectorBucketName', 'indexName', 'vectors'], + }, + summary: 'Put vectors into an index', +} as const + +interface putVectorRequest extends AuthenticatedRequest { + Body: FromSchema<(typeof putVector)['body']> +} + +export default async function routes(fastify: FastifyInstance) { + fastify.post( + '/PutVectors', + { + config: { + operation: { type: ROUTE_OPERATIONS.PUT_VECTORS }, + }, + schema: { + ...putVector, + tags: ['vector'], + }, + }, + async (request, response) => { + if (!request.s3Vector) { + throw ERRORS.FeatureNotEnabled('vectorStore', 'Vector service not configured') + } + + const indexResult = await request.s3Vector.putVectors({ + vectorBucketName: request.body.vectorBucketName, + indexName: request.body.indexName, + vectors: request.body.vectors.map((v) => { + return { + ...v, + key: v.key || undefined, + } + }), + }) + + return response.send(indexResult) + } + ) +} diff --git a/src/http/routes/vector/query-vectors.ts b/src/http/routes/vector/query-vectors.ts new file mode 100644 index 000000000..18c5f6596 --- /dev/null +++ b/src/http/routes/vector/query-vectors.ts @@ -0,0 +1,159 @@ +import { FastifyInstance, FastifySchemaCompiler } from 'fastify' +import { AuthenticatedRequest } from '../../types' +import { FromSchema } from 'json-schema-to-ts' +import { ERRORS } from '@internal/errors' +import { ROUTE_OPERATIONS } from '../operations' +import Ajv from 'ajv' + +const defs = { + $id: 'https://schemas.example.com/defs.json', + $defs: { + Primitive: { + anyOf: [{ type: 'string' }, { type: 'number' }, { type: 'boolean' }], + }, + FieldOperators: { + type: 'object', + // ensure at least one operator remains after removal + minProperties: 1, + // only allow keys that start with '$' + propertyNames: { pattern: '^\\$' }, + properties: { + $eq: { $ref: '#/$defs/Primitive' }, + $ne: { $ref: '#/$defs/Primitive' }, + $gt: { type: 'number' }, + $gte: { type: 'number' }, + $lt: { type: 'number' }, + $lte: { type: 'number' }, + $in: { type: 'array', minItems: 1, items: { $ref: '#/$defs/Primitive' } }, + $nin: { type: 'array', minItems: 1, items: { $ref: '#/$defs/Primitive' } }, + $exists: { type: 'boolean' }, + }, + additionalProperties: false, + }, + LogicalFilter: { + anyOf: [ + { + type: 'object', + properties: { + $and: { + type: 'array', + minItems: 1, + items: { $ref: '#/$defs/Filter' }, + }, + }, + required: ['$and'], + additionalProperties: false, + }, + { + type: 'object', + properties: { + $or: { + type: 'array', + minItems: 1, + items: { $ref: '#/$defs/Filter' }, + }, + }, + required: ['$or'], + additionalProperties: false, + }, + ], + }, + Filter: { + anyOf: [ + { $ref: '#/$defs/LogicalFilter' }, + { + type: 'object', + additionalProperties: { + anyOf: [{ $ref: '#/$defs/Primitive' }, { $ref: '#/$defs/FieldOperators' }], + }, + }, + ], + }, + }, +} as const + +const queryVectorBody = { + $id: 'https://schemas.example.com/queryVectorBody.json', + type: 'object', + properties: { + filter: { $ref: 'https://schemas.example.com/defs.json#/$defs/Filter' }, + indexArn: { type: 'string' }, + indexName: { + type: 'string', + minLength: 3, + maxLength: 45, + pattern: '^[a-z0-9](?:[a-z0-9.-]{1,61})?[a-z0-9]$', + }, + queryVector: { + type: 'object', + properties: { + float32: { type: 'array', items: { type: 'number' } }, + }, + required: ['float32'], + additionalProperties: false, + }, + returnDistance: { type: 'boolean' }, + returnMetadata: { type: 'boolean' }, + topK: { type: 'number' }, + vectorBucketName: { type: 'string' }, + }, + required: ['vectorBucketName', 'indexName', 'queryVector', 'topK'], + additionalProperties: false, +} as const + +interface queryVectorRequest extends AuthenticatedRequest { + Body: FromSchema +} + +export default async function routes(fastify: FastifyInstance) { + const ajvNoRemoval = new Ajv({ + allErrors: true, + removeAdditional: false, // <- key bit + coerceTypes: false, + }) + + const perRouteValidator: FastifySchemaCompiler = ({ schema }) => { + const validate = ajvNoRemoval.compile(schema as object) + return (data) => { + const ok = validate(data) + if (ok) return { value: data } + return { error: new Error(JSON.stringify(validate.errors)) } + } + } + + ajvNoRemoval.addSchema(defs) + ajvNoRemoval.addSchema(queryVectorBody) + + fastify.post( + '/QueryVectors', + { + validatorCompiler: perRouteValidator, + config: { + operation: { type: ROUTE_OPERATIONS.QUERY_VECTORS }, + }, + schema: { + body: { $ref: 'https://schemas.example.com/queryVectorBody.json' }, + tags: ['vector'], + summary: 'Query vectors', + }, + }, + async (request, response) => { + if (!request.s3Vector) { + throw ERRORS.FeatureNotEnabled('vectorStore', 'Vector service not configured') + } + + const indexResult = await request.s3Vector.queryVectors({ + vectorBucketName: request.body.vectorBucketName, + indexName: request.body.indexName, + indexArn: request.body.indexArn, + queryVector: request.body.queryVector, + topK: request.body.topK, + filter: request.body.filter, + returnDistance: request.body.returnDistance, + returnMetadata: request.body.returnMetadata, + }) + + return response.send(indexResult) + } + ) +} diff --git a/src/internal/auth/jwt.ts b/src/internal/auth/jwt.ts index 2c72cef40..edab4000f 100644 --- a/src/internal/auth/jwt.ts +++ b/src/internal/auth/jwt.ts @@ -218,3 +218,10 @@ export async function generateHS512JWK(): Promise { const secret = await generateSecret('HS512', { extractable: true }) return (await exportJWK(secret)) as JwksConfigKeyOCT } + +const JWT_SHAPE = + /^[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+(?:\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+)?$/ + +export function isJwtToken(token: string) { + return token.replace('Bearer ', '').match(JWT_SHAPE) +} diff --git a/src/internal/database/migrations/types.ts b/src/internal/database/migrations/types.ts index deb270860..fd6292b89 100644 --- a/src/internal/database/migrations/types.ts +++ b/src/internal/database/migrations/types.ts @@ -40,5 +40,8 @@ export const DBMigration = { 'add-search-v2-sort-support': 39, 'fix-prefix-race-conditions-optimized': 40, 'add-object-level-update-trigger': 41, - 'fix-object-level': 42, + 'rollback-prefix-triggers': 42, + 'fix-object-level': 43, + 'vector-bucket-type': 44, + 'vector-buckets': 45, } diff --git a/src/internal/database/tenant.ts b/src/internal/database/tenant.ts index 90d90f789..0e4b8a108 100644 --- a/src/internal/database/tenant.ts +++ b/src/internal/database/tenant.ts @@ -39,6 +39,11 @@ interface TenantConfig { } export interface Features { + vectorBuckets: { + enabled: boolean + maxBuckets: number + maxIndexes: number + } imageTransformation: { enabled: boolean maxResolution?: number @@ -63,8 +68,15 @@ export enum TenantMigrationStatus { FAILED_STALE = 'FAILED_STALE', } -const { isMultitenant, dbServiceRole, serviceKeyAsync, jwtSecret, dbMigrationFreezeAt } = - getConfig() +const { + isMultitenant, + dbServiceRole, + serviceKeyAsync, + jwtSecret, + dbMigrationFreezeAt, + icebergEnabled, + vectorEnabled, +} = getConfig() const tenantConfigCache = new Map() @@ -136,6 +148,9 @@ export async function getTenantConfig(tenantId: string): Promise { feature_iceberg_catalog_max_catalogs, feature_iceberg_catalog_max_namespaces, feature_iceberg_catalog_max_tables, + feature_vector_buckets, + feature_vector_buckets_max_buckets, + feature_vector_buckets_max_indexes, image_transformation_max_resolution, database_pool_url, max_connections, @@ -171,11 +186,16 @@ export async function getTenantConfig(tenantId: string): Promise { enabled: feature_purge_cache, }, icebergCatalog: { - enabled: feature_iceberg_catalog, + enabled: icebergEnabled || feature_iceberg_catalog, maxNamespaces: feature_iceberg_catalog_max_namespaces, maxTables: feature_iceberg_catalog_max_tables, maxCatalogs: feature_iceberg_catalog_max_catalogs, }, + vectorBuckets: { + enabled: vectorEnabled || feature_vector_buckets, + maxBuckets: feature_vector_buckets_max_buckets, + maxIndexes: feature_vector_buckets_max_indexes, + }, }, migrationVersion: migrations_version, migrationStatus: migrations_status, diff --git a/src/internal/errors/codes.ts b/src/internal/errors/codes.ts index e1c63dd88..79dd05c3a 100644 --- a/src/internal/errors/codes.ts +++ b/src/internal/errors/codes.ts @@ -24,6 +24,7 @@ export enum ErrorCode { AccessDenied = 'AccessDenied', ResourceLocked = 'ResourceLocked', DatabaseError = 'DatabaseError', + TransactionError = 'TransactionError', MissingContentLength = 'MissingContentLength', MissingParameter = 'MissingParameter', InvalidParameter = 'InvalidParameter', @@ -43,6 +44,12 @@ export enum ErrorCode { IcebergError = 'IcebergError', IcebergMaximumResourceLimit = 'IcebergMaximumResourceLimit', NoSuchCatalog = 'NoSuchCatalog', + + S3VectorConflictException = 'ConflictException', + S3VectorNotFoundException = 'NotFoundException', + S3VectorBucketNotEmpty = 'VectorBucketNotEmpty', + S3VectorMaxBucketsExceeded = 'S3VectorMaxBucketsExceeded', + S3VectorMaxIndexesExceeded = 'S3VectorMaxIndexesExceeded', } export const ERRORS = { @@ -355,6 +362,14 @@ export const ERRORS = { originalError: e, }), + TransactionError: (message: string, err?: Error) => + new StorageBackendError({ + code: ErrorCode.TransactionError, + httpStatusCode: 409, + message: message, + originalError: err, + }), + DatabaseError: (message: string, err?: Error) => new StorageBackendError({ code: ErrorCode.DatabaseError, @@ -421,6 +436,41 @@ export const ERRORS = { message: `Catalog name "${name}" not found`, }) }, + S3VectorConflictException(resource: string, name: string) { + return new StorageBackendError({ + code: ErrorCode.S3VectorConflictException, + httpStatusCode: 409, + message: `${resource} "${name}" already exists`, + }) + }, + S3VectorNotFoundException(resource: string, name: string) { + return new StorageBackendError({ + code: ErrorCode.S3VectorNotFoundException, + httpStatusCode: 404, + message: `resource "${name}" not found`, + }) + }, + S3VectorBucketNotEmpty(name: string) { + return new StorageBackendError({ + code: ErrorCode.S3VectorBucketNotEmpty, + httpStatusCode: 400, + message: `Vector Bucket "${name}" not empty`, + }) + }, + S3VectorMaxBucketsExceeded(maxBuckets: number) { + return new StorageBackendError({ + code: ErrorCode.S3VectorMaxBucketsExceeded, + httpStatusCode: 400, + message: `Maximum number of buckets exceeded. Max allowed is ${maxBuckets}. Contact support to increase your limit.`, + }) + }, + S3VectorMaxIndexesExceeded(maxIndexes: number) { + return new StorageBackendError({ + code: ErrorCode.S3VectorMaxIndexesExceeded, + httpStatusCode: 400, + message: `Maximum number of indexes exceeded. Max allowed is ${maxIndexes}. Contact support to increase your limit.`, + }) + }, } export function isStorageError(errorType: ErrorCode, error: any): error is StorageBackendError { diff --git a/src/internal/streams/byte-counter.ts b/src/internal/streams/byte-counter.ts index 0200c364e..424559540 100644 --- a/src/internal/streams/byte-counter.ts +++ b/src/internal/streams/byte-counter.ts @@ -17,3 +17,13 @@ export const createByteCounterStream = () => { }, } } + +export class RequestByteCounterStream extends Transform { + public receivedEncodedLength = 0 + + _transform(chunk: Buffer, _enc: BufferEncoding, cb: TransformCallback): void { + this.receivedEncodedLength += chunk.length + + cb(null, chunk) + } +} diff --git a/src/internal/streams/hash-stream.ts b/src/internal/streams/hash-stream.ts new file mode 100644 index 000000000..16297592d --- /dev/null +++ b/src/internal/streams/hash-stream.ts @@ -0,0 +1,261 @@ +// HashSpillWritable.ts +import fs, { WriteStream } from 'node:fs' +import * as fsp from 'node:fs/promises' +import os from 'node:os' +import path from 'node:path' +import { Readable, Writable, WritableOptions } from 'node:stream' +import { finished } from 'node:stream/promises' +import { createHash, randomUUID } from 'node:crypto' + +export interface HashSpillWritableOptions { + /** Max bytes to keep in memory before spilling to disk (required, > 0). */ + limitInMemoryBytes: number + /** Hash algorithm (default: 'sha256'). */ + alg?: string + /** Parent directory for temp dirs (default: os.tmpdir()). */ + tmpRoot?: string + /** Writable options to pass to base class (rarely needed). */ + writableOptions?: WritableOptions +} + +export interface ToReadableOptions { + /** + * If true and data spilled to disk, the spilled file/dir will be removed + * after the **last** reader closes/ends. + */ + autoCleanup?: boolean +} + +/** + * Writable that hashes all bytes and buffers in memory up to `limitBytes`. + * On first overflow, it spills to a unique temp file and appends subsequent data there. + * - Call `digestHex()` *after* 'finish' (e.g. after pipeline resolves). + * - Get a fresh readable with `toReadable({ autoCleanup })`. If multiple readers + * are created, cleanup is deferred until the last one finishes. + * - Call `cleanup()` to explicitly remove temp artifacts; it defers until readers close. + */ +export class HashSpillWritable extends Writable { + private readonly limitBytes: number + private readonly alg: string + private readonly tmpRoot: string + + // Hashing + private hash = createHash('sha256') + + // Memory buffer until first spill + private chunks: Buffer[] | null = [] + private memSize = 0 + + // Spill state + private spilled = false + private tmpDir: string | null = null + private filePath: string | null = null + private fileStream: WriteStream | null = null + private ensureFilePromise: Promise | null = null + + // Readers + cleanup + private activeReaders = 0 + private cleanupPending = false + private cleanupRunning: Promise | null = null + + // Bookkeeping + private totalBytes = 0 + private finishedFlag = false + private digestValue: string | null = null + + constructor(opts: HashSpillWritableOptions) { + super(opts?.writableOptions) + if (!(opts?.limitInMemoryBytes > 0)) throw new Error('limitBytes must be a positive number') + + this.limitBytes = opts.limitInMemoryBytes + this.alg = opts.alg ?? 'sha256' + this.tmpRoot = opts.tmpRoot ?? os.tmpdir() + + this.hash = createHash(this.alg) + + this.on('error', () => { + void this.cleanupAsync() + }) + this.on('close', () => { + if (this.fileStream && !this.fileStream.closed) { + try { + this.fileStream.destroy() + } catch {} + } + }) + } + + // Writable implementation + _write(chunk: Buffer, _enc: BufferEncoding, cb: (error?: Error | null) => void): void { + try { + this.hash.update(chunk) + this.totalBytes += chunk.length + + if (!this.spilled) { + if (this.memSize + chunk.length <= this.limitBytes) { + this.chunks!.push(chunk) + this.memSize += chunk.length + cb() + return + } + // Spill + this.spilled = true + this.spillToDiskAndWrite(chunk).then(() => cb(), cb) + } else { + this.writeToFile(chunk).then(() => cb(), cb) + } + } catch (err) { + cb(err as Error) + } + } + + _final(cb: (error?: Error | null) => void): void { + const finalize = async () => { + if (this.spilled && this.fileStream) { + await new Promise((resolve, reject) => { + this.fileStream!.end((err: Error) => (err ? reject(err) : resolve())) + }) + await finished(this.fileStream).catch(() => {}) + } + if (!this.finishedFlag) { + this.digestValue = this.hash.digest('hex') + this.finishedFlag = true + } + } + finalize().then(() => cb(), cb) + } + + // Public API + + digestHex(): string { + if (!this.finishedFlag || !this.digestValue) { + throw new Error('digestHex() called before stream finished') + } + return this.digestValue + } + + size(): number { + return this.totalBytes + } + + toReadable(opts: ToReadableOptions = {}): Readable { + const { autoCleanup = false } = opts + + if (this.spilled) { + if (!this.filePath) throw new Error('Internal error: spilled but no filePath') + + const rs = fs.createReadStream(this.filePath) + this.activeReaders++ + + const done = () => { + rs.removeListener('close', done) + rs.removeListener('end', done) + rs.removeListener('error', done) + + this.activeReaders = Math.max(0, this.activeReaders - 1) + + if (autoCleanup && this.activeReaders === 0) { + this.cleanupPending = true + void this.maybeCleanupSpill() + } + } + + rs.once('close', done) + rs.once('end', done) + rs.once('error', done) + + return rs + } + + // In-memory: nothing to clean up + const snapshot = this.chunks ?? [] + return Readable.from(snapshot) + } + + /** Explicit cleanup (deferred if readers are still active). */ + async cleanup(): Promise { + this.cleanupPending = true + await this.maybeCleanupSpill() + } + + // Internals + + private async ensureFile(): Promise { + if (this.ensureFilePromise) return this.ensureFilePromise + + this.ensureFilePromise = (async () => { + this.tmpDir = await fsp.mkdtemp(path.join(this.tmpRoot, 'hashspill-')) // unique directory + const name = `${Date.now()}-${randomUUID()}.bin` // unique filename + this.filePath = path.join(this.tmpDir, name) + this.fileStream = fs.createWriteStream(this.filePath, { flags: 'wx' }) // fail if exists + })() + + return this.ensureFilePromise + } + + private writeToFile(buf: Buffer): Promise { + return new Promise(async (resolve, reject) => { + try { + await this.ensureFile() + const ok = this.fileStream!.write(buf) + if (ok) return resolve() + this.fileStream!.once('drain', resolve) + } catch (e) { + reject(e) + } + }) + } + + private async spillToDiskAndWrite(nextChunk: Buffer): Promise { + await this.ensureFile() + + const prefix = Buffer.concat(this.chunks!, this.memSize) + // Free memory + this.chunks = null + this.memSize = 0 + + await this.writeToFile(prefix) + await this.writeToFile(nextChunk) + } + + private async maybeCleanupSpill(): Promise { + if (this.cleanupRunning) return this.cleanupRunning + + this.cleanupRunning = (async () => { + try { + if (!this.spilled) return + if (!this.cleanupPending) return + if (this.activeReaders > 0) return + + // Ensure file stream is closed + try { + if (this.fileStream && !this.fileStream.destroyed) { + this.fileStream.destroy() + } + } catch {} + + // Remove file and directory (best-effort) + try { + if (this.filePath) await fsp.rm(this.filePath, { force: true }) + } catch {} + try { + if (this.tmpDir) await fsp.rm(this.tmpDir, { force: true, recursive: true }) + } catch {} + + // Null out for GC + this.filePath = null + this.tmpDir = null + this.fileStream = null + } finally { + this.cleanupRunning = null + } + })() + + return this.cleanupRunning + } + + private async cleanupAsync(): Promise { + this.cleanupPending = true + await this.maybeCleanupSpill() + } +} diff --git a/src/internal/streams/index.ts b/src/internal/streams/index.ts index 4fc27fe17..dcff245a4 100644 --- a/src/internal/streams/index.ts +++ b/src/internal/streams/index.ts @@ -1,3 +1,4 @@ export * from './stream-speed' export * from './byte-counter' +export * from './hash-stream' export * from './monitor' diff --git a/src/internal/streams/types.d.ts b/src/internal/streams/types.d.ts new file mode 100644 index 000000000..95f61a3d0 --- /dev/null +++ b/src/internal/streams/types.d.ts @@ -0,0 +1,3 @@ +declare module 'stream' { + export function compose(s1: A, s2: B): B & A +} diff --git a/src/storage/database/adapter.ts b/src/storage/database/adapter.ts index e54f0b627..8d3f5c75b 100644 --- a/src/storage/database/adapter.ts +++ b/src/storage/database/adapter.ts @@ -77,7 +77,7 @@ export interface Database { > ): Promise> - createIcebergBucket(data: Pick): Promise + createIcebergBucket(data: Pick): Promise findBucketById( bucketId: string, @@ -219,4 +219,9 @@ export interface Database { ): Promise deleteAnalyticsBucket(id: string): Promise + + listIcebergBuckets( + columns: string, + options: ListBucketOptions | undefined + ): Promise } diff --git a/src/storage/database/knex.ts b/src/storage/database/knex.ts index c9d8912a4..50b1d27d7 100644 --- a/src/storage/database/knex.ts +++ b/src/storage/database/knex.ts @@ -111,7 +111,38 @@ export class StorageKnexDB implements Database { }) } - createIcebergBucket(data: Pick): Promise { + listIcebergBuckets( + columns: string, + options: ListBucketOptions | undefined + ): Promise { + return this.runQuery('ListIcebergBuckets', async (knex) => { + const query = knex + .from('buckets_analytics') + .select(columns.split(',').map((c) => c.trim())) + + if (options?.search !== undefined && options.search.length > 0) { + query.where('id', 'like', `%${options.search}%`) + } + + if (options?.sortColumn !== undefined) { + query.orderBy(options.sortColumn, options.sortOrder || 'asc') + } else { + query.orderBy('id', 'asc') + } + + if (options?.limit !== undefined) { + query.limit(options.limit) + } + + if (options?.offset !== undefined) { + query.offset(options.offset) + } + + return query + }) + } + + createIcebergBucket(data: Pick): Promise { const bucketData: IcebergCatalog = { id: data.id, } diff --git a/src/storage/events/index.ts b/src/storage/events/index.ts index dac72a76e..b5a872298 100644 --- a/src/storage/events/index.ts +++ b/src/storage/events/index.ts @@ -13,4 +13,5 @@ export * from './migrations/reset-migrations' export * from './jwks/jwks-create-signing-secret' export * from './pgboss/upgrade-v10' export * from './pgboss/move-jobs' +export * from './vectors/reconcile' export * from './workers' diff --git a/src/storage/events/vectors/reconcile.ts b/src/storage/events/vectors/reconcile.ts new file mode 100644 index 000000000..a416aaea7 --- /dev/null +++ b/src/storage/events/vectors/reconcile.ts @@ -0,0 +1,46 @@ +import { BaseEvent } from '../base-event' +import { getTenantConfig } from '@internal/database' +import { JobWithMetadata, Queue, SendOptions, WorkOptions } from 'pg-boss' +import { BasePayload } from '@internal/queue' +import { logger, logSchema } from '@internal/monitoring' + +interface ResetMigrationsPayload extends BasePayload { + tenantId: string +} + +export class VectorReconcile extends BaseEvent { + static queueName = 'vector-reconcile' + + static getQueueOptions(): Queue { + return { + name: this.queueName, + policy: 'exactly_once', + } as const + } + + static getWorkerOptions(): WorkOptions { + return { + includeMetadata: true, + } + } + + static getSendOptions(payload: ResetMigrationsPayload): SendOptions { + return { + expireInHours: 2, + singletonKey: payload.tenantId, + retryLimit: 3, + retryDelay: 5, + priority: 10, + } + } + + static async handle(job: JobWithMetadata) { + const tenantId = job.data.tenant.ref + const tenant = await getTenantConfig(tenantId) + + logSchema.info(logger, `[Migrations] resetting migrations for ${tenantId}`, { + type: 'migrations', + project: tenantId, + }) + } +} diff --git a/src/storage/protocols/iceberg/catalog/tenant-catalog.ts b/src/storage/protocols/iceberg/catalog/tenant-catalog.ts index fd9ae46d9..4c5b2d483 100644 --- a/src/storage/protocols/iceberg/catalog/tenant-catalog.ts +++ b/src/storage/protocols/iceberg/catalog/tenant-catalog.ts @@ -156,6 +156,7 @@ export class TenantAwareRestCatalog extends RestCatalogClient { ...rest, namespace: namespaceName, }) + await store.createTable({ name: rest.name, bucketId: catalog.id, diff --git a/src/storage/protocols/s3/signature-v4.ts b/src/storage/protocols/s3/signature-v4.ts index 9e6d5396a..6753b4e93 100644 --- a/src/storage/protocols/s3/signature-v4.ts +++ b/src/storage/protocols/s3/signature-v4.ts @@ -1,9 +1,19 @@ import crypto from 'crypto' import { ERRORS } from '@internal/errors' +import { Readable } from 'stream' +import { createHash } from 'node:crypto' +import { pipeline } from 'stream/promises' +import { Writable } from 'node:stream' + +export enum SignatureV4Service { + S3 = 's3', + S3VECTORS = 's3vectors', +} interface SignatureV4Options { enforceRegion: boolean allowForwardedHeader?: boolean + allowBodyHashing?: boolean nonCanonicalForwardedHost?: string credentials: Omit & { secretKey: string } } @@ -23,11 +33,12 @@ export interface ClientSignature { interface SignatureRequest { url: string - body?: string | ReadableStream | Buffer + body?: string | ReadableStream | Buffer | Readable headers: Record method: string query?: Record prefix?: string + payloadHasher?: Writable & { digestHex: () => string } } interface Credentials { @@ -86,12 +97,14 @@ export class SignatureV4 { public readonly serverCredentials: SignatureV4Options['credentials'] enforceRegion: boolean allowForwardedHeader?: boolean + allowBodyHashing?: boolean nonCanonicalForwardedHost?: string constructor(options: SignatureV4Options) { this.serverCredentials = options.credentials this.enforceRegion = options.enforceRegion this.allowForwardedHeader = options.allowForwardedHeader + this.allowBodyHashing = options.allowBodyHashing this.nonCanonicalForwardedHost = options.nonCanonicalForwardedHost } @@ -252,12 +265,12 @@ export class SignatureV4 { * @param clientSignature * @param request */ - verify(clientSignature: ClientSignature, request: SignatureRequest) { + async verify(clientSignature: ClientSignature, request: SignatureRequest) { if (typeof clientSignature.policy?.raw === 'string') { return this.verifyPostPolicySignature(clientSignature, clientSignature.policy.raw) } - const serverSignature = this.sign(clientSignature, request) + const serverSignature = await this.sign(clientSignature, request) return crypto.timingSafeEqual( Buffer.from(clientSignature.signature), Buffer.from(serverSignature.signature) @@ -333,7 +346,7 @@ export class SignatureV4 { * @param clientSignature * @param request */ - sign(clientSignature: ClientSignature, request: SignatureRequest) { + async sign(clientSignature: ClientSignature, request: SignatureRequest) { const serverCredentials = this.serverCredentials this.validateCredentials(clientSignature.credentials) @@ -344,11 +357,12 @@ export class SignatureV4 { } const selectedRegion = this.getSelectedRegion(clientSignature.credentials.region) - const canonicalRequest = this.constructCanonicalRequest( + const canonicalRequest = await this.constructCanonicalRequest( clientSignature, request, clientSignature.signedHeaders ) + const stringToSign = this.constructStringToSign( longDate, clientSignature.credentials.shortDate, @@ -356,6 +370,7 @@ export class SignatureV4 { serverCredentials.service, canonicalRequest ) + const signingKey = this.signingKey( serverCredentials.secretKey, clientSignature.credentials.shortDate, @@ -366,7 +381,7 @@ export class SignatureV4 { return { signature: this.hmac(signingKey, stringToSign).toString('hex'), canonicalRequest } } - protected getPayloadHash(clientSignature: ClientSignature, request: SignatureRequest) { + protected async getPayloadHash(clientSignature: ClientSignature, request: SignatureRequest) { const body = request.body // For presigned URLs and GET requests, use UNSIGNED-PAYLOAD @@ -392,11 +407,18 @@ export class SignatureV4 { .digest('hex') } + // If body is a ReadableStream, calculate the SHA256 hash of the stream + if (body instanceof Readable && this.allowBodyHashing && request.payloadHasher) { + return await pipeline(body, request.payloadHasher).then(() => { + return request.payloadHasher?.digestHex() + }) + } + // Default to UNSIGNED-PAYLOAD if body is not a string or ArrayBuffer return 'UNSIGNED-PAYLOAD' } - protected constructCanonicalRequest( + protected async constructCanonicalRequest( clientSignature: ClientSignature, request: SignatureRequest, signedHeaders: string[] @@ -407,7 +429,7 @@ export class SignatureV4 { const canonicalQueryString = this.constructCanonicalQueryString(request.query || {}) const canonicalHeaders = this.constructCanonicalHeaders(request, signedHeaders) const signedHeadersString = signedHeaders.sort().join(';') - const payloadHash = this.getPayloadHash(clientSignature, request) + const payloadHash = await this.getPayloadHash(clientSignature, request) return `${method}\n${canonicalUri}\n${canonicalQueryString}\n${canonicalHeaders}\n${signedHeadersString}\n${payloadHash}` } @@ -543,6 +565,14 @@ export class SignatureV4 { return this.hmac(kService, 'aws4_request') } + protected async sha256OfRequest(req: Readable) { + const hash = createHash('sha256') + for await (const chunk of req) { + hash.update(chunk) + } + return hash.digest('hex') + } + protected hmac(key: string | Buffer, data: string): Buffer { return crypto.createHmac('sha256', key).update(data).digest() } diff --git a/src/storage/protocols/vector/adapter/s3-vector.ts b/src/storage/protocols/vector/adapter/s3-vector.ts new file mode 100644 index 000000000..b73b46e42 --- /dev/null +++ b/src/storage/protocols/vector/adapter/s3-vector.ts @@ -0,0 +1,86 @@ +import { + CreateIndexCommand, + CreateIndexCommandInput, + CreateIndexCommandOutput, + DeleteIndexCommand, + DeleteIndexCommandInput, + DeleteIndexCommandOutput, + DeleteVectorsCommand, + DeleteVectorsInput, + DeleteVectorsOutput, + GetVectorsCommand, + GetVectorsCommandInput, + GetVectorsCommandOutput, + ListVectorsCommand, + ListVectorsInput, + ListVectorsOutput, + PutVectorsCommand, + PutVectorsInput, + PutVectorsOutput, + QueryVectorsCommand, + QueryVectorsInput, + QueryVectorsOutput, + S3VectorsClient, +} from '@aws-sdk/client-s3vectors' +import { getConfig } from '../../../../config' + +export interface VectorStore { + createVectorIndex(command: CreateIndexCommandInput): Promise + deleteVectorIndex(param: DeleteIndexCommandInput): Promise + putVectors(command: PutVectorsInput): Promise + listVectors(command: ListVectorsInput): Promise + + queryVectors(queryInput: QueryVectorsInput): Promise + + deleteVectors(deleteVectorsInput: DeleteVectorsInput): Promise + + getVectors(getVectorsInput: GetVectorsCommandInput): Promise +} + +const { storageS3Region, vectorBucketRegion } = getConfig() + +export function createS3VectorClient() { + const s3VectorClient = new S3VectorsClient({ + region: vectorBucketRegion || storageS3Region, + }) + + return new S3VectorsClient(s3VectorClient) +} + +export class S3Vector implements VectorStore { + constructor(protected readonly s3VectorClient: S3VectorsClient) {} + + getVectors(getVectorsInput: GetVectorsCommandInput): Promise { + return this.s3VectorClient.send(new GetVectorsCommand(getVectorsInput)) + } + + deleteVectors(deleteVectorsInput: DeleteVectorsInput): Promise { + return this.s3VectorClient.send(new DeleteVectorsCommand(deleteVectorsInput)) + } + + queryVectors(queryInput: QueryVectorsInput): Promise { + return this.s3VectorClient.send(new QueryVectorsCommand(queryInput)) + } + + async listVectors(command: ListVectorsInput): Promise { + return this.s3VectorClient.send(new ListVectorsCommand(command)) + } + + putVectors(command: PutVectorsInput): Promise { + const input = new PutVectorsCommand(command) + + return this.s3VectorClient.send(input) + } + + deleteVectorIndex(param: DeleteIndexCommandInput): Promise { + const command = new DeleteIndexCommand(param) + + return this.s3VectorClient.send(command) + } + + createVectorIndex(command: CreateIndexCommandInput): Promise { + const createIndexCommand = new CreateIndexCommand(command) + + return this.s3VectorClient.send(createIndexCommand) + } +} diff --git a/src/storage/protocols/vector/index.ts b/src/storage/protocols/vector/index.ts new file mode 100644 index 000000000..b872215b8 --- /dev/null +++ b/src/storage/protocols/vector/index.ts @@ -0,0 +1,3 @@ +export * from './vector-store' +export * from './adapter/s3-vector' +export * from './knex' diff --git a/src/storage/protocols/vector/knex.ts b/src/storage/protocols/vector/knex.ts new file mode 100644 index 000000000..e3a33cd3c --- /dev/null +++ b/src/storage/protocols/vector/knex.ts @@ -0,0 +1,250 @@ +import { Knex } from 'knex' +import { VectorIndex } from '@storage/schemas/vector' +import { ERRORS } from '@internal/errors' +import { VectorBucket } from '@storage/schemas' +import { ListVectorBucketsInput } from '@aws-sdk/client-s3vectors' +import { DatabaseError } from 'pg' +import { wait } from '@internal/concurrency' + +type DBVectorIndex = VectorIndex & { id: string; created_at: Date; updated_at: Date } + +interface CreateVectorIndexParams { + dataType: string + dimension: number + distanceMetric: string + indexName: string + metadataConfiguration?: { + nonFilterableMetadataKeys?: string[] + } + vectorBucketName: string +} + +export interface ListIndexesInput { + bucketId: string + maxResults?: number + nextToken?: string | undefined + prefix?: string | undefined +} + +export interface ListIndexesResult { + indexes: Pick[] + nextToken?: string +} + +export interface ListBucketResult { + vectorBuckets: VectorBucket[] + nextToken?: string +} + +export interface VectorMetadataDB { + withTransaction( + fn: (db: KnexVectorMetadataDB) => T, + config?: Knex.TransactionConfig + ): Promise + + findVectorBucket(vectorBucketName: string): Promise + createVectorBucket(bucketName: string): Promise + deleteVectorBucket(bucketName: string, vectorIndexName: string): Promise + listBuckets(param: ListVectorBucketsInput): Promise + countBuckets(): Promise + + countIndexes(bucketId: string): Promise + createVectorIndex(data: CreateVectorIndexParams): Promise + getIndex(bucketId: string, name: string): Promise + listIndexes(command: ListIndexesInput): Promise + deleteVectorIndex(bucketName: string, vectorIndexName: string): Promise + findVectorIndexForBucket(vectorBucketName: string, indexName: string): Promise +} + +export class KnexVectorMetadataDB implements VectorMetadataDB { + constructor(protected readonly knex: Knex) {} + + async countIndexes(bucketId: string): Promise { + const row = await this.knex + .withSchema('storage') + .table('vector_indexes') + .where({ bucket_id: bucketId }) + .count<{ count: string }>('id as count') + .first() + return Number(row?.count ?? 0) + } + + async countBuckets(): Promise { + const row = await this.knex + .withSchema('storage') + .table('buckets_vectors') + .count<{ count: string }>('id as count') + .first() + + return Number(row?.count ?? 0) + } + + async listBuckets(param: ListVectorBucketsInput): Promise { + const query = this.knex.withSchema('storage').table('buckets_vectors') + if (param.prefix) { + query.where('id', 'like', `${param.prefix}%`) + } + + if (param.nextToken) { + query.andWhere('id', '>', param.nextToken) + } + const maxResults = param.maxResults ? Math.min(param.maxResults, 500) : 500 + + const result = await query.orderBy('id', 'asc').limit(maxResults + 1) + + const hasMore = result.length > maxResults + + const buckets = result.slice(0, maxResults) + + return { + vectorBuckets: buckets, + nextToken: hasMore ? buckets[buckets.length - 1].id : undefined, + } + } + + async findVectorIndexForBucket( + vectorBucketName: string, + indexName: string + ): Promise { + const index = await this.knex + .withSchema('storage') + .select('*') + .table('vector_indexes') + .where({ bucket_id: vectorBucketName, name: indexName }) + .first() + + if (!index) { + throw ERRORS.S3VectorNotFoundException('vector index', indexName) + } + return index + } + + async findVectorBucket(vectorBucketName: string): Promise { + const bucket = await this.knex + .withSchema('storage') + .table('buckets_vectors') + .where({ id: vectorBucketName }) + .first() + + if (!bucket) { + throw ERRORS.S3VectorNotFoundException('vector bucket', vectorBucketName) + } + + return bucket + } + + async createVectorBucket(bucketName: string): Promise { + try { + await this.knex.withSchema('storage').table('buckets_vectors').insert({ + id: bucketName, + }) + } catch (e) { + if (e instanceof Error && e instanceof DatabaseError) { + if (e.code === '23505') { + throw ERRORS.S3VectorConflictException('vector bucket', bucketName) + } + } + + throw e + } + } + + async listIndexes(command: ListIndexesInput): Promise { + const maxResults = command.maxResults ? Math.min(command.maxResults, 500) : 500 + + const query = this.knex + .withSchema('storage') + .select('name', 'bucket_id', 'created_at') + .from('vector_indexes') + .where({ bucket_id: command.bucketId }) + .orderBy('name', 'asc') + .table('vector_indexes') + + if (command.prefix) { + query.andWhere('name', 'like', `${command.prefix}%`) + } + + if (command.nextToken) { + query.andWhere('id', '>', command.nextToken) + } + + const result = await query.limit(maxResults + 1) + const hasMore = result.length > maxResults + + const indexes = result.slice(0, maxResults) + + return { + indexes, + nextToken: hasMore ? indexes[indexes.length - 1].name : undefined, + } + } + + async getIndex(bucketId: string, name: string): Promise { + const index = await this.knex + .withSchema('storage') + .select('*') + .table('vector_indexes') + .where({ bucket_id: bucketId, name: name }) + .first() + + if (!index) { + throw ERRORS.S3VectorNotFoundException('vector index', name) + } + return index + } + + createVectorIndex(data: CreateVectorIndexParams) { + return this.knex + .withSchema('storage') + .table('vector_indexes') + .insert({ + bucket_id: data.vectorBucketName, + data_type: data.dataType, + name: data.indexName, + dimension: data.dimension, + distance_metric: data.distanceMetric, + metadata_configuration: data.metadataConfiguration, + }) + } + + async withTransaction(fn: (db: KnexVectorMetadataDB) => T): Promise { + const maxRetries = 3 + let attempt = 0 + let lastError: Error | undefined = undefined + + while (attempt < maxRetries) { + try { + return await this.knex.transaction(async (trx) => { + const trxDb = new KnexVectorMetadataDB(trx) + return fn(trxDb) + }) + } catch (error) { + attempt++ + + // Check if it's a serialization error (PostgreSQL error code 40001) + if (error instanceof DatabaseError && error.code === '40001' && attempt < maxRetries) { + // Exponential backoff: 20ms, 40ms, 80ms + await wait(20 * Math.pow(2, attempt - 1)) + lastError = error + continue + } + + throw error + } + } + + throw ERRORS.TransactionError('Transaction failed after maximum retries', lastError) + } + + deleteVectorIndex(bucketName: string, vectorIndexName: string): Promise { + return this.knex + .withSchema('storage') + .table('vector_indexes') + .where({ bucket_id: bucketName, name: vectorIndexName }) + .del() + } + + async deleteVectorBucket(bucketName: string) { + await this.knex.withSchema('storage').table('buckets_vectors').where({ id: bucketName }).del() + } +} diff --git a/src/storage/protocols/vector/vector-store.ts b/src/storage/protocols/vector/vector-store.ts new file mode 100644 index 000000000..f35a6589a --- /dev/null +++ b/src/storage/protocols/vector/vector-store.ts @@ -0,0 +1,339 @@ +import { + CreateIndexInput, + DeleteIndexInput, + DistanceMetric, + GetIndexCommandInput, + ListIndexesInput, + MetadataConfiguration, + GetIndexOutput, + PutVectorsInput, + ListVectorsInput, + ListVectorBucketsInput, + QueryVectorsInput, + DeleteVectorsInput, + GetVectorBucketInput, + GetVectorsCommandInput, + ConflictException, +} from '@aws-sdk/client-s3vectors' +import { VectorMetadataDB } from './knex' +import { VectorStore } from './adapter/s3-vector' +import { ERRORS } from '@internal/errors' + +interface VectorStoreConfig { + tenantId: string + vectorBucketName: string + maxBucketCount: number + maxIndexCount: number +} + +export class VectorStoreManager { + constructor( + protected readonly vectorStore: VectorStore, + protected readonly db: VectorMetadataDB, + protected readonly config: VectorStoreConfig + ) {} + + protected getIndexName(name: string) { + return `${this.config.tenantId}-${name}` + } + + async createBucket(bucketName: string): Promise { + await this.db.withTransaction( + async (tnx) => { + const bucketCount = await tnx.countBuckets() + if (bucketCount >= this.config.maxBucketCount) { + throw ERRORS.S3VectorMaxBucketsExceeded(this.config.maxBucketCount) + } + + try { + await tnx.createVectorBucket(bucketName) + } catch (e) { + if (e instanceof ConflictException) { + return + } + throw e + } + }, + { isolationLevel: 'serializable' } + ) + } + + async deleteBucket(bucketName: string): Promise { + await this.db.withTransaction( + async (tx) => { + const indexes = await tx.listIndexes({ bucketId: bucketName, maxResults: 1 }) + + if (indexes.indexes.length > 0) { + throw ERRORS.S3VectorBucketNotEmpty(bucketName) + } + + await tx.deleteVectorBucket(bucketName) + }, + { isolationLevel: 'serializable' } + ) + } + + async getBucket(command: GetVectorBucketInput) { + if (!command.vectorBucketName) { + throw ERRORS.MissingParameter('vectorBucketName') + } + + const vectorBucket = await this.db.findVectorBucket(command.vectorBucketName) + + return { + vectorBucket: { + vectorBucketName: vectorBucket.id, + creationTime: vectorBucket.created_at + ? Math.floor(vectorBucket.created_at.getTime() / 1000) + : undefined, + }, + } + } + + async listBuckets(command: ListVectorBucketsInput) { + const bucketResult = await this.db.listBuckets({ + maxResults: command.maxResults, + nextToken: command.nextToken, + prefix: command.prefix, + }) + + return { + vectorBuckets: bucketResult.vectorBuckets.map((bucket) => ({ + vectorBucketName: bucket.id, + creationTime: bucket.created_at + ? Math.floor(bucket.created_at.getTime() / 1000) + : undefined, + })), + nextToken: bucketResult.nextToken, + } + } + + // Store it in MultiTenantDB + // Queue Job in the same transaction + // Poll for job completion + async createVectorIndex(command: CreateIndexInput): Promise { + if (!command.indexName) { + throw ERRORS.MissingParameter('indexName') + } + + if (!command.vectorBucketName) { + throw ERRORS.MissingParameter('vectorBucketName') + } + + await this.db.findVectorBucket(command.vectorBucketName) + + const createIndexInput = { + ...command, + indexName: this.getIndexName(command.indexName), + } + + await this.db.withTransaction( + async (tx) => { + const indexCount = await tx.countIndexes(command.vectorBucketName!) + + if (indexCount >= this.config.maxIndexCount) { + throw ERRORS.S3VectorMaxIndexesExceeded(this.config.maxIndexCount) + } + + await tx.createVectorIndex({ + dataType: createIndexInput.dataType!, + dimension: createIndexInput.dimension!, + distanceMetric: createIndexInput.distanceMetric!, + indexName: command.indexName!, + metadataConfiguration: createIndexInput.metadataConfiguration, + vectorBucketName: command.vectorBucketName!, + }) + + try { + await this.vectorStore.createVectorIndex({ + ...createIndexInput, + vectorBucketName: this.config.vectorBucketName, + }) + } catch (e) { + if (e instanceof ConflictException) { + return + } + + throw e + } + }, + { isolationLevel: 'serializable' } + ) + } + + async deleteIndex(command: DeleteIndexInput): Promise { + if (!command.indexName) { + throw ERRORS.MissingParameter('indexName') + } + + if (!command.vectorBucketName) { + throw ERRORS.MissingParameter('vectorBucketName') + } + + await this.db.findVectorIndexForBucket(command.vectorBucketName, command.indexName) + + const vectorIndexName = this.getIndexName(command.indexName) + + await this.db.withTransaction(async (tx) => { + await tx.deleteVectorIndex(command.vectorBucketName!, command.indexName!) + + await this.vectorStore.deleteVectorIndex({ + vectorBucketName: this.config.vectorBucketName, + indexName: vectorIndexName, + }) + }) + } + + async getIndex(command: GetIndexCommandInput): Promise { + if (!command.indexName) { + throw ERRORS.MissingParameter('indexName') + } + + if (!command.vectorBucketName) { + throw ERRORS.MissingParameter('vectorBucketName') + } + + const index = await this.db.getIndex(command.vectorBucketName, command.indexName) + + return { + index: { + indexName: index.name, + dataType: index.data_type as 'float32', + dimension: index.dimension, + distanceMetric: index.distance_metric as DistanceMetric, + metadataConfiguration: index.metadata_configuration as MetadataConfiguration, + vectorBucketName: index.bucket_id, + creationTime: index.created_at, + indexArn: undefined, + }, + } + } + + async listIndexes(command: ListIndexesInput) { + if (!command.vectorBucketName) { + throw ERRORS.MissingParameter('vectorBucketName') + } + + const result = await this.db.listIndexes({ + bucketId: command.vectorBucketName, + maxResults: command.maxResults, + nextToken: command.nextToken, + prefix: command.prefix, + }) + + return { + indexes: result.indexes.map((i) => ({ + indexName: i.name, + vectorBucketName: i.bucket_id, + creationTime: Math.floor(i.created_at.getTime() / 1000), + })), + } + } + + async putVectors(command: PutVectorsInput) { + if (!command.indexName) { + throw ERRORS.MissingParameter('indexName') + } + + if (!command.vectorBucketName) { + throw ERRORS.MissingParameter('vectorBucketName') + } + + await this.db.findVectorIndexForBucket(command.vectorBucketName, command.indexName) + + const putVectorsInput = { + ...command, + vectorBucketName: this.config.vectorBucketName, + indexName: this.getIndexName(command.indexName), + } + await this.vectorStore.putVectors(putVectorsInput) + } + + async deleteVectors(command: DeleteVectorsInput) { + if (!command.indexName) { + throw ERRORS.MissingParameter('indexName') + } + + if (!command.vectorBucketName) { + throw ERRORS.MissingParameter('vectorBucketName') + } + + await this.db.findVectorIndexForBucket(command.vectorBucketName, command.indexName) + + const deleteVectorsInput = { + ...command, + vectorBucketName: this.config.vectorBucketName, + indexName: this.getIndexName(command.indexName), + } + + return this.vectorStore.deleteVectors(deleteVectorsInput) + } + + async listVectors(command: ListVectorsInput) { + if (!command.indexName) { + throw ERRORS.MissingParameter('indexName') + } + + if (!command.vectorBucketName) { + throw ERRORS.MissingParameter('vectorBucketName') + } + + await this.db.findVectorIndexForBucket(command.vectorBucketName, command.indexName) + + const listVectorsInput = { + ...command, + vectorBucketName: this.config.vectorBucketName, + indexName: this.getIndexName(command.indexName), + } + + const result = await this.vectorStore.listVectors(listVectorsInput) + + return { + vectors: result.vectors, + nextToken: result.nextToken, + } + } + + async queryVectors(command: QueryVectorsInput) { + if (!command.indexName) { + throw ERRORS.MissingParameter('indexName') + } + + if (!command.vectorBucketName) { + throw ERRORS.MissingParameter('vectorBucketName') + } + + await this.db.findVectorIndexForBucket(command.vectorBucketName, command.indexName) + + const queryInput = { + ...command, + vectorBucketName: this.config.vectorBucketName, + indexName: this.getIndexName(command.indexName), + } + return this.vectorStore.queryVectors(queryInput) + } + + async getVectors(command: GetVectorsCommandInput) { + if (!command.indexName) { + throw ERRORS.MissingParameter('indexName') + } + + if (!command.vectorBucketName) { + throw ERRORS.MissingParameter('vectorBucketName') + } + + await this.db.findVectorIndexForBucket(command.vectorBucketName, command.indexName) + + const getVectorsInput = { + ...command, + vectorBucketName: this.config.vectorBucketName, + indexName: this.getIndexName(command.indexName), + } + + const result = await this.vectorStore.getVectors(getVectorsInput) + + return { + vectors: result.vectors, + } + } +} diff --git a/src/storage/schemas/bucket.ts b/src/storage/schemas/bucket.ts index 08d16cf70..9ffd58d93 100644 --- a/src/storage/schemas/bucket.ts +++ b/src/storage/schemas/bucket.ts @@ -33,3 +33,4 @@ export const bucketSchema = { export type Bucket = FromSchema export type IcebergCatalog = Pick +export type VectorBucket = Pick & { created_at: Date } diff --git a/src/storage/schemas/vector.ts b/src/storage/schemas/vector.ts new file mode 100644 index 000000000..aaec24de3 --- /dev/null +++ b/src/storage/schemas/vector.ts @@ -0,0 +1,26 @@ +import { FromSchema } from 'json-schema-to-ts' + +const vectorIndex = { + type: 'object', + properties: { + name: { type: 'string' }, + data_type: { type: 'string' }, + dimension: { type: 'number' }, + distance_metric: { type: 'string' }, + status: { type: 'string' }, + metadata_configuration: { + type: 'object', + properties: { + nonFilterableMetadataKeys: { + type: 'array', + items: { type: 'string' }, + }, + }, + }, + bucket_id: { type: 'string' }, + }, + required: ['name', 'dimension', 'distance_metric', 'bucket_id'], + additionalProperties: false, +} as const + +export type VectorIndex = FromSchema diff --git a/src/storage/storage.ts b/src/storage/storage.ts index 15aa5c4ec..e2b5fcad2 100644 --- a/src/storage/storage.ts +++ b/src/storage/storage.ts @@ -89,6 +89,10 @@ export class Storage { return this.db.listBuckets(columns, options) } + listIcebergBuckets(columns = 'id', options?: ListBucketOptions) { + return this.db.listIcebergBuckets(columns, options) + } + /** * Creates a bucket * @param data diff --git a/src/test/hash-stream.test.ts b/src/test/hash-stream.test.ts new file mode 100644 index 000000000..a10737d4a --- /dev/null +++ b/src/test/hash-stream.test.ts @@ -0,0 +1,598 @@ +import { Readable, Writable } from 'node:stream' +import { pipeline } from 'node:stream/promises' +import * as fsp from 'node:fs/promises' +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import { createHash } from 'node:crypto' +import { HashSpillWritable } from '@internal/streams/hash-stream' + +function randBuf(size: number): Buffer { + const b = Buffer.allocUnsafe(size) + for (let i = 0; i < size; i++) b[i] = (i * 131 + 17) & 0xff // deterministic-ish + return b +} + +function readableFrom(...chunks: Buffer[]): Readable { + return Readable.from(chunks) +} + +async function dirEntries(p: string): Promise { + try { + const names = await fsp.readdir(p) + return names + } catch { + return [] + } +} + +async function countHashspillDirs(root: string): Promise { + const names = await dirEntries(root) + return names.filter((n) => n.startsWith('hashspill-')).length +} + +async function findSpillFilePath(root: string): Promise { + try { + const entries = await fsp.readdir(root, { withFileTypes: true }) + const dir = entries.find((e) => e.isDirectory() && e.name.startsWith('hashspill-')) + if (!dir) return null + const dirPath = path.join(root, dir.name) + const files = await fsp.readdir(dirPath) + if (files.length === 0) return null + return path.join(dirPath, files[0]) // our class writes a single file + } catch { + return null + } +} + +class SlowWritable extends Writable { + private delayMs: number + constructor(delayMs = 5) { + super({ highWaterMark: 16 * 1024 }) // small HWM to induce backpressure + this.delayMs = delayMs + } + _write(chunk: Buffer, _enc: BufferEncoding, cb: (e?: Error | null) => void) { + setTimeout(() => cb(), this.delayMs) + } +} + +describe('HashSpillWritable', () => { + let tmpRoot: string + + beforeEach(async () => { + tmpRoot = await fsp.mkdtemp(path.join(os.tmpdir(), 'hsw-tests-')) + }) + + afterEach(async () => { + // best-effort cleanup of left-overs + try { + await fsp.rm(tmpRoot, { recursive: true, force: true }) + } catch {} + }) + + test('in-memory: under limit stays in memory; digest & size are correct', async () => { + const limit = 1024 * 64 + const payload = randBuf(limit - 7) + const expectedDigest = createHash('sha256').update(payload).digest('hex') + + const sink = new HashSpillWritable({ limitInMemoryBytes: limit, tmpRoot }) + await pipeline(readableFrom(payload), sink) + + expect(sink.size()).toBe(payload.length) + expect(sink.digestHex()).toBe(expectedDigest) + + // toReadable returns in-memory stream + const collected: Buffer[] = [] + await pipeline( + sink.toReadable(), + new Writable({ + write(chunk, _enc, cb) { + collected.push(chunk as Buffer) + cb() + }, + }) + ) + expect(Buffer.concat(collected)).toEqual(payload) + + // No hashspill-* dirs should have been created + expect(await countHashspillDirs(tmpRoot)).toBe(0) + + // cleanup() should be a no-op + await expect(sink.cleanup()).resolves.toBeUndefined() + }) + + test('in-memory: exactly at limit does not spill', async () => { + const limit = 32 * 1024 + const payload = randBuf(limit) + const sink = new HashSpillWritable({ limitInMemoryBytes: limit, tmpRoot }) + + await pipeline(readableFrom(payload), sink) + expect(sink.size()).toBe(limit) + expect(await countHashspillDirs(tmpRoot)).toBe(0) + }) + + test('spill: just over limit triggers spill; autoCleanup on reader removes artifacts', async () => { + const limit = 1024 * 32 + const payload = randBuf(limit + 1) + const expectedDigest = createHash('sha256').update(payload).digest('hex') + + const sink = new HashSpillWritable({ limitInMemoryBytes: limit, tmpRoot }) + await pipeline(readableFrom(payload), sink) + + expect(sink.digestHex()).toBe(expectedDigest) + // A spill should have created exactly one temp dir + expect(await countHashspillDirs(tmpRoot)).toBe(1) + + // Read with autoCleanup so artifacts get removed when last reader ends + await pipeline( + sink.toReadable({ autoCleanup: true }), + new Writable({ + write(_c, _e, cb) { + cb() + }, + }) + ) + + // Allow event loop to process cleanup + await new Promise((r) => setTimeout(r, 10)) + + // The hashspill dir should be gone + expect(await countHashspillDirs(tmpRoot)).toBe(0) + }) + + test('spill: multiple readers, autoCleanup waits for the last reader', async () => { + const limit = 8 * 1024 + const payload = randBuf(limit * 3) // force spill + const sink = new HashSpillWritable({ limitInMemoryBytes: limit, tmpRoot }) + + await pipeline(readableFrom(payload), sink) + expect(await countHashspillDirs(tmpRoot)).toBe(1) + + const r1 = sink.toReadable({ autoCleanup: true }) + const r2 = sink.toReadable({ autoCleanup: true }) + + // pipe r1 quickly + const fastConsumer = new Writable({ + write(_c, _e, cb) { + cb() + }, + }) + // r2 is slower + const slowConsumer = new SlowWritable(3) + + const p1 = pipeline(r1, fastConsumer) + const p2 = pipeline(r2, slowConsumer) + await Promise.all([p1, p2]) + + // wait a tick for cleanup to run + await new Promise((r) => setTimeout(r, 10)) + + expect(await countHashspillDirs(tmpRoot)).toBe(0) + }) + + test('manual cleanup: delete after readers close (call cleanup after reading)', async () => { + const limit = 4096 + const payload = randBuf(limit * 5) + const sink = new HashSpillWritable({ limitInMemoryBytes: limit, tmpRoot }) + + await pipeline(readableFrom(payload), sink) + expect(await countHashspillDirs(tmpRoot)).toBe(1) + + // No autoCleanup; we clean manually after + const r = sink.toReadable() + await pipeline( + r, + new Writable({ + write(_c, _e, cb) { + cb() + }, + }) + ) + + // Now manual cleanup removes artifacts + await sink.cleanup() + expect(await countHashspillDirs(tmpRoot)).toBe(0) + }) + + test('backpressure respected with slow downstream while hashing', async () => { + const limit = 16 * 1024 + const pieces = Array.from({ length: 50 }, (_, i) => randBuf(2048 + (i % 5))) + const payload = Buffer.concat(pieces) + const expectedDigest = createHash('sha256').update(payload).digest('hex') + + const sink = new HashSpillWritable({ limitInMemoryBytes: limit, tmpRoot }) + + // Write into sink, then read out to a slow consumer to ensure stream semantics hold + await pipeline(Readable.from(pieces), sink) + const outPieces: Buffer[] = [] + await pipeline( + sink.toReadable(), + new SlowWritable(2).on('pipe', function () {}) + ) + + expect(sink.digestHex()).toBe(expectedDigest) + }) + + test('multiple concurrent instances (no collisions, all succeed)', async () => { + const N = 8 + const limit = 8 * 1024 + const jobs = Array.from({ length: N }, async (_, idx) => { + const buf = randBuf(limit + 1024 + idx) // force spill + const exp = createHash('sha256').update(buf).digest('hex') + + const sink = new HashSpillWritable({ limitInMemoryBytes: limit, tmpRoot }) + await pipeline(readableFrom(buf), sink) + + expect(sink.digestHex()).toBe(exp) + // Use autoCleanup to clean right after reading + await pipeline( + sink.toReadable({ autoCleanup: true }), + new Writable({ + write(_c, _e, cb) { + cb() + }, + }) + ) + }) + + await Promise.all(jobs) + + // Allow cleanup to finish + await new Promise((r) => setTimeout(r, 10)) + expect(await countHashspillDirs(tmpRoot)).toBe(0) + }) + + test('size() tracks total bytes written', async () => { + const limit = 10 * 1024 + const parts = [randBuf(1111), randBuf(2222), randBuf(3333)] + const total = parts.reduce((n, b) => n + b.length, 0) + const sink = new HashSpillWritable({ limitInMemoryBytes: limit, tmpRoot }) + + await pipeline(Readable.from(parts), sink) + expect(sink.size()).toBe(total) + }) + + test('toReadable() can be called multiple times (consistent replay)', async () => { + const limit = 4096 + const payload = randBuf(limit * 2) // spill + const sink = new HashSpillWritable({ limitInMemoryBytes: limit, tmpRoot }) + await pipeline(readableFrom(payload), sink) + + const readAll = async () => { + const chunks: Buffer[] = [] + await pipeline( + sink.toReadable(), + new Writable({ + write(c, _e, cb) { + chunks.push(c as Buffer) + cb() + }, + }) + ) + return Buffer.concat(chunks) + } + + const a = await readAll() + const b = await readAll() + expect(a).toEqual(payload) + expect(b).toEqual(payload) + + await sink.cleanup() + }) + + test('cleanup is a no-op for non-spilled streams', async () => { + const limit = 1 << 20 + const payload = randBuf(12345) // under limit + const sink = new HashSpillWritable({ limitInMemoryBytes: limit, tmpRoot }) + await pipeline(readableFrom(payload), sink) + await expect(sink.cleanup()).resolves.toBeUndefined() + // Nothing created on disk + expect(await countHashspillDirs(tmpRoot)).toBe(0) + }) + + test('errors if digestHex() is called before finish', async () => { + const limit = 1024 + const sink = new HashSpillWritable({ limitInMemoryBytes: limit, tmpRoot }) + + // start write but don't finish + const r = new Readable({ + read() { + this.push(randBuf(200)) + this.push(null) + }, + }) + await pipeline(r, sink) + // now finished — valid + expect(() => sink.digestHex()).not.toThrow() + }) + + test('spill: if file cannot be created/written, pipeline rejects with a handled error', async () => { + const limit = 8 * 1024 + const payload = randBuf(limit * 3) // force spill + const sink = new HashSpillWritable({ limitInMemoryBytes: limit, tmpRoot }) + + // Stub createWriteStream to fail on creation + const spy = jest.spyOn(fs, 'createWriteStream').mockImplementation(() => { + throw Object.assign(new Error('simulated createWriteStream failure'), { code: 'EACCES' }) + }) + + try { + await expect(pipeline(readableFrom(payload), sink)).rejects.toThrow( + /createWriteStream failure|EACCES|simulated/i + ) + } finally { + spy.mockRestore() + } + + // Ensure no lingering temp dirs/files (best-effort) + await new Promise((r) => setTimeout(r, 10)) + expect(await countHashspillDirs(tmpRoot)).toBe(0) + }) + + test('spill: spilled file exists before read and is deleted after autoCleanup', async () => { + const limit = 16 * 1024 + const payload = randBuf(limit * 2 + 123) // force spill + const sink = new HashSpillWritable({ limitInMemoryBytes: limit, tmpRoot }) + + await pipeline(readableFrom(payload), sink) + + // The spill dir & file should exist now + const prePath = await findSpillFilePath(tmpRoot) + expect(prePath).not.toBeNull() + expect(fs.existsSync(prePath!)).toBe(true) + + // Read with autoCleanup + await pipeline( + sink.toReadable({ autoCleanup: true }), + new Writable({ + write(_c, _e, cb) { + cb() + }, + }) + ) + + // Give the event loop a tick for cleanup + await new Promise((r) => setTimeout(r, 15)) + + // The specific spilled file AND directory should be gone + const postPath = await findSpillFilePath(tmpRoot) + expect(postPath).toBeNull() + expect(fs.existsSync(prePath!)).toBe(false) + + // And no hashspill dirs remain + expect(await countHashspillDirs(tmpRoot)).toBe(0) + }) + test('concurrent spill operations: no temp file name collisions with rapid creation', async () => { + const limit = 4 * 1024 + const N = 20 // More instances to increase collision probability + + // Create all instances simultaneously to maximize collision chance + const sinks = Array.from( + { length: N }, + () => new HashSpillWritable({ limitInMemoryBytes: limit, tmpRoot }) + ) + + // Start all writes concurrently + const writePromises = sinks.map(async (sink, idx) => { + const buf = randBuf(limit + 100 + idx) // force spill on all + await pipeline(readableFrom(buf), sink) + return { sink, expected: createHash('sha256').update(buf).digest('hex') } + }) + + const results = await Promise.all(writePromises) + + // Verify all succeeded with correct digests + for (const { sink, expected } of results) { + expect(sink.digestHex()).toBe(expected) + } + + // Verify all created separate temp directories + expect(await countHashspillDirs(tmpRoot)).toBe(N) + + // Clean up all with autoCleanup + const readPromises = results.map(({ sink }) => + pipeline( + sink.toReadable({ autoCleanup: true }), + new Writable({ + write(_c, _e, cb) { + cb() + }, + }) + ) + ) + + await Promise.all(readPromises) + await new Promise((r) => setTimeout(r, 20)) // Allow cleanup + + expect(await countHashspillDirs(tmpRoot)).toBe(0) + }) + + test('concurrent spill with identical timestamps: UUID ensures uniqueness', async () => { + const limit = 2 * 1024 + const N = 10 + + // Mock Date.now to return same timestamp for all instances + const originalDateNow = Date.now + const fixedTimestamp = 1234567890123 + jest.spyOn(Date, 'now').mockReturnValue(fixedTimestamp) + + try { + const jobs = Array.from({ length: N }, async (_, idx) => { + const payload = randBuf(limit + 50 + idx) + const sink = new HashSpillWritable({ limitInMemoryBytes: limit, tmpRoot }) + await pipeline(readableFrom(payload), sink) + + // Verify file was created successfully despite same timestamp + const spillPath = await findSpillFilePath(tmpRoot) + expect(spillPath).not.toBeNull() + + await sink.cleanup() + return sink.digestHex() + }) + + // All should succeed despite identical timestamps + const digests = await Promise.all(jobs) + expect(digests).toHaveLength(N) + + // All temp dirs should be cleaned up + await new Promise((r) => setTimeout(r, 10)) + expect(await countHashspillDirs(tmpRoot)).toBe(0) + } finally { + jest.restoreAllMocks() + } + }) + + test('concurrent readers on same spilled stream with mixed cleanup strategies', async () => { + const limit = 8 * 1024 + const payload = randBuf(limit * 2) + const sink = new HashSpillWritable({ limitInMemoryBytes: limit, tmpRoot }) + + await pipeline(readableFrom(payload), sink) + expect(await countHashspillDirs(tmpRoot)).toBe(1) + + // Create multiple readers: some with autoCleanup, some without + const readers = [ + { stream: sink.toReadable({ autoCleanup: true }), name: 'auto1' }, + { stream: sink.toReadable({ autoCleanup: false }), name: 'manual1' }, + { stream: sink.toReadable({ autoCleanup: true }), name: 'auto2' }, + { stream: sink.toReadable({ autoCleanup: false }), name: 'manual2' }, + { stream: sink.toReadable({ autoCleanup: true }), name: 'auto3' }, + ] + + // Read from all concurrently with varying speeds + const readPromises = readers.map(({ stream, name }, idx) => { + const consumer = new SlowWritable(100 * idx + 1) // Different speeds + return pipeline(stream, consumer) + }) + + await Promise.all(readPromises) + + // Even though some had autoCleanup=true, cleanup should be deferred + // because other readers existed. Only manual cleanup should work now. + expect(await countHashspillDirs(tmpRoot)).toBe(1) + + await new Promise((r) => setTimeout(r, 200)) + // Manual cleanup should now succeed + await sink.cleanup() + expect(await countHashspillDirs(tmpRoot)).toBe(0) + }) + + test('rapid spill/cleanup cycles: no resource leaks or race conditions', async () => { + const limit = 4 * 1024 + const cycles = 15 + + for (let i = 0; i < cycles; i++) { + const payload = randBuf(limit + 100 + i) + const sink = new HashSpillWritable({ limitInMemoryBytes: limit, tmpRoot }) + + await pipeline(readableFrom(payload), sink) + + // Immediately read and cleanup + await pipeline( + sink.toReadable({ autoCleanup: true }), + new Writable({ + write(_c, _e, cb) { + cb() + }, + }) + ) + + // Brief pause to allow cleanup + await new Promise((r) => setTimeout(r, 2)) + } + + // All temp artifacts should be cleaned up + await new Promise((r) => setTimeout(r, 20)) + expect(await countHashspillDirs(tmpRoot)).toBe(0) + }) + + test('spill during concurrent writes to different tmp roots: isolation verified', async () => { + const limit = 6 * 1024 + const tmpRoot2 = await fsp.mkdtemp(path.join(os.tmpdir(), 'hsw-tests2-')) + + try { + const payload1 = randBuf(limit + 200) + const payload2 = randBuf(limit + 300) + + const sink1 = new HashSpillWritable({ limitInMemoryBytes: limit, tmpRoot }) + const sink2 = new HashSpillWritable({ limitInMemoryBytes: limit, tmpRoot: tmpRoot2 }) + + // Write to both concurrently + await Promise.all([ + pipeline(readableFrom(payload1), sink1), + pipeline(readableFrom(payload2), sink2), + ]) + + // Each should have created temp dirs in their respective roots + expect(await countHashspillDirs(tmpRoot)).toBe(1) + expect(await countHashspillDirs(tmpRoot2)).toBe(1) + + // Cleanup both + await Promise.all([sink1.cleanup(), sink2.cleanup()]) + + expect(await countHashspillDirs(tmpRoot)).toBe(0) + expect(await countHashspillDirs(tmpRoot2)).toBe(0) + } finally { + await fsp.rm(tmpRoot2, { recursive: true, force: true }).catch(() => {}) + } + }) + + test('stress test: many concurrent spills with overlapping lifecycles', async () => { + const limit = 8 * 1024 + const batchSize = 12 + + // Create overlapping batches + const batch1Promise = Promise.all( + Array.from({ length: batchSize }, async (_, i) => { + const payload = randBuf(limit * 2 + i) + const sink = new HashSpillWritable({ limitInMemoryBytes: limit, tmpRoot }) + await pipeline(readableFrom(payload), sink) + + // Delay before reading to create overlap with batch2 + await new Promise((r) => setTimeout(r, 10 + (i % 3))) + + await pipeline( + sink.toReadable({ autoCleanup: true }), + new Writable({ + write(_c, _e, cb) { + cb() + }, + }) + ) + + return sink.digestHex() + }) + ) + + // Start second batch while first is still running + await new Promise((r) => setTimeout(r, 20)) + + const batch2Promise = Promise.all( + Array.from({ length: batchSize }, async (_, i) => { + const payload = randBuf(limit * 3 + i) + const sink = new HashSpillWritable({ limitInMemoryBytes: limit, tmpRoot }) + await pipeline(readableFrom(payload), sink) + + await pipeline( + sink.toReadable({ autoCleanup: true }), + new Writable({ + write(_c, _e, cb) { + cb() + }, + }) + ) + + return sink.digestHex() + }) + ) + + const [results1, results2] = await Promise.all([batch1Promise, batch2Promise]) + + expect(results1).toHaveLength(batchSize) + expect(results2).toHaveLength(batchSize) + + // Allow all cleanup to complete + await new Promise((r) => setTimeout(r, 50)) + expect(await countHashspillDirs(tmpRoot)).toBe(0) + }) +}) diff --git a/src/test/iceberg.test.ts b/src/test/iceberg.test.ts index 5239a7860..67016044f 100644 --- a/src/test/iceberg.test.ts +++ b/src/test/iceberg.test.ts @@ -42,6 +42,64 @@ describe('Iceberg Catalog', () => { await t.database.connection.pool.destroy() }) + it('can create an analytic bucket', async () => { + const bucketName = t.random.name('ice-bucket') + + const response = await app.inject({ + method: 'POST', + url: '/iceberg/bucket', + headers: { + 'Content-Type': 'application/json', + Authorization: `Bearer ${await serviceKeyAsync}`, + }, + payload: { + name: bucketName, + }, + }) + + const resp = await response.json() + expect(response.statusCode).toBe(200) + expect(resp.id).toBe(bucketName) + }) + + it('can list analytic buckets', async () => { + const bucketName = t.random.name('ice-bucket') + await t.storage.createIcebergBucket({ + id: bucketName, + }) + + const response = await app.inject({ + method: 'GET', + url: '/iceberg/bucket', + headers: { + 'Content-Type': 'application/json', + Authorization: `Bearer ${await serviceKeyAsync}`, + }, + }) + + const resp = await response.json() + expect(response.statusCode).toBe(200) + expect(resp.length).toBeGreaterThan(0) + }) + + it('can delete analytic bucket', async () => { + const bucketName = t.random.name('ice-bucket') + await t.storage.createIcebergBucket({ + id: bucketName, + }) + + const response = await app.inject({ + method: 'DELETE', + url: `/iceberg/bucket/${bucketName}`, + headers: { + 'Content-Type': 'application/json', + Authorization: `Bearer ${await serviceKeyAsync}`, + }, + }) + + expect(response.statusCode).toBe(200) + }) + it('can create a table bucket', async () => { const bucketName = t.random.name('ice-bucket') @@ -66,7 +124,6 @@ describe('Iceberg Catalog', () => { it('can get catalog config', async () => { const bucketName = t.random.name('ice-bucket') await t.storage.createIcebergBucket({ - name: bucketName, id: bucketName, }) @@ -104,7 +161,6 @@ describe('Iceberg Catalog', () => { it('can create namespaces', async () => { const bucketName = t.random.name('ice-bucket') await t.storage.createIcebergBucket({ - name: bucketName, id: bucketName, }) @@ -148,7 +204,6 @@ describe('Iceberg Catalog', () => { const bucketName = t.random.name('ice-bucket') const bucket = await t.storage.createIcebergBucket({ - name: bucketName, id: bucketName, }) @@ -186,7 +241,6 @@ describe('Iceberg Catalog', () => { const bucketName = t.random.name('ice-bucket') const bucket = await t.storage.createIcebergBucket({ - name: bucketName, id: bucketName, }) @@ -227,7 +281,6 @@ describe('Iceberg Catalog', () => { const bucketName = t.random.name('ice-bucket') const bucket = await t.storage.createIcebergBucket({ - name: bucketName, id: bucketName, }) @@ -269,7 +322,6 @@ describe('Iceberg Catalog', () => { const bucketName = t.random.name('ice-bucket') const bucket = await t.storage.createIcebergBucket({ - name: bucketName, id: bucketName, }) @@ -300,7 +352,6 @@ describe('Iceberg Catalog', () => { it('can create a table', async () => { const bucketName = t.random.name('ice-bucket') await t.storage.createIcebergBucket({ - name: bucketName, id: bucketName, }) @@ -408,7 +459,6 @@ describe('Iceberg Catalog', () => { it('can list tables in a namespace', async () => { const bucketName = t.random.name('ice-bucket') await t.storage.createIcebergBucket({ - name: bucketName, id: bucketName, }) @@ -467,7 +517,6 @@ describe('Iceberg Catalog', () => { it('check if table exists', async () => { const bucketName = t.random.name('ice-bucket') await t.storage.createIcebergBucket({ - name: bucketName, id: bucketName, }) @@ -502,7 +551,6 @@ describe('Iceberg Catalog', () => { it('can drop a table', async () => { const bucketName = t.random.name('ice-bucket') await t.storage.createIcebergBucket({ - name: bucketName, id: bucketName, }) @@ -538,7 +586,6 @@ describe('Iceberg Catalog', () => { it('can load table metadata', async () => { const bucketName = t.random.name('ice-bucket') await t.storage.createIcebergBucket({ - name: bucketName, id: bucketName, }) @@ -684,7 +731,6 @@ describe('Iceberg Catalog', () => { await createBucketIfNotExists(internalBucketName, minioClient) await t.storage.createIcebergBucket({ - name: bucketName, id: bucketName, }) diff --git a/src/test/tenant.test.ts b/src/test/tenant.test.ts index f1da239fc..78799cf9e 100644 --- a/src/test/tenant.test.ts +++ b/src/test/tenant.test.ts @@ -47,11 +47,16 @@ const payload = { enabled: false, }, icebergCatalog: { - enabled: false, + enabled: true, maxCatalogs: 2, maxNamespaces: 10, maxTables: 10, }, + vectorBuckets: { + enabled: true, + maxBuckets: 2, + maxIndexes: 10, + }, }, disableEvents: null, } @@ -85,11 +90,16 @@ const payload2 = { enabled: true, }, icebergCatalog: { - enabled: false, + enabled: true, maxCatalogs: 2, maxNamespaces: 10, maxTables: 10, }, + vectorBuckets: { + enabled: true, + maxBuckets: 2, + maxIndexes: 10, + }, }, disableEvents: null, } diff --git a/src/test/vectors.test.ts b/src/test/vectors.test.ts new file mode 100644 index 000000000..470213b58 --- /dev/null +++ b/src/test/vectors.test.ts @@ -0,0 +1,1940 @@ +'use strict' + +import app from '../app' +import { getConfig, mergeConfig } from '../config' +import { FastifyInstance } from 'fastify' +import { useMockObject, useMockQueue } from './common' +import { + CreateIndexCommandOutput, + DeleteVectorsOutput, + GetVectorsCommandOutput, + ListVectorsOutput, + PutVectorsOutput, + QueryVectorsOutput, +} from '@aws-sdk/client-s3vectors' +import { KnexVectorMetadataDB, VectorStore, VectorStoreManager } from '@storage/protocols/vector' +import { useStorage } from './utils/storage' +import { signJWT } from '@internal/auth' + +const { serviceKeyAsync, vectorBucketS3, tenantId, jwtSecret } = getConfig() + +let appInstance: FastifyInstance +let serviceToken: string + +// Use the common mock helpers +useMockObject() +useMockQueue() + +jest.mock('@storage/protocols/vector/adapter/s3-vector', () => { + const mockS3Vector = { + deleteVectorIndex: jest.fn().mockResolvedValue({} as CreateIndexCommandOutput), + createVectorIndex: jest.fn().mockResolvedValue({} as CreateIndexCommandOutput), + putVectors: jest.fn().mockResolvedValue({} as PutVectorsOutput), + listVectors: jest.fn().mockResolvedValue({} as ListVectorsOutput), + queryVectors: jest.fn().mockResolvedValue({} as QueryVectorsOutput), + deleteVectors: jest.fn().mockResolvedValue({} as DeleteVectorsOutput), + getVectors: jest.fn().mockResolvedValue({} as GetVectorsCommandOutput), + createS3VectorClient: jest.fn().mockReturnValue({}), + } + + return { + S3Vector: jest.fn().mockImplementation(() => mockS3Vector), + ...mockS3Vector, + } +}) + +const mockVectorStore = jest.mocked( + jest.requireMock('@storage/protocols/vector/adapter/s3-vector') +) + +let vectorBucketName: string +let s3Vector: VectorStoreManager + +describe('Vectors API', () => { + const storageTest = useStorage() + + beforeAll(async () => { + appInstance = app() + + // Create service role token + serviceToken = await serviceKeyAsync + + // Create real S3Vector instance with mocked client and mock DB + const mockVectorDB = new KnexVectorMetadataDB(storageTest.database.connection.pool.acquire()) + s3Vector = new VectorStoreManager(mockVectorStore, mockVectorDB, { + tenantId: 'test-tenant', + vectorBucketName: 'test-bucket', + maxBucketCount: Infinity, + maxIndexCount: Infinity, + }) + + // Decorate fastify instance with real S3Vector + appInstance.decorate('s3Vector', s3Vector) + }) + + afterAll(async () => { + await appInstance.close() + await storageTest.database.connection.dispose() + }) + + beforeEach(async () => { + jest.clearAllMocks() + jest.resetAllMocks() + + getConfig({ reload: true }) + mergeConfig({ vectorMaxBucketsCount: Infinity, vectorMaxIndexesCount: Infinity }) + + vectorBucketName = `test-bucket-${Date.now()}` + await s3Vector.createBucket(vectorBucketName) + }) + + describe('POST /vectors/CreateIndex', () => { + let validCreateIndexRequest: { + dataType: 'float32' + dimension: number + distanceMetric: 'cosine' | 'euclidean' + indexName: string + vectorBucketName: string + metadataConfiguration?: { + nonFilterableMetadataKeys: string[] + } + } + beforeEach(async () => { + validCreateIndexRequest = { + dataType: 'float32', + dimension: 1536, + distanceMetric: 'cosine', + indexName: 'test-index', + vectorBucketName: vectorBucketName, + metadataConfiguration: { + nonFilterableMetadataKeys: ['key1', 'key2'], + }, + } + }) + + it('should create vector index successfully with valid request', async () => { + const response = await appInstance.inject({ + method: 'POST', + url: '/vectors/CreateIndex', + headers: { + authorization: `Bearer ${serviceToken}`, + }, + payload: validCreateIndexRequest, + }) + + expect(response.statusCode).toBe(200) + + // Verify the CreateIndexCommand was called with correct parameters including tenantId prefix + const createIndexCommand = mockVectorStore.createVectorIndex + expect(createIndexCommand).toBeCalledWith({ + ...validCreateIndexRequest, + vectorBucketName: vectorBucketS3, + indexName: `${tenantId}-test-index`, + }) + + const indexMetadata = await storageTest.database.connection.pool + .acquire() + .table('storage.vector_indexes') + .where({ + name: validCreateIndexRequest.indexName, + bucket_id: validCreateIndexRequest.vectorBucketName, + }) + .first() + + expect(indexMetadata).toBeDefined() + expect(indexMetadata?.data_type).toBe(validCreateIndexRequest.dataType) + expect(indexMetadata?.dimension).toBe(validCreateIndexRequest.dimension) + expect(indexMetadata?.distance_metric).toBe(validCreateIndexRequest.distanceMetric) + expect(indexMetadata?.metadata_configuration).toEqual( + validCreateIndexRequest.metadataConfiguration + ) + }) + + it('should require authentication with service role', async () => { + const response = await appInstance.inject({ + method: 'POST', + url: '/vectors/CreateIndex', + payload: validCreateIndexRequest, + }) + + expect(response.statusCode).toBe(403) + // Vector service not called when validation fails + }) + + it('should reject request with invalid JWT role', async () => { + const invalidToken = 'invalid-token' + + const response = await appInstance.inject({ + method: 'POST', + url: '/vectors/CreateIndex', + headers: { + authorization: `Bearer ${invalidToken}`, + }, + payload: validCreateIndexRequest, + }) + + expect(response.statusCode).toBe(400) + // Vector service not called when validation fails + }) + + it('should validate required fields', async () => { + const incompleteRequest = { + dataType: 'float32', + dimension: 1536, + // missing required fields + } + + const response = await appInstance.inject({ + method: 'POST', + url: '/vectors/CreateIndex', + headers: { + authorization: `Bearer ${serviceToken}`, + }, + payload: incompleteRequest, + }) + + expect(response.statusCode).toBe(400) + // Vector service not called when validation fails + }) + + it('should validate dataType enum', async () => { + const invalidRequest = { + ...validCreateIndexRequest, + dataType: 'invalid-type', + } + + const response = await appInstance.inject({ + method: 'POST', + url: '/vectors/CreateIndex', + headers: { + authorization: `Bearer ${serviceToken}`, + }, + payload: invalidRequest, + }) + + expect(response.statusCode).toBe(400) + // Vector service not called when validation fails + }) + + it('should validate distanceMetric enum', async () => { + const invalidRequest = { + ...validCreateIndexRequest, + distanceMetric: 'invalid-metric', + } + + const response = await appInstance.inject({ + method: 'POST', + url: '/vectors/CreateIndex', + headers: { + authorization: `Bearer ${serviceToken}`, + }, + payload: invalidRequest, + }) + + expect(response.statusCode).toBe(400) + // Vector service not called when validation fails + }) + + it('should validate dimension is a number', async () => { + const invalidRequest = { + ...validCreateIndexRequest, + dimension: 'not-a-number', + } + + const response = await appInstance.inject({ + method: 'POST', + url: '/vectors/CreateIndex', + headers: { + authorization: `Bearer ${serviceToken}`, + }, + payload: invalidRequest, + }) + + expect(response.statusCode).toBe(400) + // Vector service not called when validation fails + }) + + it('should validate metadataConfiguration structure', async () => { + const invalidRequest = { + ...validCreateIndexRequest, + metadataConfiguration: { + // missing required nonFilterableMetadataKeys + invalidKey: 'value', + }, + } + + const response = await appInstance.inject({ + method: 'POST', + url: '/vectors/CreateIndex', + headers: { + authorization: `Bearer ${serviceToken}`, + }, + payload: invalidRequest, + }) + + expect(response.statusCode).toBe(400) + // Vector service not called when validation fails + }) + + it('should handle vector service not configured', async () => { + // Mock app without s3Vector service + const appWithoutVector = app() + mergeConfig({ vectorEnabled: false }) + + const response = await appWithoutVector.inject({ + method: 'POST', + url: '/vectors/CreateIndex', + headers: { + authorization: `Bearer ${serviceToken}`, + }, + payload: validCreateIndexRequest, + }) + + expect(response.statusCode).toBe(404) + const body = JSON.parse(response.body) + expect(body.error).toBe('Not Found') + + await appWithoutVector.close() + }) + + it('should handle S3Vector service errors', async () => { + const s3Error = new Error('S3VectorsClient error') + // Mock error - need to cast to bypass type restrictions + mockVectorStore.createVectorIndex.mockRejectedValue(s3Error) + + const response = await appInstance.inject({ + method: 'POST', + url: '/vectors/CreateIndex', + headers: { + authorization: `Bearer ${serviceToken}`, + }, + payload: validCreateIndexRequest, + }) + + expect(response.statusCode).toBe(500) + expect(mockVectorStore.createVectorIndex).toHaveBeenCalledTimes(1) + }) + + it('should accept valid request without optional metadataConfiguration', async () => { + const requestWithoutMetadata = { + dataType: 'float32' as const, + dimension: 1536, + distanceMetric: 'euclidean' as const, + indexName: 'test-index-2', + vectorBucketName: vectorBucketName, + } + + const response = await appInstance.inject({ + method: 'POST', + url: '/vectors/CreateIndex', + headers: { + authorization: `Bearer ${serviceToken}`, + }, + payload: requestWithoutMetadata, + }) + + expect(response.statusCode).toBe(200) + expect(mockVectorStore.createVectorIndex).toHaveBeenCalledTimes(1) + expect(mockVectorStore.createVectorIndex).toHaveBeenCalledWith({ + ...requestWithoutMetadata, + vectorBucketName: vectorBucketS3, + indexName: `${tenantId}-test-index-2`, + }) + }) + }) + + describe('POST /vectors/CreateVectorBucket', () => { + beforeEach(async () => {}) + + it('should create vector bucket successfully with valid request', async () => { + const newBucketName = `test-bucket-${Date.now()}-new` + const response = await appInstance.inject({ + method: 'POST', + url: '/vectors/CreateVectorBucket', + headers: { + authorization: `Bearer ${serviceToken}`, + }, + payload: { + vectorBucketName: newBucketName, + }, + }) + + expect(response.statusCode).toBe(200) + + // Verify bucket was created in database + const bucketRecord = await storageTest.database.connection.pool + .acquire() + .table('storage.buckets_vectors') + .where({ id: newBucketName }) + .first() + + expect(bucketRecord).toBeDefined() + expect(bucketRecord?.id).toBe(newBucketName) + expect(bucketRecord?.created_at).toBeDefined() + }) + + it('should require authentication with service role', async () => { + const response = await appInstance.inject({ + method: 'POST', + url: '/vectors/CreateVectorBucket', + payload: { + vectorBucketName: 'test-bucket', + }, + }) + + expect(response.statusCode).toBe(403) + }) + + it('should reject request with invalid JWT role', async () => { + const token = await signJWT({ role: 'auth', sub: '1234' }, jwtSecret, '1h') + + const response = await appInstance.inject({ + method: 'POST', + url: '/vectors/CreateVectorBucket', + headers: { + authorization: `Bearer ${token}`, + }, + payload: { + vectorBucketName: 'test-bucket', + }, + }) + + expect(response.statusCode).toBe(403) + }) + + it('should validate required fields', async () => { + const response = await appInstance.inject({ + method: 'POST', + url: '/vectors/CreateVectorBucket', + headers: { + authorization: `Bearer ${serviceToken}`, + }, + payload: {}, + }) + + expect(response.statusCode).toBe(400) + }) + + it('should handle duplicate bucket creation gracefully', async () => { + // First creation + const newVectorBucketName = `test-bucket-${Date.now()}-dup` + const response1 = await appInstance.inject({ + method: 'POST', + url: '/vectors/CreateVectorBucket', + headers: { + authorization: `Bearer ${serviceToken}`, + }, + payload: { + vectorBucketName: newVectorBucketName, + }, + }) + + expect(response1.statusCode).toBe(200) + + // Second creation should return conflict + const response2 = await appInstance.inject({ + method: 'POST', + url: '/vectors/CreateVectorBucket', + headers: { + authorization: `Bearer ${serviceToken}`, + }, + payload: { + vectorBucketName: newVectorBucketName, + }, + }) + + expect(response2.statusCode).toBe(409) + }) + }) + + describe('POST /vectors/DeleteVectorBucket', () => { + beforeEach(async () => {}) + it('should delete empty vector bucket successfully', async () => { + const response = await appInstance.inject({ + method: 'POST', + url: '/vectors/DeleteVectorBucket', + headers: { + authorization: `Bearer ${serviceToken}`, + }, + payload: { + vectorBucketName: vectorBucketName, + }, + }) + + expect(response.statusCode).toBe(200) + + // Verify bucket was deleted from database + const bucketRecord = await storageTest.database.connection.pool + .acquire() + .table('storage.buckets_vectors') + .where({ id: vectorBucketName }) + .first() + + expect(bucketRecord).toBeUndefined() + }) + + it('should fail when trying to delete bucket with indexes', async () => { + // First create an index in the bucket + await appInstance.inject({ + method: 'POST', + url: '/vectors/CreateIndex', + headers: { + authorization: `Bearer ${serviceToken}`, + }, + payload: { + dataType: 'float32', + dimension: 1536, + distanceMetric: 'cosine', + indexName: 'test-index', + vectorBucketName: vectorBucketName, + }, + }) + + // Try to delete the bucket + const response = await appInstance.inject({ + method: 'POST', + url: '/vectors/DeleteVectorBucket', + headers: { + authorization: `Bearer ${serviceToken}`, + }, + payload: { + vectorBucketName: vectorBucketName, + }, + }) + + expect(response.statusCode).toBe(400) + const body = JSON.parse(response.body) + expect(body.error).toBe('VectorBucketNotEmpty') + }) + + it('should require authentication with service role', async () => { + const response = await appInstance.inject({ + method: 'POST', + url: '/vectors/DeleteVectorBucket', + payload: { + vectorBucketName: vectorBucketName, + }, + }) + + expect(response.statusCode).toBe(403) + }) + + it('should validate required fields', async () => { + const response = await appInstance.inject({ + method: 'POST', + url: '/vectors/DeleteVectorBucket', + headers: { + authorization: `Bearer ${serviceToken}`, + }, + payload: {}, + }) + + expect(response.statusCode).toBe(400) + }) + + it('should handle non-existent bucket', async () => { + const response = await appInstance.inject({ + method: 'POST', + url: '/vectors/DeleteVectorBucket', + headers: { + authorization: `Bearer ${serviceToken}`, + }, + payload: { + vectorBucketName: 'non-existent-bucket', + }, + }) + + expect(response.statusCode).toBe(200) + }) + }) + + describe('POST /vectors/ListVectorBuckets', () => { + beforeEach(async () => { + // Create multiple buckets for listing + await s3Vector.createBucket(`test-bucket-a-${Date.now()}`) + await s3Vector.createBucket(`test-bucket-b-${Date.now()}`) + await s3Vector.createBucket(`test-bucket-c-${Date.now()}`) + }) + + it('should list all vector buckets', async () => { + const response = await appInstance.inject({ + method: 'POST', + url: '/vectors/ListVectorBuckets', + headers: { + authorization: `Bearer ${serviceToken}`, + }, + payload: {}, + }) + + expect(response.statusCode).toBe(200) + const body = JSON.parse(response.body) + expect(body.vectorBuckets).toBeDefined() + expect(Array.isArray(body.vectorBuckets)).toBe(true) + expect(body.vectorBuckets.length).toBeGreaterThan(0) + + // Verify structure of bucket objects + body.vectorBuckets.forEach((bucket: any) => { + expect(bucket.vectorBucketName).toBeDefined() + expect(bucket.creationTime).toBeDefined() + expect(typeof bucket.creationTime).toBe('number') + }) + }) + + it('should support maxResults parameter', async () => { + const response = await appInstance.inject({ + method: 'POST', + url: '/vectors/ListVectorBuckets', + headers: { + authorization: `Bearer ${serviceToken}`, + }, + payload: { + maxResults: 2, + }, + }) + + expect(response.statusCode).toBe(200) + const body = JSON.parse(response.body) + expect(body.vectorBuckets.length).toBeLessThanOrEqual(2) + if (body.vectorBuckets.length === 2) { + expect(body.nextToken).toBeDefined() + } + }) + + it('should support pagination with nextToken', async () => { + const response1 = await appInstance.inject({ + method: 'POST', + url: '/vectors/ListVectorBuckets', + headers: { + authorization: `Bearer ${serviceToken}`, + }, + payload: { + maxResults: 1, + }, + }) + + const body1 = JSON.parse(response1.body) + + if (body1.nextToken) { + const response2 = await appInstance.inject({ + method: 'POST', + url: '/vectors/ListVectorBuckets', + headers: { + authorization: `Bearer ${serviceToken}`, + }, + payload: { + maxResults: 1, + nextToken: body1.nextToken, + }, + }) + + expect(response2.statusCode).toBe(200) + const body2 = JSON.parse(response2.body) + expect(body2.vectorBuckets).toBeDefined() + + // Ensure different buckets are returned + if (body2.vectorBuckets.length > 0) { + expect(body1.vectorBuckets[0].vectorBucketName).not.toBe( + body2.vectorBuckets[0].vectorBucketName + ) + } + } + }) + + it('should support prefix filtering', async () => { + const prefix = 'test-bucket-a' + const response = await appInstance.inject({ + method: 'POST', + url: '/vectors/ListVectorBuckets', + headers: { + authorization: `Bearer ${serviceToken}`, + }, + payload: { + prefix, + }, + }) + + expect(response.statusCode).toBe(200) + const body = JSON.parse(response.body) + body.vectorBuckets.forEach((bucket: any) => { + expect(bucket.vectorBucketName).toMatch(new RegExp(`^${prefix}`)) + }) + }) + + it('should require authentication with service role', async () => { + const response = await appInstance.inject({ + method: 'POST', + url: '/vectors/ListVectorBuckets', + payload: {}, + }) + + expect(response.statusCode).toBe(403) + }) + }) + + describe('POST /vectors/GetVectorBucket', () => { + it('should get vector bucket details successfully', async () => { + const response = await appInstance.inject({ + method: 'POST', + url: '/vectors/GetVectorBucket', + headers: { + authorization: `Bearer ${serviceToken}`, + }, + payload: { + vectorBucketName: vectorBucketName, + }, + }) + + expect(response.statusCode).toBe(200) + const body = JSON.parse(response.body) + expect(body.vectorBucket).toBeDefined() + expect(body.vectorBucket.vectorBucketName).toBe(vectorBucketName) + expect(body.vectorBucket.creationTime).toBeDefined() + expect(typeof body.vectorBucket.creationTime).toBe('number') + }) + + it('should require authentication with service role', async () => { + const response = await appInstance.inject({ + method: 'POST', + url: '/vectors/GetVectorBucket', + payload: { + vectorBucketName: vectorBucketName, + }, + }) + + expect(response.statusCode).toBe(403) + }) + + it('should validate required fields', async () => { + const response = await appInstance.inject({ + method: 'POST', + url: '/vectors/GetVectorBucket', + headers: { + authorization: `Bearer ${serviceToken}`, + }, + payload: {}, + }) + + expect(response.statusCode).toBe(400) + }) + + it('should handle non-existent bucket', async () => { + const response = await appInstance.inject({ + method: 'POST', + url: '/vectors/GetVectorBucket', + headers: { + authorization: `Bearer ${serviceToken}`, + }, + payload: { + vectorBucketName: 'non-existent-bucket', + }, + }) + + expect(response.statusCode).toBe(404) + }) + }) + + describe('POST /vectors/DeleteIndex', () => { + let indexName: string + + beforeEach(async () => { + vectorBucketName = `test-delete-index-${Date.now()}` + await s3Vector.createBucket(vectorBucketName) + + indexName = `test-index-${Date.now()}` + // Create an index first + + await s3Vector.createVectorIndex({ + dataType: 'float32', + dimension: 1536, + distanceMetric: 'cosine', + indexName: indexName, + vectorBucketName: vectorBucketName, + }) + }) + + it('should delete vector index successfully', async () => { + const response = await appInstance.inject({ + method: 'POST', + url: '/vectors/DeleteIndex', + headers: { + authorization: `Bearer ${serviceToken}`, + }, + payload: { + indexName: indexName, + vectorBucketName: vectorBucketName, + }, + }) + + expect(response.statusCode).toBe(200) + + // Verify the index was deleted from database + const indexRecord = await storageTest.database.connection.pool + .acquire() + .table('storage.vector_indexes') + .where({ + name: indexName, + bucket_id: vectorBucketName, + }) + .first() + + expect(indexRecord).toBeUndefined() + + // Verify deleteVectorIndex was called with correct parameters + expect(mockVectorStore.deleteVectorIndex).toHaveBeenCalledWith({ + vectorBucketName: vectorBucketS3, + indexName: `${tenantId}-${indexName}`, + }) + }) + + it('should require authentication with service role', async () => { + const response = await appInstance.inject({ + method: 'POST', + url: '/vectors/DeleteIndex', + payload: { + indexName: indexName, + vectorBucketName: vectorBucketName, + }, + }) + + expect(response.statusCode).toBe(403) + }) + + it('should validate required fields', async () => { + const response = await appInstance.inject({ + method: 'POST', + url: '/vectors/DeleteIndex', + headers: { + authorization: `Bearer ${serviceToken}`, + }, + payload: { + indexName: indexName, + }, + }) + + expect(response.statusCode).toBe(400) + }) + + it('should validate indexName pattern', async () => { + const response = await appInstance.inject({ + method: 'POST', + url: '/vectors/DeleteIndex', + headers: { + authorization: `Bearer ${serviceToken}`, + }, + payload: { + indexName: 'INVALID_NAME', + vectorBucketName: vectorBucketName, + }, + }) + + expect(response.statusCode).toBe(400) + }) + + it('should handle non-existent index', async () => { + const response = await appInstance.inject({ + method: 'POST', + url: '/vectors/DeleteIndex', + headers: { + authorization: `Bearer ${serviceToken}`, + }, + payload: { + indexName: 'non-existent-index', + vectorBucketName: vectorBucketName, + }, + }) + + expect(response.statusCode).toBe(404) + }) + }) + + describe('POST /vectors/ListIndexes', () => { + beforeEach(async () => { + // Create multiple indexes for listing + await appInstance.inject({ + method: 'POST', + url: '/vectors/CreateIndex', + headers: { + authorization: `Bearer ${serviceToken}`, + }, + payload: { + dataType: 'float32', + dimension: 1536, + distanceMetric: 'cosine', + indexName: `index-a-${Date.now()}`, + vectorBucketName: vectorBucketName, + }, + }) + + await appInstance.inject({ + method: 'POST', + url: '/vectors/CreateIndex', + headers: { + authorization: `Bearer ${serviceToken}`, + }, + payload: { + dataType: 'float32', + dimension: 768, + distanceMetric: 'euclidean', + indexName: `index-b-${Date.now()}`, + vectorBucketName: vectorBucketName, + }, + }) + }) + + it('should list all indexes in a bucket', async () => { + const response = await appInstance.inject({ + method: 'POST', + url: '/vectors/ListIndexes', + headers: { + authorization: `Bearer ${serviceToken}`, + }, + payload: { + vectorBucketName: vectorBucketName, + }, + }) + + expect(response.statusCode).toBe(200) + const body = JSON.parse(response.body) + expect(body.indexes).toBeDefined() + expect(Array.isArray(body.indexes)).toBe(true) + expect(body.indexes.length).toBeGreaterThanOrEqual(2) + + // Verify structure of index objects + body.indexes.forEach((index: any) => { + expect(index.indexName).toBeDefined() + expect(index.vectorBucketName).toBe(vectorBucketName) + expect(index.creationTime).toBeDefined() + expect(typeof index.creationTime).toBe('number') + }) + }) + + it('should support maxResults parameter', async () => { + const response = await appInstance.inject({ + method: 'POST', + url: '/vectors/ListIndexes', + headers: { + authorization: `Bearer ${serviceToken}`, + }, + payload: { + vectorBucketName: vectorBucketName, + maxResults: 1, + }, + }) + + expect(response.statusCode).toBe(200) + const body = JSON.parse(response.body) + expect(body.indexes.length).toBeLessThanOrEqual(1) + }) + + it('should support prefix filtering', async () => { + const prefix = 'index-a' + const response = await appInstance.inject({ + method: 'POST', + url: '/vectors/ListIndexes', + headers: { + authorization: `Bearer ${serviceToken}`, + }, + payload: { + vectorBucketName: vectorBucketName, + prefix, + }, + }) + + expect(response.statusCode).toBe(200) + const body = JSON.parse(response.body) + body.indexes.forEach((index: any) => { + expect(index.indexName).toMatch(new RegExp(`^${prefix}`)) + }) + }) + + it('should require authentication with service role', async () => { + const response = await appInstance.inject({ + method: 'POST', + url: '/vectors/ListIndexes', + payload: { + vectorBucketName: vectorBucketName, + }, + }) + + expect(response.statusCode).toBe(403) + }) + + it('should validate required fields', async () => { + const response = await appInstance.inject({ + method: 'POST', + url: '/vectors/ListIndexes', + headers: { + authorization: `Bearer ${serviceToken}`, + }, + payload: {}, + }) + + expect(response.statusCode).toBe(400) + }) + }) + + describe('POST /vectors/GetIndex', () => { + let indexName: string + + beforeEach(async () => { + indexName = `test-index-${Date.now()}` + await appInstance.inject({ + method: 'POST', + url: '/vectors/CreateIndex', + headers: { + authorization: `Bearer ${serviceToken}`, + }, + payload: { + dataType: 'float32', + dimension: 1536, + distanceMetric: 'cosine', + indexName: indexName, + vectorBucketName: vectorBucketName, + metadataConfiguration: { + nonFilterableMetadataKeys: ['key1'], + }, + }, + }) + }) + + it('should get index details successfully', async () => { + const response = await appInstance.inject({ + method: 'POST', + url: '/vectors/GetIndex', + headers: { + authorization: `Bearer ${serviceToken}`, + }, + payload: { + indexName: indexName, + vectorBucketName: vectorBucketName, + }, + }) + + expect(response.statusCode).toBe(200) + const body = JSON.parse(response.body) + expect(body.index).toBeDefined() + expect(body.index.indexName).toBe(indexName) + expect(body.index.vectorBucketName).toBe(vectorBucketName) + expect(body.index.dataType).toBe('float32') + expect(body.index.dimension).toBe(1536) + expect(body.index.distanceMetric).toBe('cosine') + expect(body.index.metadataConfiguration).toEqual({ + nonFilterableMetadataKeys: ['key1'], + }) + expect(body.index.creationTime).toBeDefined() + expect(typeof body.index.creationTime).toBe('number') + }) + + it('should require authentication with service role', async () => { + const response = await appInstance.inject({ + method: 'POST', + url: '/vectors/GetIndex', + payload: { + indexName: indexName, + vectorBucketName: vectorBucketName, + }, + }) + + expect(response.statusCode).toBe(403) + }) + + it('should validate required fields', async () => { + const response = await appInstance.inject({ + method: 'POST', + url: '/vectors/GetIndex', + headers: { + authorization: `Bearer ${serviceToken}`, + }, + payload: { + indexName: indexName, + }, + }) + + expect(response.statusCode).toBe(400) + }) + + it('should validate indexName pattern', async () => { + const response = await appInstance.inject({ + method: 'POST', + url: '/vectors/GetIndex', + headers: { + authorization: `Bearer ${serviceToken}`, + }, + payload: { + indexName: 'INVALID_NAME', + vectorBucketName: vectorBucketName, + }, + }) + + expect(response.statusCode).toBe(400) + }) + + it('should handle non-existent index', async () => { + const response = await appInstance.inject({ + method: 'POST', + url: '/vectors/GetIndex', + headers: { + authorization: `Bearer ${serviceToken}`, + }, + payload: { + indexName: 'non-existent-index', + vectorBucketName: vectorBucketName, + }, + }) + + expect(response.statusCode).toBe(404) + }) + }) + + describe('POST /vectors/PutVectors', () => { + let indexName: string + + beforeEach(async () => { + indexName = `test-index-${Date.now()}` + await appInstance.inject({ + method: 'POST', + url: '/vectors/CreateIndex', + headers: { + authorization: `Bearer ${serviceToken}`, + }, + payload: { + dataType: 'float32', + dimension: 3, + distanceMetric: 'cosine', + indexName: indexName, + vectorBucketName: vectorBucketName, + }, + }) + + mockVectorStore.putVectors.mockResolvedValue({ + vectorKeys: [{ key: 'vec1' }, { key: 'vec2' }], + } as PutVectorsOutput) + }) + + it('should put vectors successfully', async () => { + const response = await appInstance.inject({ + method: 'POST', + url: '/vectors/PutVectors', + headers: { + authorization: `Bearer ${serviceToken}`, + }, + payload: { + vectorBucketName: vectorBucketName, + indexName: indexName, + vectors: [ + { + key: 'vec1', + data: { + float32: [1.0, 2.0, 3.0], + }, + metadata: { + category: 'test', + }, + }, + { + data: { + float32: [4.0, 5.0, 6.0], + }, + }, + ], + }, + }) + + expect(response.statusCode).toBe(200) + + // Verify putVectors was called with correct parameters + expect(mockVectorStore.putVectors).toHaveBeenCalledWith({ + vectorBucketName: vectorBucketS3, + indexName: `${tenantId}-${indexName}`, + vectors: [ + { + key: 'vec1', + data: { + float32: [1.0, 2.0, 3.0], + }, + metadata: { + category: 'test', + }, + }, + { + data: { + float32: [4.0, 5.0, 6.0], + }, + key: undefined, + }, + ], + }) + }) + + it('should require authentication with service role', async () => { + const response = await appInstance.inject({ + method: 'POST', + url: '/vectors/PutVectors', + payload: { + vectorBucketName: vectorBucketName, + indexName: indexName, + vectors: [ + { + data: { + float32: [1.0, 2.0, 3.0], + }, + }, + ], + }, + }) + + expect(response.statusCode).toBe(403) + }) + + it('should validate required fields', async () => { + const response = await appInstance.inject({ + method: 'POST', + url: '/vectors/PutVectors', + headers: { + authorization: `Bearer ${serviceToken}`, + }, + payload: { + vectorBucketName: vectorBucketName, + indexName: indexName, + }, + }) + + expect(response.statusCode).toBe(400) + }) + + it('should validate vector data structure', async () => { + const response = await appInstance.inject({ + method: 'POST', + url: '/vectors/PutVectors', + headers: { + authorization: `Bearer ${serviceToken}`, + }, + payload: { + vectorBucketName: vectorBucketName, + indexName: indexName, + vectors: [ + { + data: { + // missing float32 + }, + }, + ], + }, + }) + + expect(response.statusCode).toBe(400) + }) + + it('should validate maxItems limit', async () => { + const tooManyVectors = Array.from({ length: 501 }, (_, i) => ({ + data: { + float32: [1.0, 2.0, 3.0], + }, + })) + + const response = await appInstance.inject({ + method: 'POST', + url: '/vectors/PutVectors', + headers: { + authorization: `Bearer ${serviceToken}`, + }, + payload: { + vectorBucketName: vectorBucketName, + indexName: indexName, + vectors: tooManyVectors, + }, + }) + + expect(response.statusCode).toBe(400) + }) + + it('should handle non-existent index', async () => { + const response = await appInstance.inject({ + method: 'POST', + url: '/vectors/PutVectors', + headers: { + authorization: `Bearer ${serviceToken}`, + }, + payload: { + vectorBucketName: vectorBucketName, + indexName: 'non-existent-index', + vectors: [ + { + data: { + float32: [1.0, 2.0, 3.0], + }, + }, + ], + }, + }) + + expect(response.statusCode).toBe(404) + }) + }) + + describe('POST /vectors/QueryVectors', () => { + let indexName: string + + beforeEach(async () => { + indexName = `test-index-${Date.now()}` + await appInstance.inject({ + method: 'POST', + url: '/vectors/CreateIndex', + headers: { + authorization: `Bearer ${serviceToken}`, + }, + payload: { + dataType: 'float32', + dimension: 3, + distanceMetric: 'cosine', + indexName: indexName, + vectorBucketName: vectorBucketName, + }, + }) + + mockVectorStore.queryVectors.mockResolvedValue({ + vectors: [ + { + key: 'vec1', + distance: 0.95, + }, + ], + } as QueryVectorsOutput) + }) + + it('should query vectors successfully', async () => { + const response = await appInstance.inject({ + method: 'POST', + url: '/vectors/QueryVectors', + headers: { + authorization: `Bearer ${serviceToken}`, + }, + payload: { + vectorBucketName: vectorBucketName, + indexName: indexName, + queryVector: { + float32: [1.0, 2.0, 3.0], + }, + topK: 10, + returnDistance: true, + returnMetadata: true, + }, + }) + + expect(response.statusCode).toBe(200) + const body = JSON.parse(response.body) + expect(body.vectors).toBeDefined() + + // Verify queryVectors was called with correct parameters + expect(mockVectorStore.queryVectors).toHaveBeenCalledWith({ + vectorBucketName: vectorBucketS3, + indexName: `${tenantId}-${indexName}`, + indexArn: undefined, + queryVector: { + float32: [1.0, 2.0, 3.0], + }, + topK: 10, + returnDistance: true, + returnMetadata: true, + filter: undefined, + }) + }) + + it('should support metadata filtering', async () => { + const response = await appInstance.inject({ + method: 'POST', + url: '/vectors/QueryVectors', + headers: { + authorization: `Bearer ${serviceToken}`, + }, + payload: { + vectorBucketName: vectorBucketName, + indexName: indexName, + queryVector: { + float32: [1.0, 2.0, 3.0], + }, + topK: 5, + filter: { + category: 'test', + }, + }, + }) + + expect(response.statusCode).toBe(200) + expect(mockVectorStore.queryVectors).toHaveBeenCalledWith( + expect.objectContaining({ + filter: { + category: 'test', + }, + }) + ) + }) + + it('should support complex logical filters', async () => { + const response = await appInstance.inject({ + method: 'POST', + url: '/vectors/QueryVectors', + headers: { + authorization: `Bearer ${serviceToken}`, + }, + payload: { + vectorBucketName: vectorBucketName, + indexName: indexName, + queryVector: { + float32: [1.0, 2.0, 3.0], + }, + topK: 5, + filter: { + $and: [{ category: 'test' }, { score: { $gt: 0.5 } }], + }, + }, + }) + + expect(response.statusCode).toBe(200) + expect(mockVectorStore.queryVectors).toHaveBeenCalledWith( + expect.objectContaining({ + filter: { + $and: [{ category: 'test' }, { score: { $gt: 0.5 } }], + }, + }) + ) + }) + + it('should require authentication with service role', async () => { + const response = await appInstance.inject({ + method: 'POST', + url: '/vectors/QueryVectors', + payload: { + vectorBucketName: vectorBucketName, + indexName: indexName, + queryVector: { + float32: [1.0, 2.0, 3.0], + }, + topK: 10, + }, + }) + + expect(response.statusCode).toBe(403) + }) + + it('should validate required fields', async () => { + const response = await appInstance.inject({ + method: 'POST', + url: '/vectors/QueryVectors', + headers: { + authorization: `Bearer ${serviceToken}`, + }, + payload: { + vectorBucketName: vectorBucketName, + indexName: indexName, + queryVector: { + float32: [1.0, 2.0, 3.0], + }, + }, + }) + + expect(response.statusCode).toBe(400) + }) + + it('should validate queryVector structure', async () => { + const response = await appInstance.inject({ + method: 'POST', + url: '/vectors/QueryVectors', + headers: { + authorization: `Bearer ${serviceToken}`, + }, + payload: { + vectorBucketName: vectorBucketName, + indexName: indexName, + queryVector: { + // missing float32 + }, + topK: 10, + }, + }) + + expect(response.statusCode).toBe(400) + }) + + it('should handle non-existent index', async () => { + const response = await appInstance.inject({ + method: 'POST', + url: '/vectors/QueryVectors', + headers: { + authorization: `Bearer ${serviceToken}`, + }, + payload: { + vectorBucketName: vectorBucketName, + indexName: 'non-existent-index', + queryVector: { + float32: [1.0, 2.0, 3.0], + }, + topK: 10, + }, + }) + + expect(response.statusCode).toBe(404) + }) + }) + + describe('POST /vectors/DeleteVectors', () => { + let indexName: string + + beforeEach(async () => { + indexName = `test-index-${Date.now()}` + await appInstance.inject({ + method: 'POST', + url: '/vectors/CreateIndex', + headers: { + authorization: `Bearer ${serviceToken}`, + }, + payload: { + dataType: 'float32', + dimension: 3, + distanceMetric: 'cosine', + indexName: indexName, + vectorBucketName: vectorBucketName, + }, + }) + + mockVectorStore.deleteVectors.mockResolvedValue({} as DeleteVectorsOutput) + }) + + it('should delete vectors successfully', async () => { + const response = await appInstance.inject({ + method: 'POST', + url: '/vectors/DeleteVectors', + headers: { + authorization: `Bearer ${serviceToken}`, + }, + payload: { + vectorBucketName: vectorBucketName, + indexName: indexName, + keys: ['vec1', 'vec2', 'vec3'], + }, + }) + + expect(response.statusCode).toBe(200) + + // Verify deleteVectors was called with correct parameters + expect(mockVectorStore.deleteVectors).toHaveBeenCalledWith({ + vectorBucketName: vectorBucketS3, + indexName: `${tenantId}-${indexName}`, + keys: ['vec1', 'vec2', 'vec3'], + }) + }) + + it('should require authentication with service role', async () => { + const response = await appInstance.inject({ + method: 'POST', + url: '/vectors/DeleteVectors', + payload: { + vectorBucketName: vectorBucketName, + indexName: indexName, + keys: ['vec1'], + }, + }) + + expect(response.statusCode).toBe(403) + }) + + it('should validate required fields', async () => { + const response = await appInstance.inject({ + method: 'POST', + url: '/vectors/DeleteVectors', + headers: { + authorization: `Bearer ${serviceToken}`, + }, + payload: { + vectorBucketName: vectorBucketName, + indexName: indexName, + }, + }) + + expect(response.statusCode).toBe(400) + }) + + it('should handle non-existent index', async () => { + const response = await appInstance.inject({ + method: 'POST', + url: '/vectors/DeleteVectors', + headers: { + authorization: `Bearer ${serviceToken}`, + }, + payload: { + vectorBucketName: vectorBucketName, + indexName: 'non-existent-index', + keys: ['vec1'], + }, + }) + + expect(response.statusCode).toBe(404) + }) + }) + + describe('POST /vectors/ListVectors', () => { + let indexName: string + + beforeEach(async () => { + indexName = `test-index-${Date.now()}` + await appInstance.inject({ + method: 'POST', + url: '/vectors/CreateIndex', + headers: { + authorization: `Bearer ${serviceToken}`, + }, + payload: { + dataType: 'float32', + dimension: 3, + distanceMetric: 'cosine', + indexName: indexName, + vectorBucketName: vectorBucketName, + }, + }) + + mockVectorStore.listVectors.mockResolvedValue({ + vectors: [{ key: 'vec1' }, { key: 'vec2' }, { key: 'vec3' }], + nextToken: undefined, + } as ListVectorsOutput) + }) + + it('should list vectors successfully', async () => { + const response = await appInstance.inject({ + method: 'POST', + url: '/vectors/ListVectors', + headers: { + authorization: `Bearer ${serviceToken}`, + }, + payload: { + vectorBucketName: vectorBucketName, + indexName: indexName, + }, + }) + + expect(response.statusCode).toBe(200) + const body = JSON.parse(response.body) + expect(body.vectors).toBeDefined() + expect(Array.isArray(body.vectors)).toBe(true) + + // Verify listVectors was called with correct parameters + expect(mockVectorStore.listVectors).toHaveBeenCalledWith( + expect.objectContaining({ + vectorBucketName: vectorBucketS3, + indexName: `${tenantId}-${indexName}`, + }) + ) + }) + + it('should support maxResults parameter', async () => { + const response = await appInstance.inject({ + method: 'POST', + url: '/vectors/ListVectors', + headers: { + authorization: `Bearer ${serviceToken}`, + }, + payload: { + vectorBucketName: vectorBucketName, + indexName: indexName, + maxResults: 10, + }, + }) + + expect(response.statusCode).toBe(200) + expect(mockVectorStore.listVectors).toHaveBeenCalledWith( + expect.objectContaining({ + maxResults: 10, + }) + ) + }) + + it('should support returnData and returnMetadata flags', async () => { + const response = await appInstance.inject({ + method: 'POST', + url: '/vectors/ListVectors', + headers: { + authorization: `Bearer ${serviceToken}`, + }, + payload: { + vectorBucketName: vectorBucketName, + indexName: indexName, + returnData: true, + returnMetadata: true, + }, + }) + + expect(response.statusCode).toBe(200) + expect(mockVectorStore.listVectors).toHaveBeenCalledWith( + expect.objectContaining({ + returnData: true, + returnMetadata: true, + }) + ) + }) + + it('should support pagination with nextToken', async () => { + const response = await appInstance.inject({ + method: 'POST', + url: '/vectors/ListVectors', + headers: { + authorization: `Bearer ${serviceToken}`, + }, + payload: { + vectorBucketName: vectorBucketName, + indexName: indexName, + nextToken: 'some-token', + }, + }) + + expect(response.statusCode).toBe(200) + expect(mockVectorStore.listVectors).toHaveBeenCalledWith( + expect.objectContaining({ + nextToken: 'some-token', + }) + ) + }) + + it('should support segmentation parameters', async () => { + const response = await appInstance.inject({ + method: 'POST', + url: '/vectors/ListVectors', + headers: { + authorization: `Bearer ${serviceToken}`, + }, + payload: { + vectorBucketName: vectorBucketName, + indexName: indexName, + segmentCount: 4, + segmentIndex: 2, + }, + }) + + expect(response.statusCode).toBe(200) + expect(mockVectorStore.listVectors).toHaveBeenCalledWith( + expect.objectContaining({ + segmentCount: 4, + segmentIndex: 2, + }) + ) + }) + + it('should require authentication with service role', async () => { + const response = await appInstance.inject({ + method: 'POST', + url: '/vectors/ListVectors', + payload: { + vectorBucketName: vectorBucketName, + indexName: indexName, + }, + }) + + expect(response.statusCode).toBe(403) + }) + + it('should validate required fields', async () => { + const response = await appInstance.inject({ + method: 'POST', + url: '/vectors/ListVectors', + headers: { + authorization: `Bearer ${serviceToken}`, + }, + payload: { + vectorBucketName: vectorBucketName, + }, + }) + + expect(response.statusCode).toBe(400) + }) + + it('should validate maxResults range', async () => { + const response = await appInstance.inject({ + method: 'POST', + url: '/vectors/ListVectors', + headers: { + authorization: `Bearer ${serviceToken}`, + }, + payload: { + vectorBucketName: vectorBucketName, + indexName: indexName, + maxResults: 501, + }, + }) + + expect(response.statusCode).toBe(400) + }) + + it('should validate segmentIndex range', async () => { + const response = await appInstance.inject({ + method: 'POST', + url: '/vectors/ListVectors', + headers: { + authorization: `Bearer ${serviceToken}`, + }, + payload: { + vectorBucketName: vectorBucketName, + indexName: indexName, + segmentCount: 4, + segmentIndex: 16, + }, + }) + + expect(response.statusCode).toBe(400) + }) + + it('should handle non-existent index', async () => { + const response = await appInstance.inject({ + method: 'POST', + url: '/vectors/ListVectors', + headers: { + authorization: `Bearer ${serviceToken}`, + }, + payload: { + vectorBucketName: vectorBucketName, + indexName: 'non-existent-index', + }, + }) + + expect(response.statusCode).toBe(404) + }) + }) + + describe('POST /vectors/GetVectors', () => { + let indexName: string + + beforeEach(async () => { + indexName = `test-index-${Date.now()}` + await appInstance.inject({ + method: 'POST', + url: '/vectors/CreateIndex', + headers: { + authorization: `Bearer ${serviceToken}`, + }, + payload: { + dataType: 'float32', + dimension: 3, + distanceMetric: 'cosine', + indexName: indexName, + vectorBucketName: vectorBucketName, + }, + }) + + mockVectorStore.getVectors.mockResolvedValue({ + vectors: [ + { + key: 'vec1', + data: { float32: [1.0, 2.0, 3.0] }, + metadata: { category: 'test' }, + }, + { + key: 'vec2', + data: { float32: [4.0, 5.0, 6.0] }, + metadata: { category: 'test2' }, + }, + ], + $metadata: { + httpStatusCode: 200, + }, + } as GetVectorsCommandOutput) + }) + + it('should get vectors successfully', async () => { + const response = await appInstance.inject({ + method: 'POST', + url: '/vectors/GetVectors', + headers: { + authorization: `Bearer ${serviceToken}`, + }, + payload: { + vectorBucketName: vectorBucketName, + indexName: indexName, + keys: ['vec1', 'vec2'], + returnData: true, + returnMetadata: true, + }, + }) + + expect(response.statusCode).toBe(200) + const body = JSON.parse(response.body) + expect(body.vectors).toBeDefined() + expect(Array.isArray(body.vectors)).toBe(true) + + // Verify getVectors was called with correct parameters + expect(mockVectorStore.getVectors).toHaveBeenCalledWith({ + vectorBucketName: vectorBucketS3, + indexName: `${tenantId}-${indexName}`, + keys: ['vec1', 'vec2'], + returnData: true, + returnMetadata: true, + }) + }) + + it('should work with default returnData and returnMetadata', async () => { + const response = await appInstance.inject({ + method: 'POST', + url: '/vectors/GetVectors', + headers: { + authorization: `Bearer ${serviceToken}`, + }, + payload: { + vectorBucketName: vectorBucketName, + indexName: indexName, + keys: ['vec1'], + }, + }) + + expect(response.statusCode).toBe(200) + expect(mockVectorStore.getVectors).toHaveBeenCalledWith( + expect.objectContaining({ + returnData: false, + returnMetadata: false, + }) + ) + }) + + it('should require authentication with service role', async () => { + const response = await appInstance.inject({ + method: 'POST', + url: '/vectors/GetVectors', + payload: { + vectorBucketName: vectorBucketName, + indexName: indexName, + keys: ['vec1'], + }, + }) + + expect(response.statusCode).toBe(403) + }) + + it('should validate required fields', async () => { + const response = await appInstance.inject({ + method: 'POST', + url: '/vectors/GetVectors', + headers: { + authorization: `Bearer ${serviceToken}`, + }, + payload: { + vectorBucketName: vectorBucketName, + indexName: indexName, + }, + }) + + expect(response.statusCode).toBe(400) + }) + + it('should handle non-existent index', async () => { + const response = await appInstance.inject({ + method: 'POST', + url: '/vectors/GetVectors', + headers: { + authorization: `Bearer ${serviceToken}`, + }, + payload: { + vectorBucketName: vectorBucketName, + indexName: 'non-existent-index', + keys: ['vec1'], + }, + }) + + expect(response.statusCode).toBe(404) + }) + }) +}) diff --git a/src/test/x-forwarded-host.test.ts b/src/test/x-forwarded-host.test.ts index 809491f7a..3b752f8a2 100644 --- a/src/test/x-forwarded-host.test.ts +++ b/src/test/x-forwarded-host.test.ts @@ -31,6 +31,11 @@ jest.spyOn(tenant, 'getTenantConfig').mockImplementation(async () => ({ maxNamespaces: 30, maxTables: 20, }, + vectorBuckets: { + enabled: true, + maxBuckets: 5, + maxIndexes: 10, + }, }, })) From a879634ced63ae6725ee346dfe936b840cf16758 Mon Sep 17 00:00:00 2001 From: fenos Date: Tue, 28 Oct 2025 10:17:20 +0100 Subject: [PATCH 2/2] feat: sharding of vector buckets --- .eslintrc.js | 6 +- .github/workflows/ci.yml | 2 +- .gitignore | 3 +- .../multitenant/0021-sharding-resources.sql | 78 + migrations/tenant/0045-vector-buckets.sql | 1 - package-lock.json | 2046 +++++++++-------- package.json | 4 +- src/config.ts | 4 +- src/http/plugins/vector.ts | 17 +- src/http/routes/iceberg/bucket.ts | 11 +- src/http/routes/iceberg/table.ts | 190 +- src/http/routes/vector/index.ts | 1 + src/http/routes/vector/put-vectors.ts | 4 +- src/internal/errors/codes.ts | 8 + src/internal/hashing/index.ts | 1 + src/internal/hashing/string-to-int.ts | 9 + src/internal/monitoring/logger.ts | 8 +- src/internal/queue/event.ts | 40 +- src/internal/sharding/architecture.md | 471 ++++ src/internal/sharding/errors.ts | 34 + src/internal/sharding/index.ts | 5 + src/internal/sharding/knex.ts | 396 ++++ src/internal/sharding/sharder.ts | 41 + src/internal/sharding/store.ts | 104 + src/internal/sharding/strategy/catalog.ts | 335 +++ .../sharding/strategy/single-shard.ts | 84 + src/start/server.ts | 24 +- src/start/worker.ts | 5 +- src/storage/database/knex.ts | 13 +- src/storage/events/index.ts | 1 - .../events/lifecycle/bucket-deleted.ts | 5 + src/storage/events/vectors/reconcile.ts | 46 - src/storage/limits.ts | 2 +- .../iceberg/catalog/rest-catalog-client.ts | 47 +- src/storage/protocols/iceberg/knex.ts | 18 +- .../protocols/vector/adapter/s3-vector.ts | 70 +- src/storage/protocols/vector/knex.ts | 53 +- src/storage/protocols/vector/vector-store.ts | 166 +- src/storage/storage.ts | 3 +- src/test/sharding.test.ts | 778 +++++++ src/test/vectors.test.ts | 232 +- 41 files changed, 4075 insertions(+), 1291 deletions(-) create mode 100644 migrations/multitenant/0021-sharding-resources.sql create mode 100644 src/internal/hashing/index.ts create mode 100644 src/internal/hashing/string-to-int.ts create mode 100644 src/internal/sharding/architecture.md create mode 100644 src/internal/sharding/errors.ts create mode 100644 src/internal/sharding/index.ts create mode 100644 src/internal/sharding/knex.ts create mode 100644 src/internal/sharding/sharder.ts create mode 100644 src/internal/sharding/store.ts create mode 100644 src/internal/sharding/strategy/catalog.ts create mode 100644 src/internal/sharding/strategy/single-shard.ts delete mode 100644 src/storage/events/vectors/reconcile.ts create mode 100644 src/test/sharding.test.ts diff --git a/.eslintrc.js b/.eslintrc.js index bd385f7ab..7bce02421 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -1,18 +1,18 @@ module.exports = { - ignorePatterns: ['src/test/assets/**', 'src/test/db/**', 'src/test/*.yaml'], + ignorePatterns: ['src/test/assets/**', 'src/test/db/**', 'src/test/*.yaml', 'src/**/**/*.md'], parser: '@typescript-eslint/parser', extends: ['plugin:@typescript-eslint/recommended', 'plugin:prettier/recommended'], parserOptions: { ecmaVersion: 2020, // Allows for the parsing of modern ECMAScript features sourceType: 'module', // Allows for the use of imports - "project": "./tsconfig.json", + project: './tsconfig.json', }, rules: { '@typescript-eslint/no-floating-promises': 'error', '@typescript-eslint/no-explicit-any': 'warn', '@typescript-eslint/no-unused-vars': [ 'warn', - { 'argsIgnorePattern': '^_+$', 'varsIgnorePattern': '^_+$' } // allows intentionally unused variables named _ + { argsIgnorePattern: '^_+$', varsIgnorePattern: '^_+$' }, // allows intentionally unused variables named _ ], '@typescript-eslint/no-require-imports': 'warn', }, diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 98f00789c..dc10147e8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -82,7 +82,7 @@ jobs: MULTI_TENANT: false S3_PROTOCOL_ACCESS_KEY_ID: ${{ secrets.TENANT_ID }} S3_PROTOCOL_ACCESS_KEY_SECRET: ${{ secrets.SERVICE_KEY }} - VECTOR_BUCKET_S3: supa-test-local-dev + VECTOR_S3_BUCKETS: supa-test-local-dev VECTOR_ENABLED: true ICEBERG_ENABLED: true diff --git a/.gitignore b/.gitignore index 7b565b162..5b4e62bb7 100644 --- a/.gitignore +++ b/.gitignore @@ -8,4 +8,5 @@ static/api.json data/ bin/ coverage/ -.idea/ \ No newline at end of file +.idea/ +src/scripts/*.py \ No newline at end of file diff --git a/migrations/multitenant/0021-sharding-resources.sql b/migrations/multitenant/0021-sharding-resources.sql new file mode 100644 index 000000000..c1fee4797 --- /dev/null +++ b/migrations/multitenant/0021-sharding-resources.sql @@ -0,0 +1,78 @@ + + +-- Main shards table. +CREATE TABLE IF NOT EXISTS shard ( + id BIGSERIAL PRIMARY KEY, + kind TEXT NOT NULL, + shard_key TEXT NOT NULL, + capacity INT NOT NULL DEFAULT 10000, + next_slot INT NOT NULL DEFAULT 0, + status TEXT NOT NULL DEFAULT 'active', -- active|draining|disabled + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + UNIQUE (kind, shard_key) +); + +-- Sparse slot rows: only the slots that have ever been used exist here. +-- A "free" slot is a row with reservation_id NULL and resource_id NULL. +CREATE TABLE IF NOT EXISTS shard_slots ( + shard_id BIGINT NOT NULL REFERENCES shard(id) ON DELETE CASCADE, + slot_no INT NOT NULL, + tenant_id TEXT, + resource_id TEXT, -- set when confirmed + PRIMARY KEY (shard_id, slot_no) +); + +-- Reservations with short leases +CREATE TABLE IF NOT EXISTS shard_reservation ( + id UUID PRIMARY KEY default gen_random_uuid(), + kind text NOT NULL, + tenant_id TEXT, + resource_id TEXT NOT NULL, -- e.g. "vector::bucket::name" + shard_id BIGINT NOT NULL, + slot_no INT NOT NULL, + status TEXT NOT NULL DEFAULT 'pending', -- pending|confirmed|expired|cancelled + lease_expires_at TIMESTAMPTZ NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + UNIQUE (kind, resource_id), + UNIQUE (shard_id, slot_no) +); + +-- Fast “used count” per shard +CREATE INDEX IF NOT EXISTS shard_slots_used_idx + ON shard_slots (shard_id) + WHERE resource_id IS NOT NULL; + +ALTER TABLE shard + ADD CONSTRAINT shard_capacity_not_less_than_minted + CHECK (capacity >= next_slot); + + +-- Create index for counting slots by tenant +CREATE INDEX IF NOT EXISTS shard_slots_tenant_id_idx + ON shard_slots (tenant_id); + +-- Create index for counting reservations by tenant +CREATE INDEX IF NOT EXISTS shard_reservation_tenant_id_idx + ON shard_reservation (tenant_id); + +-- Create index for counting used slots by tenant +CREATE INDEX IF NOT EXISTS shard_slots_tenant_resource_idx + ON shard_slots (tenant_id, shard_id) + WHERE resource_id IS NOT NULL; + + +ALTER TABLE shard_reservation + ADD CONSTRAINT fk_shard_slot + FOREIGN KEY (shard_id, slot_no) + REFERENCES shard_slots(shard_id, slot_no) + ON DELETE RESTRICT; + + +CREATE INDEX IF NOT EXISTS shard_slots_free_idx + ON shard_slots (shard_id, slot_no) + WHERE resource_id IS NULL; + +-- Add index for finding active reservations by slot +CREATE INDEX IF NOT EXISTS shard_reservation_active_slot_idx + ON shard_reservation (shard_id, slot_no, lease_expires_at) + WHERE status = 'pending'; \ No newline at end of file diff --git a/migrations/tenant/0045-vector-buckets.sql b/migrations/tenant/0045-vector-buckets.sql index ff5108ca3..23152dfd0 100644 --- a/migrations/tenant/0045-vector-buckets.sql +++ b/migrations/tenant/0045-vector-buckets.sql @@ -20,7 +20,6 @@ DO $$ dimension integer NOT NULL, distance_metric text NOT NULL, metadata_configuration jsonb NULL, - status text NOT NULL default 'PENDING', created_at timestamptz NOT NULL default now(), updated_at timestamptz NOT NULL default now() ); diff --git a/package-lock.json b/package-lock.json index 67997c10d..13521652f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11,7 +11,7 @@ "dependencies": { "@aws-sdk/client-ecs": "^3.795.0", "@aws-sdk/client-s3": "3.654.0", - "@aws-sdk/client-s3vectors": "^3.883.0", + "@aws-sdk/client-s3vectors": "^3.919.0", "@aws-sdk/lib-storage": "3.654.0", "@aws-sdk/s3-request-presigner": "3.654.0", "@fastify/accepts": "^4.3.0", @@ -53,6 +53,7 @@ "ioredis": "^5.2.4", "ip-address": "^10.0.1", "jose": "^6.0.10", + "json-bigint": "^1.0.0", "knex": "^3.1.0", "lru-cache": "^10.2.0", "md5-file": "^5.0.0", @@ -82,6 +83,7 @@ "@types/glob": "^8.1.0", "@types/jest": "^29.2.1", "@types/js-yaml": "^4.0.5", + "@types/json-bigint": "^1.0.4", "@types/multistream": "^4.1.3", "@types/mustache": "^4.2.2", "@types/node": "^22.18.8", @@ -1650,48 +1652,48 @@ } }, "node_modules/@aws-sdk/client-s3vectors": { - "version": "3.883.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-s3vectors/-/client-s3vectors-3.883.0.tgz", - "integrity": "sha512-QtGyHqvih3it25HFnxudHg6J6FvlpSQs9aSN2IgpofAQGLENGVtKFtVlFay2KOlUFPh5aL1OOl5ONR70O8q/Eg==", + "version": "3.919.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-s3vectors/-/client-s3vectors-3.919.0.tgz", + "integrity": "sha512-zEGUf1Nhijx5ldZJYSusrXCHMOOCZ53YYif4HYCfveUD3xp0ZmWuRnhlhGU/UDykZEa4OzZx7nxK+4UkcnKhdA==", "dependencies": { "@aws-crypto/sha256-browser": "5.2.0", "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "3.883.0", - "@aws-sdk/credential-provider-node": "3.883.0", - "@aws-sdk/middleware-host-header": "3.873.0", - "@aws-sdk/middleware-logger": "3.876.0", - "@aws-sdk/middleware-recursion-detection": "3.873.0", - "@aws-sdk/middleware-user-agent": "3.883.0", - "@aws-sdk/region-config-resolver": "3.873.0", - "@aws-sdk/types": "3.862.0", - "@aws-sdk/util-endpoints": "3.879.0", - "@aws-sdk/util-user-agent-browser": "3.873.0", - "@aws-sdk/util-user-agent-node": "3.883.0", - "@smithy/config-resolver": "^4.1.5", - "@smithy/core": "^3.9.2", - "@smithy/fetch-http-handler": "^5.1.1", - "@smithy/hash-node": "^4.0.5", - "@smithy/invalid-dependency": "^4.0.5", - "@smithy/middleware-content-length": "^4.0.5", - "@smithy/middleware-endpoint": "^4.1.21", - "@smithy/middleware-retry": "^4.1.22", - "@smithy/middleware-serde": "^4.0.9", - "@smithy/middleware-stack": "^4.0.5", - "@smithy/node-config-provider": "^4.1.4", - "@smithy/node-http-handler": "^4.1.1", - "@smithy/protocol-http": "^5.1.3", - "@smithy/smithy-client": "^4.5.2", - "@smithy/types": "^4.3.2", - "@smithy/url-parser": "^4.0.5", - "@smithy/util-base64": "^4.0.0", - "@smithy/util-body-length-browser": "^4.0.0", - "@smithy/util-body-length-node": "^4.0.0", - "@smithy/util-defaults-mode-browser": "^4.0.29", - "@smithy/util-defaults-mode-node": "^4.0.29", - "@smithy/util-endpoints": "^3.0.7", - "@smithy/util-middleware": "^4.0.5", - "@smithy/util-retry": "^4.0.7", - "@smithy/util-utf8": "^4.0.0", + "@aws-sdk/core": "3.916.0", + "@aws-sdk/credential-provider-node": "3.919.0", + "@aws-sdk/middleware-host-header": "3.914.0", + "@aws-sdk/middleware-logger": "3.914.0", + "@aws-sdk/middleware-recursion-detection": "3.919.0", + "@aws-sdk/middleware-user-agent": "3.916.0", + "@aws-sdk/region-config-resolver": "3.914.0", + "@aws-sdk/types": "3.914.0", + "@aws-sdk/util-endpoints": "3.916.0", + "@aws-sdk/util-user-agent-browser": "3.914.0", + "@aws-sdk/util-user-agent-node": "3.916.0", + "@smithy/config-resolver": "^4.4.0", + "@smithy/core": "^3.17.1", + "@smithy/fetch-http-handler": "^5.3.4", + "@smithy/hash-node": "^4.2.3", + "@smithy/invalid-dependency": "^4.2.3", + "@smithy/middleware-content-length": "^4.2.3", + "@smithy/middleware-endpoint": "^4.3.5", + "@smithy/middleware-retry": "^4.4.5", + "@smithy/middleware-serde": "^4.2.3", + "@smithy/middleware-stack": "^4.2.3", + "@smithy/node-config-provider": "^4.3.3", + "@smithy/node-http-handler": "^4.4.3", + "@smithy/protocol-http": "^5.3.3", + "@smithy/smithy-client": "^4.9.1", + "@smithy/types": "^4.8.0", + "@smithy/url-parser": "^4.2.3", + "@smithy/util-base64": "^4.3.0", + "@smithy/util-body-length-browser": "^4.2.0", + "@smithy/util-body-length-node": "^4.2.1", + "@smithy/util-defaults-mode-browser": "^4.3.4", + "@smithy/util-defaults-mode-node": "^4.2.6", + "@smithy/util-endpoints": "^3.2.3", + "@smithy/util-middleware": "^4.2.3", + "@smithy/util-retry": "^4.2.3", + "@smithy/util-utf8": "^4.2.0", "tslib": "^2.6.2" }, "engines": { @@ -1699,47 +1701,47 @@ } }, "node_modules/@aws-sdk/client-s3vectors/node_modules/@aws-sdk/client-sso": { - "version": "3.883.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-sso/-/client-sso-3.883.0.tgz", - "integrity": "sha512-Ybjw76yPceEBO7+VLjy5+/Gr0A1UNymSDHda5w8tfsS2iHZt/vuD6wrYpHdLoUx4H5la8ZhwcSfK/+kmE+QLPw==", + "version": "3.919.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-sso/-/client-sso-3.919.0.tgz", + "integrity": "sha512-9DVw/1DCzZ9G7Jofnhpg/XDC3wdJ3NAJdNWY1TrgE5ZcpTM+UTIQMGyaljCv9rgxggutHBgmBI5lP3YMcPk9ZQ==", "dependencies": { "@aws-crypto/sha256-browser": "5.2.0", "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "3.883.0", - "@aws-sdk/middleware-host-header": "3.873.0", - "@aws-sdk/middleware-logger": "3.876.0", - "@aws-sdk/middleware-recursion-detection": "3.873.0", - "@aws-sdk/middleware-user-agent": "3.883.0", - "@aws-sdk/region-config-resolver": "3.873.0", - "@aws-sdk/types": "3.862.0", - "@aws-sdk/util-endpoints": "3.879.0", - "@aws-sdk/util-user-agent-browser": "3.873.0", - "@aws-sdk/util-user-agent-node": "3.883.0", - "@smithy/config-resolver": "^4.1.5", - "@smithy/core": "^3.9.2", - "@smithy/fetch-http-handler": "^5.1.1", - "@smithy/hash-node": "^4.0.5", - "@smithy/invalid-dependency": "^4.0.5", - "@smithy/middleware-content-length": "^4.0.5", - "@smithy/middleware-endpoint": "^4.1.21", - "@smithy/middleware-retry": "^4.1.22", - "@smithy/middleware-serde": "^4.0.9", - "@smithy/middleware-stack": "^4.0.5", - "@smithy/node-config-provider": "^4.1.4", - "@smithy/node-http-handler": "^4.1.1", - "@smithy/protocol-http": "^5.1.3", - "@smithy/smithy-client": "^4.5.2", - "@smithy/types": "^4.3.2", - "@smithy/url-parser": "^4.0.5", - "@smithy/util-base64": "^4.0.0", - "@smithy/util-body-length-browser": "^4.0.0", - "@smithy/util-body-length-node": "^4.0.0", - "@smithy/util-defaults-mode-browser": "^4.0.29", - "@smithy/util-defaults-mode-node": "^4.0.29", - "@smithy/util-endpoints": "^3.0.7", - "@smithy/util-middleware": "^4.0.5", - "@smithy/util-retry": "^4.0.7", - "@smithy/util-utf8": "^4.0.0", + "@aws-sdk/core": "3.916.0", + "@aws-sdk/middleware-host-header": "3.914.0", + "@aws-sdk/middleware-logger": "3.914.0", + "@aws-sdk/middleware-recursion-detection": "3.919.0", + "@aws-sdk/middleware-user-agent": "3.916.0", + "@aws-sdk/region-config-resolver": "3.914.0", + "@aws-sdk/types": "3.914.0", + "@aws-sdk/util-endpoints": "3.916.0", + "@aws-sdk/util-user-agent-browser": "3.914.0", + "@aws-sdk/util-user-agent-node": "3.916.0", + "@smithy/config-resolver": "^4.4.0", + "@smithy/core": "^3.17.1", + "@smithy/fetch-http-handler": "^5.3.4", + "@smithy/hash-node": "^4.2.3", + "@smithy/invalid-dependency": "^4.2.3", + "@smithy/middleware-content-length": "^4.2.3", + "@smithy/middleware-endpoint": "^4.3.5", + "@smithy/middleware-retry": "^4.4.5", + "@smithy/middleware-serde": "^4.2.3", + "@smithy/middleware-stack": "^4.2.3", + "@smithy/node-config-provider": "^4.3.3", + "@smithy/node-http-handler": "^4.4.3", + "@smithy/protocol-http": "^5.3.3", + "@smithy/smithy-client": "^4.9.1", + "@smithy/types": "^4.8.0", + "@smithy/url-parser": "^4.2.3", + "@smithy/util-base64": "^4.3.0", + "@smithy/util-body-length-browser": "^4.2.0", + "@smithy/util-body-length-node": "^4.2.1", + "@smithy/util-defaults-mode-browser": "^4.3.4", + "@smithy/util-defaults-mode-node": "^4.2.6", + "@smithy/util-endpoints": "^3.2.3", + "@smithy/util-middleware": "^4.2.3", + "@smithy/util-retry": "^4.2.3", + "@smithy/util-utf8": "^4.2.0", "tslib": "^2.6.2" }, "engines": { @@ -1747,24 +1749,22 @@ } }, "node_modules/@aws-sdk/client-s3vectors/node_modules/@aws-sdk/core": { - "version": "3.883.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/core/-/core-3.883.0.tgz", - "integrity": "sha512-FmkqnqBLkXi4YsBPbF6vzPa0m4XKUuvgKDbamfw4DZX2CzfBZH6UU4IwmjNV3ZM38m0xraHarK8KIbGSadN3wg==", - "dependencies": { - "@aws-sdk/types": "3.862.0", - "@aws-sdk/xml-builder": "3.873.0", - "@smithy/core": "^3.9.2", - "@smithy/node-config-provider": "^4.1.4", - "@smithy/property-provider": "^4.0.5", - "@smithy/protocol-http": "^5.1.3", - "@smithy/signature-v4": "^5.1.3", - "@smithy/smithy-client": "^4.5.2", - "@smithy/types": "^4.3.2", - "@smithy/util-base64": "^4.0.0", - "@smithy/util-body-length-browser": "^4.0.0", - "@smithy/util-middleware": "^4.0.5", - "@smithy/util-utf8": "^4.0.0", - "fast-xml-parser": "5.2.5", + "version": "3.916.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/core/-/core-3.916.0.tgz", + "integrity": "sha512-1JHE5s6MD5PKGovmx/F1e01hUbds/1y3X8rD+Gvi/gWVfdg5noO7ZCerpRsWgfzgvCMZC9VicopBqNHCKLykZA==", + "dependencies": { + "@aws-sdk/types": "3.914.0", + "@aws-sdk/xml-builder": "3.914.0", + "@smithy/core": "^3.17.1", + "@smithy/node-config-provider": "^4.3.3", + "@smithy/property-provider": "^4.2.3", + "@smithy/protocol-http": "^5.3.3", + "@smithy/signature-v4": "^5.3.3", + "@smithy/smithy-client": "^4.9.1", + "@smithy/types": "^4.8.0", + "@smithy/util-base64": "^4.3.0", + "@smithy/util-middleware": "^4.2.3", + "@smithy/util-utf8": "^4.2.0", "tslib": "^2.6.2" }, "engines": { @@ -1772,14 +1772,14 @@ } }, "node_modules/@aws-sdk/client-s3vectors/node_modules/@aws-sdk/credential-provider-env": { - "version": "3.883.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-env/-/credential-provider-env-3.883.0.tgz", - "integrity": "sha512-Z6tPBXPCodfhIF1rvQKoeRGMkwL6TK0xdl1UoMIA1x4AfBpPICAF77JkFBExk/pdiFYq1d04Qzddd/IiujSlLg==", + "version": "3.916.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-env/-/credential-provider-env-3.916.0.tgz", + "integrity": "sha512-3gDeqOXcBRXGHScc6xb7358Lyf64NRG2P08g6Bu5mv1Vbg9PKDyCAZvhKLkG7hkdfAM8Yc6UJNhbFxr1ud/tCQ==", "dependencies": { - "@aws-sdk/core": "3.883.0", - "@aws-sdk/types": "3.862.0", - "@smithy/property-provider": "^4.0.5", - "@smithy/types": "^4.3.2", + "@aws-sdk/core": "3.916.0", + "@aws-sdk/types": "3.914.0", + "@smithy/property-provider": "^4.2.3", + "@smithy/types": "^4.8.0", "tslib": "^2.6.2" }, "engines": { @@ -1787,19 +1787,19 @@ } }, "node_modules/@aws-sdk/client-s3vectors/node_modules/@aws-sdk/credential-provider-http": { - "version": "3.883.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-http/-/credential-provider-http-3.883.0.tgz", - "integrity": "sha512-P589ug1lMOOEYLTaQJjSP+Gee34za8Kk2LfteNQfO9SpByHFgGj++Sg8VyIe30eZL8Q+i4qTt24WDCz1c+dgYg==", + "version": "3.916.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-http/-/credential-provider-http-3.916.0.tgz", + "integrity": "sha512-NmooA5Z4/kPFJdsyoJgDxuqXC1C6oPMmreJjbOPqcwo6E/h2jxaG8utlQFgXe5F9FeJsMx668dtxVxSYnAAqHQ==", "dependencies": { - "@aws-sdk/core": "3.883.0", - "@aws-sdk/types": "3.862.0", - "@smithy/fetch-http-handler": "^5.1.1", - "@smithy/node-http-handler": "^4.1.1", - "@smithy/property-provider": "^4.0.5", - "@smithy/protocol-http": "^5.1.3", - "@smithy/smithy-client": "^4.5.2", - "@smithy/types": "^4.3.2", - "@smithy/util-stream": "^4.2.4", + "@aws-sdk/core": "3.916.0", + "@aws-sdk/types": "3.914.0", + "@smithy/fetch-http-handler": "^5.3.4", + "@smithy/node-http-handler": "^4.4.3", + "@smithy/property-provider": "^4.2.3", + "@smithy/protocol-http": "^5.3.3", + "@smithy/smithy-client": "^4.9.1", + "@smithy/types": "^4.8.0", + "@smithy/util-stream": "^4.5.4", "tslib": "^2.6.2" }, "engines": { @@ -1807,22 +1807,22 @@ } }, "node_modules/@aws-sdk/client-s3vectors/node_modules/@aws-sdk/credential-provider-ini": { - "version": "3.883.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-ini/-/credential-provider-ini-3.883.0.tgz", - "integrity": "sha512-n6z9HTzuDEdugXvPiE/95VJXbF4/gBffdV/SRHDJKtDHaRuvp/gggbfmfVSTFouGVnlKPb2pQWQsW3Nr/Y3Lrw==", + "version": "3.919.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-ini/-/credential-provider-ini-3.919.0.tgz", + "integrity": "sha512-fAWVfh0P54UFbyAK4tmIPh/X3COFAyXYSp8b2Pc1R6GRwDDMvrAigwGJuyZS4BmpPlXij1gB0nXbhM5Yo4MMMA==", "dependencies": { - "@aws-sdk/core": "3.883.0", - "@aws-sdk/credential-provider-env": "3.883.0", - "@aws-sdk/credential-provider-http": "3.883.0", - "@aws-sdk/credential-provider-process": "3.883.0", - "@aws-sdk/credential-provider-sso": "3.883.0", - "@aws-sdk/credential-provider-web-identity": "3.883.0", - "@aws-sdk/nested-clients": "3.883.0", - "@aws-sdk/types": "3.862.0", - "@smithy/credential-provider-imds": "^4.0.7", - "@smithy/property-provider": "^4.0.5", - "@smithy/shared-ini-file-loader": "^4.0.5", - "@smithy/types": "^4.3.2", + "@aws-sdk/core": "3.916.0", + "@aws-sdk/credential-provider-env": "3.916.0", + "@aws-sdk/credential-provider-http": "3.916.0", + "@aws-sdk/credential-provider-process": "3.916.0", + "@aws-sdk/credential-provider-sso": "3.919.0", + "@aws-sdk/credential-provider-web-identity": "3.919.0", + "@aws-sdk/nested-clients": "3.919.0", + "@aws-sdk/types": "3.914.0", + "@smithy/credential-provider-imds": "^4.2.3", + "@smithy/property-provider": "^4.2.3", + "@smithy/shared-ini-file-loader": "^4.3.3", + "@smithy/types": "^4.8.0", "tslib": "^2.6.2" }, "engines": { @@ -1830,21 +1830,21 @@ } }, "node_modules/@aws-sdk/client-s3vectors/node_modules/@aws-sdk/credential-provider-node": { - "version": "3.883.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-node/-/credential-provider-node-3.883.0.tgz", - "integrity": "sha512-QIUhsatsrwfB9ZsKpmi0EySSfexVP61wgN7hr493DOileh2QsKW4XATEfsWNmx0dj9323Vg1Mix7bXtRfl9cGg==", + "version": "3.919.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-node/-/credential-provider-node-3.919.0.tgz", + "integrity": "sha512-GL5filyxYS+eZq8ZMQnY5hh79Wxor7Rljo0SUJxZVwEj8cf3zY0MMuwoXU1HQrVabvYtkPDOWSreX8GkIBtBCw==", "dependencies": { - "@aws-sdk/credential-provider-env": "3.883.0", - "@aws-sdk/credential-provider-http": "3.883.0", - "@aws-sdk/credential-provider-ini": "3.883.0", - "@aws-sdk/credential-provider-process": "3.883.0", - "@aws-sdk/credential-provider-sso": "3.883.0", - "@aws-sdk/credential-provider-web-identity": "3.883.0", - "@aws-sdk/types": "3.862.0", - "@smithy/credential-provider-imds": "^4.0.7", - "@smithy/property-provider": "^4.0.5", - "@smithy/shared-ini-file-loader": "^4.0.5", - "@smithy/types": "^4.3.2", + "@aws-sdk/credential-provider-env": "3.916.0", + "@aws-sdk/credential-provider-http": "3.916.0", + "@aws-sdk/credential-provider-ini": "3.919.0", + "@aws-sdk/credential-provider-process": "3.916.0", + "@aws-sdk/credential-provider-sso": "3.919.0", + "@aws-sdk/credential-provider-web-identity": "3.919.0", + "@aws-sdk/types": "3.914.0", + "@smithy/credential-provider-imds": "^4.2.3", + "@smithy/property-provider": "^4.2.3", + "@smithy/shared-ini-file-loader": "^4.3.3", + "@smithy/types": "^4.8.0", "tslib": "^2.6.2" }, "engines": { @@ -1852,15 +1852,15 @@ } }, "node_modules/@aws-sdk/client-s3vectors/node_modules/@aws-sdk/credential-provider-process": { - "version": "3.883.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-process/-/credential-provider-process-3.883.0.tgz", - "integrity": "sha512-m1shbHY/Vppy4EdddG9r8x64TO/9FsCjokp5HbKcZvVoTOTgUJrdT8q2TAQJ89+zYIJDqsKbqfrmfwJ1zOdnGQ==", + "version": "3.916.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-process/-/credential-provider-process-3.916.0.tgz", + "integrity": "sha512-SXDyDvpJ1+WbotZDLJW1lqP6gYGaXfZJrgFSXIuZjHb75fKeNRgPkQX/wZDdUvCwdrscvxmtyJorp2sVYkMcvA==", "dependencies": { - "@aws-sdk/core": "3.883.0", - "@aws-sdk/types": "3.862.0", - "@smithy/property-provider": "^4.0.5", - "@smithy/shared-ini-file-loader": "^4.0.5", - "@smithy/types": "^4.3.2", + "@aws-sdk/core": "3.916.0", + "@aws-sdk/types": "3.914.0", + "@smithy/property-provider": "^4.2.3", + "@smithy/shared-ini-file-loader": "^4.3.3", + "@smithy/types": "^4.8.0", "tslib": "^2.6.2" }, "engines": { @@ -1868,17 +1868,17 @@ } }, "node_modules/@aws-sdk/client-s3vectors/node_modules/@aws-sdk/credential-provider-sso": { - "version": "3.883.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-sso/-/credential-provider-sso-3.883.0.tgz", - "integrity": "sha512-37ve9Tult08HLXrJFHJM/sGB/vO7wzI6v1RUUfeTiShqx8ZQ5fTzCTNY/duO96jCtCexmFNSycpQzh7lDIf0aA==", + "version": "3.919.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-sso/-/credential-provider-sso-3.919.0.tgz", + "integrity": "sha512-oN1XG/frOc2K2KdVwRQjLTBLM1oSFJLtOhuV/6g9N0ASD+44uVJai1CF9JJv5GjHGV+wsqAt+/Dzde0tZEXirA==", "dependencies": { - "@aws-sdk/client-sso": "3.883.0", - "@aws-sdk/core": "3.883.0", - "@aws-sdk/token-providers": "3.883.0", - "@aws-sdk/types": "3.862.0", - "@smithy/property-provider": "^4.0.5", - "@smithy/shared-ini-file-loader": "^4.0.5", - "@smithy/types": "^4.3.2", + "@aws-sdk/client-sso": "3.919.0", + "@aws-sdk/core": "3.916.0", + "@aws-sdk/token-providers": "3.919.0", + "@aws-sdk/types": "3.914.0", + "@smithy/property-provider": "^4.2.3", + "@smithy/shared-ini-file-loader": "^4.3.3", + "@smithy/types": "^4.8.0", "tslib": "^2.6.2" }, "engines": { @@ -1886,15 +1886,16 @@ } }, "node_modules/@aws-sdk/client-s3vectors/node_modules/@aws-sdk/credential-provider-web-identity": { - "version": "3.883.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-web-identity/-/credential-provider-web-identity-3.883.0.tgz", - "integrity": "sha512-SL82K9Jb0vpuTadqTO4Fpdu7SKtebZ3Yo4LZvk/U0UauVMlJj5ZTos0mFx1QSMB9/4TpqifYrSZcdnxgYg8Eqw==", + "version": "3.919.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-web-identity/-/credential-provider-web-identity-3.919.0.tgz", + "integrity": "sha512-Wi7RmyWA8kUJ++/8YceC7U5r4LyvOHGCnJLDHliP8rOC8HLdSgxw/Upeq3WmC+RPw1zyGOtEDRS/caop2xLXEA==", "dependencies": { - "@aws-sdk/core": "3.883.0", - "@aws-sdk/nested-clients": "3.883.0", - "@aws-sdk/types": "3.862.0", - "@smithy/property-provider": "^4.0.5", - "@smithy/types": "^4.3.2", + "@aws-sdk/core": "3.916.0", + "@aws-sdk/nested-clients": "3.919.0", + "@aws-sdk/types": "3.914.0", + "@smithy/property-provider": "^4.2.3", + "@smithy/shared-ini-file-loader": "^4.3.3", + "@smithy/types": "^4.8.0", "tslib": "^2.6.2" }, "engines": { @@ -1902,13 +1903,13 @@ } }, "node_modules/@aws-sdk/client-s3vectors/node_modules/@aws-sdk/middleware-host-header": { - "version": "3.873.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-host-header/-/middleware-host-header-3.873.0.tgz", - "integrity": "sha512-KZ/W1uruWtMOs7D5j3KquOxzCnV79KQW9MjJFZM/M0l6KI8J6V3718MXxFHsTjUE4fpdV6SeCNLV1lwGygsjJA==", + "version": "3.914.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-host-header/-/middleware-host-header-3.914.0.tgz", + "integrity": "sha512-7r9ToySQ15+iIgXMF/h616PcQStByylVkCshmQqcdeynD/lCn2l667ynckxW4+ql0Q+Bo/URljuhJRxVJzydNA==", "dependencies": { - "@aws-sdk/types": "3.862.0", - "@smithy/protocol-http": "^5.1.3", - "@smithy/types": "^4.3.2", + "@aws-sdk/types": "3.914.0", + "@smithy/protocol-http": "^5.3.3", + "@smithy/types": "^4.8.0", "tslib": "^2.6.2" }, "engines": { @@ -1916,12 +1917,12 @@ } }, "node_modules/@aws-sdk/client-s3vectors/node_modules/@aws-sdk/middleware-logger": { - "version": "3.876.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-logger/-/middleware-logger-3.876.0.tgz", - "integrity": "sha512-cpWJhOuMSyz9oV25Z/CMHCBTgafDCbv7fHR80nlRrPdPZ8ETNsahwRgltXP1QJJ8r3X/c1kwpOR7tc+RabVzNA==", + "version": "3.914.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-logger/-/middleware-logger-3.914.0.tgz", + "integrity": "sha512-/gaW2VENS5vKvJbcE1umV4Ag3NuiVzpsANxtrqISxT3ovyro29o1RezW/Avz/6oJqjnmgz8soe9J1t65jJdiNg==", "dependencies": { - "@aws-sdk/types": "3.862.0", - "@smithy/types": "^4.3.2", + "@aws-sdk/types": "3.914.0", + "@smithy/types": "^4.8.0", "tslib": "^2.6.2" }, "engines": { @@ -1929,13 +1930,14 @@ } }, "node_modules/@aws-sdk/client-s3vectors/node_modules/@aws-sdk/middleware-recursion-detection": { - "version": "3.873.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-recursion-detection/-/middleware-recursion-detection-3.873.0.tgz", - "integrity": "sha512-OtgY8EXOzRdEWR//WfPkA/fXl0+WwE8hq0y9iw2caNyKPtca85dzrrZWnPqyBK/cpImosrpR1iKMYr41XshsCg==", + "version": "3.919.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-recursion-detection/-/middleware-recursion-detection-3.919.0.tgz", + "integrity": "sha512-q3MAUxLQve4rTfAannUCx2q1kAHkBBsxt6hVUpzi63KC4lBLScc1ltr7TI+hDxlfGRWGo54jRegb2SsY9Jm+Mw==", "dependencies": { - "@aws-sdk/types": "3.862.0", - "@smithy/protocol-http": "^5.1.3", - "@smithy/types": "^4.3.2", + "@aws-sdk/types": "3.914.0", + "@aws/lambda-invoke-store": "^0.1.1", + "@smithy/protocol-http": "^5.3.3", + "@smithy/types": "^4.8.0", "tslib": "^2.6.2" }, "engines": { @@ -1943,16 +1945,16 @@ } }, "node_modules/@aws-sdk/client-s3vectors/node_modules/@aws-sdk/middleware-user-agent": { - "version": "3.883.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-user-agent/-/middleware-user-agent-3.883.0.tgz", - "integrity": "sha512-q58uLYnGLg7hsnWpdj7Cd1Ulsq1/PUJOHvAfgcBuiDE/+Fwh0DZxZZyjrU+Cr+dbeowIdUaOO8BEDDJ0CUenJw==", + "version": "3.916.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-user-agent/-/middleware-user-agent-3.916.0.tgz", + "integrity": "sha512-mzF5AdrpQXc2SOmAoaQeHpDFsK2GE6EGcEACeNuoESluPI2uYMpuuNMYrUufdnIAIyqgKlis0NVxiahA5jG42w==", "dependencies": { - "@aws-sdk/core": "3.883.0", - "@aws-sdk/types": "3.862.0", - "@aws-sdk/util-endpoints": "3.879.0", - "@smithy/core": "^3.9.2", - "@smithy/protocol-http": "^5.1.3", - "@smithy/types": "^4.3.2", + "@aws-sdk/core": "3.916.0", + "@aws-sdk/types": "3.914.0", + "@aws-sdk/util-endpoints": "3.916.0", + "@smithy/core": "^3.17.1", + "@smithy/protocol-http": "^5.3.3", + "@smithy/types": "^4.8.0", "tslib": "^2.6.2" }, "engines": { @@ -1960,47 +1962,47 @@ } }, "node_modules/@aws-sdk/client-s3vectors/node_modules/@aws-sdk/nested-clients": { - "version": "3.883.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/nested-clients/-/nested-clients-3.883.0.tgz", - "integrity": "sha512-IhzDM+v0ga53GOOrZ9jmGNr7JU5OR6h6ZK9NgB7GXaa+gsDbqfUuXRwyKDYXldrTXf1sUR3vy1okWDXA7S2ejQ==", + "version": "3.919.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/nested-clients/-/nested-clients-3.919.0.tgz", + "integrity": "sha512-5D9OQsMPkbkp4KHM7JZv/RcGCpr3E1L7XX7U9sCxY+sFGeysltoviTmaIBXsJ2IjAJbBULtf0G/J+2cfH5OP+w==", "dependencies": { "@aws-crypto/sha256-browser": "5.2.0", "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "3.883.0", - "@aws-sdk/middleware-host-header": "3.873.0", - "@aws-sdk/middleware-logger": "3.876.0", - "@aws-sdk/middleware-recursion-detection": "3.873.0", - "@aws-sdk/middleware-user-agent": "3.883.0", - "@aws-sdk/region-config-resolver": "3.873.0", - "@aws-sdk/types": "3.862.0", - "@aws-sdk/util-endpoints": "3.879.0", - "@aws-sdk/util-user-agent-browser": "3.873.0", - "@aws-sdk/util-user-agent-node": "3.883.0", - "@smithy/config-resolver": "^4.1.5", - "@smithy/core": "^3.9.2", - "@smithy/fetch-http-handler": "^5.1.1", - "@smithy/hash-node": "^4.0.5", - "@smithy/invalid-dependency": "^4.0.5", - "@smithy/middleware-content-length": "^4.0.5", - "@smithy/middleware-endpoint": "^4.1.21", - "@smithy/middleware-retry": "^4.1.22", - "@smithy/middleware-serde": "^4.0.9", - "@smithy/middleware-stack": "^4.0.5", - "@smithy/node-config-provider": "^4.1.4", - "@smithy/node-http-handler": "^4.1.1", - "@smithy/protocol-http": "^5.1.3", - "@smithy/smithy-client": "^4.5.2", - "@smithy/types": "^4.3.2", - "@smithy/url-parser": "^4.0.5", - "@smithy/util-base64": "^4.0.0", - "@smithy/util-body-length-browser": "^4.0.0", - "@smithy/util-body-length-node": "^4.0.0", - "@smithy/util-defaults-mode-browser": "^4.0.29", - "@smithy/util-defaults-mode-node": "^4.0.29", - "@smithy/util-endpoints": "^3.0.7", - "@smithy/util-middleware": "^4.0.5", - "@smithy/util-retry": "^4.0.7", - "@smithy/util-utf8": "^4.0.0", + "@aws-sdk/core": "3.916.0", + "@aws-sdk/middleware-host-header": "3.914.0", + "@aws-sdk/middleware-logger": "3.914.0", + "@aws-sdk/middleware-recursion-detection": "3.919.0", + "@aws-sdk/middleware-user-agent": "3.916.0", + "@aws-sdk/region-config-resolver": "3.914.0", + "@aws-sdk/types": "3.914.0", + "@aws-sdk/util-endpoints": "3.916.0", + "@aws-sdk/util-user-agent-browser": "3.914.0", + "@aws-sdk/util-user-agent-node": "3.916.0", + "@smithy/config-resolver": "^4.4.0", + "@smithy/core": "^3.17.1", + "@smithy/fetch-http-handler": "^5.3.4", + "@smithy/hash-node": "^4.2.3", + "@smithy/invalid-dependency": "^4.2.3", + "@smithy/middleware-content-length": "^4.2.3", + "@smithy/middleware-endpoint": "^4.3.5", + "@smithy/middleware-retry": "^4.4.5", + "@smithy/middleware-serde": "^4.2.3", + "@smithy/middleware-stack": "^4.2.3", + "@smithy/node-config-provider": "^4.3.3", + "@smithy/node-http-handler": "^4.4.3", + "@smithy/protocol-http": "^5.3.3", + "@smithy/smithy-client": "^4.9.1", + "@smithy/types": "^4.8.0", + "@smithy/url-parser": "^4.2.3", + "@smithy/util-base64": "^4.3.0", + "@smithy/util-body-length-browser": "^4.2.0", + "@smithy/util-body-length-node": "^4.2.1", + "@smithy/util-defaults-mode-browser": "^4.3.4", + "@smithy/util-defaults-mode-node": "^4.2.6", + "@smithy/util-endpoints": "^3.2.3", + "@smithy/util-middleware": "^4.2.3", + "@smithy/util-retry": "^4.2.3", + "@smithy/util-utf8": "^4.2.0", "tslib": "^2.6.2" }, "engines": { @@ -2008,15 +2010,13 @@ } }, "node_modules/@aws-sdk/client-s3vectors/node_modules/@aws-sdk/region-config-resolver": { - "version": "3.873.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/region-config-resolver/-/region-config-resolver-3.873.0.tgz", - "integrity": "sha512-q9sPoef+BBG6PJnc4x60vK/bfVwvRWsPgcoQyIra057S/QGjq5VkjvNk6H8xedf6vnKlXNBwq9BaANBXnldUJg==", + "version": "3.914.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/region-config-resolver/-/region-config-resolver-3.914.0.tgz", + "integrity": "sha512-KlmHhRbn1qdwXUdsdrJ7S/MAkkC1jLpQ11n+XvxUUUCGAJd1gjC7AjxPZUM7ieQ2zcb8bfEzIU7al+Q3ZT0u7Q==", "dependencies": { - "@aws-sdk/types": "3.862.0", - "@smithy/node-config-provider": "^4.1.4", - "@smithy/types": "^4.3.2", - "@smithy/util-config-provider": "^4.0.0", - "@smithy/util-middleware": "^4.0.5", + "@aws-sdk/types": "3.914.0", + "@smithy/config-resolver": "^4.4.0", + "@smithy/types": "^4.8.0", "tslib": "^2.6.2" }, "engines": { @@ -2024,16 +2024,16 @@ } }, "node_modules/@aws-sdk/client-s3vectors/node_modules/@aws-sdk/token-providers": { - "version": "3.883.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/token-providers/-/token-providers-3.883.0.tgz", - "integrity": "sha512-tcj/Z5paGn9esxhmmkEW7gt39uNoIRbXG1UwJrfKu4zcTr89h86PDiIE2nxUO3CMQf1KgncPpr5WouPGzkh/QQ==", + "version": "3.919.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/token-providers/-/token-providers-3.919.0.tgz", + "integrity": "sha512-6aFv4lzXbfbkl0Pv37Us8S/ZkqplOQZIEgQg7bfMru7P96Wv2jVnDGsEc5YyxMnnRyIB90naQ5JgslZ4rkpknw==", "dependencies": { - "@aws-sdk/core": "3.883.0", - "@aws-sdk/nested-clients": "3.883.0", - "@aws-sdk/types": "3.862.0", - "@smithy/property-provider": "^4.0.5", - "@smithy/shared-ini-file-loader": "^4.0.5", - "@smithy/types": "^4.3.2", + "@aws-sdk/core": "3.916.0", + "@aws-sdk/nested-clients": "3.919.0", + "@aws-sdk/types": "3.914.0", + "@smithy/property-provider": "^4.2.3", + "@smithy/shared-ini-file-loader": "^4.3.3", + "@smithy/types": "^4.8.0", "tslib": "^2.6.2" }, "engines": { @@ -2041,11 +2041,11 @@ } }, "node_modules/@aws-sdk/client-s3vectors/node_modules/@aws-sdk/types": { - "version": "3.862.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.862.0.tgz", - "integrity": "sha512-Bei+RL0cDxxV+lW2UezLbCYYNeJm6Nzee0TpW0FfyTRBhH9C1XQh4+x+IClriXvgBnRquTMMYsmJfvx8iyLKrg==", + "version": "3.914.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.914.0.tgz", + "integrity": "sha512-kQWPsRDmom4yvAfyG6L1lMmlwnTzm1XwMHOU+G5IFlsP4YEaMtXidDzW/wiivY0QFrhfCz/4TVmu0a2aPU57ug==", "dependencies": { - "@smithy/types": "^4.3.2", + "@smithy/types": "^4.8.0", "tslib": "^2.6.2" }, "engines": { @@ -2053,14 +2053,14 @@ } }, "node_modules/@aws-sdk/client-s3vectors/node_modules/@aws-sdk/util-endpoints": { - "version": "3.879.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/util-endpoints/-/util-endpoints-3.879.0.tgz", - "integrity": "sha512-aVAJwGecYoEmbEFju3127TyJDF9qJsKDUUTRMDuS8tGn+QiWQFnfInmbt+el9GU1gEJupNTXV+E3e74y51fb7A==", + "version": "3.916.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-endpoints/-/util-endpoints-3.916.0.tgz", + "integrity": "sha512-bAgUQwvixdsiGNcuZSDAOWbyHlnPtg8G8TyHD6DTfTmKTHUW6tAn+af/ZYJPXEzXhhpwgJqi58vWnsiDhmr7NQ==", "dependencies": { - "@aws-sdk/types": "3.862.0", - "@smithy/types": "^4.3.2", - "@smithy/url-parser": "^4.0.5", - "@smithy/util-endpoints": "^3.0.7", + "@aws-sdk/types": "3.914.0", + "@smithy/types": "^4.8.0", + "@smithy/url-parser": "^4.2.3", + "@smithy/util-endpoints": "^3.2.3", "tslib": "^2.6.2" }, "engines": { @@ -2068,25 +2068,25 @@ } }, "node_modules/@aws-sdk/client-s3vectors/node_modules/@aws-sdk/util-user-agent-browser": { - "version": "3.873.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/util-user-agent-browser/-/util-user-agent-browser-3.873.0.tgz", - "integrity": "sha512-AcRdbK6o19yehEcywI43blIBhOCSo6UgyWcuOJX5CFF8k39xm1ILCjQlRRjchLAxWrm0lU0Q7XV90RiMMFMZtA==", + "version": "3.914.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-user-agent-browser/-/util-user-agent-browser-3.914.0.tgz", + "integrity": "sha512-rMQUrM1ECH4kmIwlGl9UB0BtbHy6ZuKdWFrIknu8yGTRI/saAucqNTh5EI1vWBxZ0ElhK5+g7zOnUuhSmVQYUA==", "dependencies": { - "@aws-sdk/types": "3.862.0", - "@smithy/types": "^4.3.2", + "@aws-sdk/types": "3.914.0", + "@smithy/types": "^4.8.0", "bowser": "^2.11.0", "tslib": "^2.6.2" } }, "node_modules/@aws-sdk/client-s3vectors/node_modules/@aws-sdk/util-user-agent-node": { - "version": "3.883.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/util-user-agent-node/-/util-user-agent-node-3.883.0.tgz", - "integrity": "sha512-28cQZqC+wsKUHGpTBr+afoIdjS6IoEJkMqcZsmo2Ag8LzmTa6BUWQenFYB0/9BmDy4PZFPUn+uX+rJgWKB+jzA==", + "version": "3.916.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-user-agent-node/-/util-user-agent-node-3.916.0.tgz", + "integrity": "sha512-CwfWV2ch6UdjuSV75ZU99N03seEUb31FIUrXBnwa6oONqj/xqXwrxtlUMLx6WH3OJEE4zI3zt5PjlTdGcVwf4g==", "dependencies": { - "@aws-sdk/middleware-user-agent": "3.883.0", - "@aws-sdk/types": "3.862.0", - "@smithy/node-config-provider": "^4.1.4", - "@smithy/types": "^4.3.2", + "@aws-sdk/middleware-user-agent": "3.916.0", + "@aws-sdk/types": "3.914.0", + "@smithy/node-config-provider": "^4.3.3", + "@smithy/types": "^4.8.0", "tslib": "^2.6.2" }, "engines": { @@ -2102,11 +2102,12 @@ } }, "node_modules/@aws-sdk/client-s3vectors/node_modules/@aws-sdk/xml-builder": { - "version": "3.873.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/xml-builder/-/xml-builder-3.873.0.tgz", - "integrity": "sha512-kLO7k7cGJ6KaHiExSJWojZurF7SnGMDHXRuQunFnEoD0n1yB6Lqy/S/zHiQ7oJnBhPr9q0TW9qFkrsZb1Uc54w==", + "version": "3.914.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/xml-builder/-/xml-builder-3.914.0.tgz", + "integrity": "sha512-k75evsBD5TcIjedycYS7QXQ98AmOtbnxRJOPtCo0IwYRmy7UvqgS/gBL5SmrIqeV6FDSYRQMgdBxSMp6MLmdew==", "dependencies": { - "@smithy/types": "^4.3.2", + "@smithy/types": "^4.8.0", + "fast-xml-parser": "5.2.5", "tslib": "^2.6.2" }, "engines": { @@ -2114,11 +2115,11 @@ } }, "node_modules/@aws-sdk/client-s3vectors/node_modules/@smithy/abort-controller": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/@smithy/abort-controller/-/abort-controller-4.1.0.tgz", - "integrity": "sha512-wEhSYznxOmx7EdwK1tYEWJF5+/wmSFsff9BfTOn8oO/+KPl3gsmThrb6MJlWbOC391+Ya31s5JuHiC2RlT80Zg==", + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/@smithy/abort-controller/-/abort-controller-4.2.3.tgz", + "integrity": "sha512-xWL9Mf8b7tIFuAlpjKtRPnHrR8XVrwTj5NPYO/QwZPtc0SDLsPxb56V5tzi5yspSMytISHybifez+4jlrx0vkQ==", "dependencies": { - "@smithy/types": "^4.4.0", + "@smithy/types": "^4.8.0", "tslib": "^2.6.2" }, "engines": { @@ -2126,14 +2127,15 @@ } }, "node_modules/@aws-sdk/client-s3vectors/node_modules/@smithy/config-resolver": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/@smithy/config-resolver/-/config-resolver-4.2.0.tgz", - "integrity": "sha512-FA10YhPFLy23uxeWu7pOM2ctlw+gzbPMTZQwrZ8FRIfyJ/p8YIVz7AVTB5jjLD+QIerydyKcVMZur8qzzDILAQ==", + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/@smithy/config-resolver/-/config-resolver-4.4.0.tgz", + "integrity": "sha512-Kkmz3Mup2PGp/HNJxhCWkLNdlajJORLSjwkcfrj0E7nu6STAEdcMR1ir5P9/xOmncx8xXfru0fbUYLlZog/cFg==", "dependencies": { - "@smithy/node-config-provider": "^4.2.0", - "@smithy/types": "^4.4.0", - "@smithy/util-config-provider": "^4.1.0", - "@smithy/util-middleware": "^4.1.0", + "@smithy/node-config-provider": "^4.3.3", + "@smithy/types": "^4.8.0", + "@smithy/util-config-provider": "^4.2.0", + "@smithy/util-endpoints": "^3.2.3", + "@smithy/util-middleware": "^4.2.3", "tslib": "^2.6.2" }, "engines": { @@ -2141,35 +2143,34 @@ } }, "node_modules/@aws-sdk/client-s3vectors/node_modules/@smithy/core": { - "version": "3.10.0", - "resolved": "https://registry.npmjs.org/@smithy/core/-/core-3.10.0.tgz", - "integrity": "sha512-bXyD3Ij6b1qDymEYlEcF+QIjwb9gObwZNaRjETJsUEvSIzxFdynSQ3E4ysY7lUFSBzeWBNaFvX+5A0smbC2q6A==", - "dependencies": { - "@smithy/middleware-serde": "^4.1.0", - "@smithy/protocol-http": "^5.2.0", - "@smithy/types": "^4.4.0", - "@smithy/util-base64": "^4.1.0", - "@smithy/util-body-length-browser": "^4.1.0", - "@smithy/util-middleware": "^4.1.0", - "@smithy/util-stream": "^4.3.0", - "@smithy/util-utf8": "^4.1.0", - "@types/uuid": "^9.0.1", - "tslib": "^2.6.2", - "uuid": "^9.0.1" + "version": "3.17.1", + "resolved": "https://registry.npmjs.org/@smithy/core/-/core-3.17.1.tgz", + "integrity": "sha512-V4Qc2CIb5McABYfaGiIYLTmo/vwNIK7WXI5aGveBd9UcdhbOMwcvIMxIw/DJj1S9QgOMa/7FBkarMdIC0EOTEQ==", + "dependencies": { + "@smithy/middleware-serde": "^4.2.3", + "@smithy/protocol-http": "^5.3.3", + "@smithy/types": "^4.8.0", + "@smithy/util-base64": "^4.3.0", + "@smithy/util-body-length-browser": "^4.2.0", + "@smithy/util-middleware": "^4.2.3", + "@smithy/util-stream": "^4.5.4", + "@smithy/util-utf8": "^4.2.0", + "@smithy/uuid": "^1.1.0", + "tslib": "^2.6.2" }, "engines": { "node": ">=18.0.0" } }, "node_modules/@aws-sdk/client-s3vectors/node_modules/@smithy/credential-provider-imds": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/@smithy/credential-provider-imds/-/credential-provider-imds-4.1.0.tgz", - "integrity": "sha512-iVwNhxTsCQTPdp++4C/d9xvaDmuEWhXi55qJobMp9QMaEHRGH3kErU4F8gohtdsawRqnUy/ANylCjKuhcR2mPw==", + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/@smithy/credential-provider-imds/-/credential-provider-imds-4.2.3.tgz", + "integrity": "sha512-hA1MQ/WAHly4SYltJKitEsIDVsNmXcQfYBRv2e+q04fnqtAX5qXaybxy/fhUeAMCnQIdAjaGDb04fMHQefWRhw==", "dependencies": { - "@smithy/node-config-provider": "^4.2.0", - "@smithy/property-provider": "^4.1.0", - "@smithy/types": "^4.4.0", - "@smithy/url-parser": "^4.1.0", + "@smithy/node-config-provider": "^4.3.3", + "@smithy/property-provider": "^4.2.3", + "@smithy/types": "^4.8.0", + "@smithy/url-parser": "^4.2.3", "tslib": "^2.6.2" }, "engines": { @@ -2177,14 +2178,14 @@ } }, "node_modules/@aws-sdk/client-s3vectors/node_modules/@smithy/fetch-http-handler": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/@smithy/fetch-http-handler/-/fetch-http-handler-5.2.0.tgz", - "integrity": "sha512-VZenjDdVaUGiy3hwQtxm75nhXZrhFG+3xyL93qCQAlYDyhT/jeDWM8/3r5uCFMlTmmyrIjiDyiOynVFchb0BSg==", + "version": "5.3.4", + "resolved": "https://registry.npmjs.org/@smithy/fetch-http-handler/-/fetch-http-handler-5.3.4.tgz", + "integrity": "sha512-bwigPylvivpRLCm+YK9I5wRIYjFESSVwl8JQ1vVx/XhCw0PtCi558NwTnT2DaVCl5pYlImGuQTSwMsZ+pIavRw==", "dependencies": { - "@smithy/protocol-http": "^5.2.0", - "@smithy/querystring-builder": "^4.1.0", - "@smithy/types": "^4.4.0", - "@smithy/util-base64": "^4.1.0", + "@smithy/protocol-http": "^5.3.3", + "@smithy/querystring-builder": "^4.2.3", + "@smithy/types": "^4.8.0", + "@smithy/util-base64": "^4.3.0", "tslib": "^2.6.2" }, "engines": { @@ -2192,13 +2193,13 @@ } }, "node_modules/@aws-sdk/client-s3vectors/node_modules/@smithy/hash-node": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/@smithy/hash-node/-/hash-node-4.1.0.tgz", - "integrity": "sha512-mXkJQ/6lAXTuoSsEH+d/fHa4ms4qV5LqYoPLYhmhCRTNcMMdg+4Ya8cMgU1W8+OR40eX0kzsExT7fAILqtTl2w==", + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/@smithy/hash-node/-/hash-node-4.2.3.tgz", + "integrity": "sha512-6+NOdZDbfuU6s1ISp3UOk5Rg953RJ2aBLNLLBEcamLjHAg1Po9Ha7QIB5ZWhdRUVuOUrT8BVFR+O2KIPmw027g==", "dependencies": { - "@smithy/types": "^4.4.0", - "@smithy/util-buffer-from": "^4.1.0", - "@smithy/util-utf8": "^4.1.0", + "@smithy/types": "^4.8.0", + "@smithy/util-buffer-from": "^4.2.0", + "@smithy/util-utf8": "^4.2.0", "tslib": "^2.6.2" }, "engines": { @@ -2206,11 +2207,11 @@ } }, "node_modules/@aws-sdk/client-s3vectors/node_modules/@smithy/invalid-dependency": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/@smithy/invalid-dependency/-/invalid-dependency-4.1.0.tgz", - "integrity": "sha512-4/FcV6aCMzgpM4YyA/GRzTtG28G0RQJcWK722MmpIgzOyfSceWcI9T9c8matpHU9qYYLaWtk8pSGNCLn5kzDRw==", + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/@smithy/invalid-dependency/-/invalid-dependency-4.2.3.tgz", + "integrity": "sha512-Cc9W5DwDuebXEDMpOpl4iERo8I0KFjTnomK2RMdhhR87GwrSmUmwMxS4P5JdRf+LsjOdIqumcerwRgYMr/tZ9Q==", "dependencies": { - "@smithy/types": "^4.4.0", + "@smithy/types": "^4.8.0", "tslib": "^2.6.2" }, "engines": { @@ -2218,9 +2219,9 @@ } }, "node_modules/@aws-sdk/client-s3vectors/node_modules/@smithy/is-array-buffer": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-4.1.0.tgz", - "integrity": "sha512-ePTYUOV54wMogio+he4pBybe8fwg4sDvEVDBU8ZlHOZXbXK3/C0XfJgUCu6qAZcawv05ZhZzODGUerFBPsPUDQ==", + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-4.2.0.tgz", + "integrity": "sha512-DZZZBvC7sjcYh4MazJSGiWMI2L7E0oCiRHREDzIxi/M2LY79/21iXt6aPLHge82wi5LsuRF5A06Ds3+0mlh6CQ==", "dependencies": { "tslib": "^2.6.2" }, @@ -2229,12 +2230,12 @@ } }, "node_modules/@aws-sdk/client-s3vectors/node_modules/@smithy/middleware-content-length": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/@smithy/middleware-content-length/-/middleware-content-length-4.1.0.tgz", - "integrity": "sha512-x3dgLFubk/ClKVniJu+ELeZGk4mq7Iv0HgCRUlxNUIcerHTLVmq7Q5eGJL0tOnUltY6KFw5YOKaYxwdcMwox/w==", + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/@smithy/middleware-content-length/-/middleware-content-length-4.2.3.tgz", + "integrity": "sha512-/atXLsT88GwKtfp5Jr0Ks1CSa4+lB+IgRnkNrrYP0h1wL4swHNb0YONEvTceNKNdZGJsye+W2HH8W7olbcPUeA==", "dependencies": { - "@smithy/protocol-http": "^5.2.0", - "@smithy/types": "^4.4.0", + "@smithy/protocol-http": "^5.3.3", + "@smithy/types": "^4.8.0", "tslib": "^2.6.2" }, "engines": { @@ -2242,17 +2243,17 @@ } }, "node_modules/@aws-sdk/client-s3vectors/node_modules/@smithy/middleware-endpoint": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/@smithy/middleware-endpoint/-/middleware-endpoint-4.2.0.tgz", - "integrity": "sha512-J1eCF7pPDwgv7fGwRd2+Y+H9hlIolF3OZ2PjptonzzyOXXGh/1KGJAHpEcY1EX+WLlclKu2yC5k+9jWXdUG4YQ==", + "version": "4.3.5", + "resolved": "https://registry.npmjs.org/@smithy/middleware-endpoint/-/middleware-endpoint-4.3.5.tgz", + "integrity": "sha512-SIzKVTvEudFWJbxAaq7f2GvP3jh2FHDpIFI6/VAf4FOWGFZy0vnYMPSRj8PGYI8Hjt29mvmwSRgKuO3bK4ixDw==", "dependencies": { - "@smithy/core": "^3.10.0", - "@smithy/middleware-serde": "^4.1.0", - "@smithy/node-config-provider": "^4.2.0", - "@smithy/shared-ini-file-loader": "^4.1.0", - "@smithy/types": "^4.4.0", - "@smithy/url-parser": "^4.1.0", - "@smithy/util-middleware": "^4.1.0", + "@smithy/core": "^3.17.1", + "@smithy/middleware-serde": "^4.2.3", + "@smithy/node-config-provider": "^4.3.3", + "@smithy/shared-ini-file-loader": "^4.3.3", + "@smithy/types": "^4.8.0", + "@smithy/url-parser": "^4.2.3", + "@smithy/util-middleware": "^4.2.3", "tslib": "^2.6.2" }, "engines": { @@ -2260,32 +2261,31 @@ } }, "node_modules/@aws-sdk/client-s3vectors/node_modules/@smithy/middleware-retry": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/@smithy/middleware-retry/-/middleware-retry-4.2.0.tgz", - "integrity": "sha512-raL5oWYf5ALl3jCJrajE8enKJEnV/2wZkKS6mb3ZRY2tg3nj66ssdWy5Ps8E6Yu8Wqh3Tt+Sb9LozjvwZupq+A==", - "dependencies": { - "@smithy/node-config-provider": "^4.2.0", - "@smithy/protocol-http": "^5.2.0", - "@smithy/service-error-classification": "^4.1.0", - "@smithy/smithy-client": "^4.6.0", - "@smithy/types": "^4.4.0", - "@smithy/util-middleware": "^4.1.0", - "@smithy/util-retry": "^4.1.0", - "@types/uuid": "^9.0.1", - "tslib": "^2.6.2", - "uuid": "^9.0.1" + "version": "4.4.5", + "resolved": "https://registry.npmjs.org/@smithy/middleware-retry/-/middleware-retry-4.4.5.tgz", + "integrity": "sha512-DCaXbQqcZ4tONMvvdz+zccDE21sLcbwWoNqzPLFlZaxt1lDtOE2tlVpRSwcTOJrjJSUThdgEYn7HrX5oLGlK9A==", + "dependencies": { + "@smithy/node-config-provider": "^4.3.3", + "@smithy/protocol-http": "^5.3.3", + "@smithy/service-error-classification": "^4.2.3", + "@smithy/smithy-client": "^4.9.1", + "@smithy/types": "^4.8.0", + "@smithy/util-middleware": "^4.2.3", + "@smithy/util-retry": "^4.2.3", + "@smithy/uuid": "^1.1.0", + "tslib": "^2.6.2" }, "engines": { "node": ">=18.0.0" } }, "node_modules/@aws-sdk/client-s3vectors/node_modules/@smithy/middleware-serde": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/@smithy/middleware-serde/-/middleware-serde-4.1.0.tgz", - "integrity": "sha512-CtLFYlHt7c2VcztyVRc+25JLV4aGpmaSv9F1sPB0AGFL6S+RPythkqpGDa2XBQLJQooKkjLA1g7Xe4450knShg==", + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/@smithy/middleware-serde/-/middleware-serde-4.2.3.tgz", + "integrity": "sha512-8g4NuUINpYccxiCXM5s1/V+uLtts8NcX4+sPEbvYQDZk4XoJfDpq5y2FQxfmUL89syoldpzNzA0R9nhzdtdKnQ==", "dependencies": { - "@smithy/protocol-http": "^5.2.0", - "@smithy/types": "^4.4.0", + "@smithy/protocol-http": "^5.3.3", + "@smithy/types": "^4.8.0", "tslib": "^2.6.2" }, "engines": { @@ -2293,11 +2293,11 @@ } }, "node_modules/@aws-sdk/client-s3vectors/node_modules/@smithy/middleware-stack": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/@smithy/middleware-stack/-/middleware-stack-4.1.0.tgz", - "integrity": "sha512-91Fuw4IKp0eK8PNhMXrHRcYA1jvbZ9BJGT91wwPy3bTQT8mHTcQNius/EhSQTlT9QUI3Ki1wjHeNXbWK0tO8YQ==", + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/@smithy/middleware-stack/-/middleware-stack-4.2.3.tgz", + "integrity": "sha512-iGuOJkH71faPNgOj/gWuEGS6xvQashpLwWB1HjHq1lNNiVfbiJLpZVbhddPuDbx9l4Cgl0vPLq5ltRfSaHfspA==", "dependencies": { - "@smithy/types": "^4.4.0", + "@smithy/types": "^4.8.0", "tslib": "^2.6.2" }, "engines": { @@ -2305,13 +2305,13 @@ } }, "node_modules/@aws-sdk/client-s3vectors/node_modules/@smithy/node-config-provider": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/@smithy/node-config-provider/-/node-config-provider-4.2.0.tgz", - "integrity": "sha512-8/fpilqKurQ+f8nFvoFkJ0lrymoMJ+5/CQV5IcTv/MyKhk2Q/EFYCAgTSWHD4nMi9ux9NyBBynkyE9SLg2uSLA==", + "version": "4.3.3", + "resolved": "https://registry.npmjs.org/@smithy/node-config-provider/-/node-config-provider-4.3.3.tgz", + "integrity": "sha512-NzI1eBpBSViOav8NVy1fqOlSfkLgkUjUTlohUSgAEhHaFWA3XJiLditvavIP7OpvTjDp5u2LhtlBhkBlEisMwA==", "dependencies": { - "@smithy/property-provider": "^4.1.0", - "@smithy/shared-ini-file-loader": "^4.1.0", - "@smithy/types": "^4.4.0", + "@smithy/property-provider": "^4.2.3", + "@smithy/shared-ini-file-loader": "^4.3.3", + "@smithy/types": "^4.8.0", "tslib": "^2.6.2" }, "engines": { @@ -2319,14 +2319,14 @@ } }, "node_modules/@aws-sdk/client-s3vectors/node_modules/@smithy/node-http-handler": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-4.2.0.tgz", - "integrity": "sha512-G4NV70B4hF9vBrUkkvNfWO6+QR4jYjeO4tc+4XrKCb4nPYj49V9Hu8Ftio7Mb0/0IlFyEOORudHrm+isY29nCA==", + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-4.4.3.tgz", + "integrity": "sha512-MAwltrDB0lZB/H6/2M5PIsISSwdI5yIh6DaBB9r0Flo9nx3y0dzl/qTMJPd7tJvPdsx6Ks/cwVzheGNYzXyNbQ==", "dependencies": { - "@smithy/abort-controller": "^4.1.0", - "@smithy/protocol-http": "^5.2.0", - "@smithy/querystring-builder": "^4.1.0", - "@smithy/types": "^4.4.0", + "@smithy/abort-controller": "^4.2.3", + "@smithy/protocol-http": "^5.3.3", + "@smithy/querystring-builder": "^4.2.3", + "@smithy/types": "^4.8.0", "tslib": "^2.6.2" }, "engines": { @@ -2334,11 +2334,11 @@ } }, "node_modules/@aws-sdk/client-s3vectors/node_modules/@smithy/property-provider": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/@smithy/property-provider/-/property-provider-4.1.0.tgz", - "integrity": "sha512-eksMjMHUlG5PwOUWO3k+rfLNOPVPJ70mUzyYNKb5lvyIuAwS4zpWGsxGiuT74DFWonW0xRNy+jgzGauUzX7SyA==", + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/@smithy/property-provider/-/property-provider-4.2.3.tgz", + "integrity": "sha512-+1EZ+Y+njiefCohjlhyOcy1UNYjT+1PwGFHCxA/gYctjg3DQWAU19WigOXAco/Ql8hZokNehpzLd0/+3uCreqQ==", "dependencies": { - "@smithy/types": "^4.4.0", + "@smithy/types": "^4.8.0", "tslib": "^2.6.2" }, "engines": { @@ -2346,11 +2346,11 @@ } }, "node_modules/@aws-sdk/client-s3vectors/node_modules/@smithy/protocol-http": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/@smithy/protocol-http/-/protocol-http-5.2.0.tgz", - "integrity": "sha512-bwjlh5JwdOQnA01be+5UvHK4HQz4iaRKlVG46hHSJuqi0Ribt3K06Z1oQ29i35Np4G9MCDgkOGcHVyLMreMcbg==", + "version": "5.3.3", + "resolved": "https://registry.npmjs.org/@smithy/protocol-http/-/protocol-http-5.3.3.tgz", + "integrity": "sha512-Mn7f/1aN2/jecywDcRDvWWWJF4uwg/A0XjFMJtj72DsgHTByfjRltSqcT9NyE9RTdBSN6X1RSXrhn/YWQl8xlw==", "dependencies": { - "@smithy/types": "^4.4.0", + "@smithy/types": "^4.8.0", "tslib": "^2.6.2" }, "engines": { @@ -2358,12 +2358,12 @@ } }, "node_modules/@aws-sdk/client-s3vectors/node_modules/@smithy/querystring-builder": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/@smithy/querystring-builder/-/querystring-builder-4.1.0.tgz", - "integrity": "sha512-JqTWmVIq4AF8R8OK/2cCCiQo5ZJ0SRPsDkDgLO5/3z8xxuUp1oMIBBjfuueEe+11hGTZ6rRebzYikpKc6yQV9Q==", + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/@smithy/querystring-builder/-/querystring-builder-4.2.3.tgz", + "integrity": "sha512-LOVCGCmwMahYUM/P0YnU/AlDQFjcu+gWbFJooC417QRB/lDJlWSn8qmPSDp+s4YVAHOgtgbNG4sR+SxF/VOcJQ==", "dependencies": { - "@smithy/types": "^4.4.0", - "@smithy/util-uri-escape": "^4.1.0", + "@smithy/types": "^4.8.0", + "@smithy/util-uri-escape": "^4.2.0", "tslib": "^2.6.2" }, "engines": { @@ -2371,11 +2371,11 @@ } }, "node_modules/@aws-sdk/client-s3vectors/node_modules/@smithy/querystring-parser": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/@smithy/querystring-parser/-/querystring-parser-4.1.0.tgz", - "integrity": "sha512-VgdHhr8YTRsjOl4hnKFm7xEMOCRTnKw3FJ1nU+dlWNhdt/7eEtxtkdrJdx7PlRTabdANTmvyjE4umUl9cK4awg==", + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/@smithy/querystring-parser/-/querystring-parser-4.2.3.tgz", + "integrity": "sha512-cYlSNHcTAX/wc1rpblli3aUlLMGgKZ/Oqn8hhjFASXMCXjIqeuQBei0cnq2JR8t4RtU9FpG6uyl6PxyArTiwKA==", "dependencies": { - "@smithy/types": "^4.4.0", + "@smithy/types": "^4.8.0", "tslib": "^2.6.2" }, "engines": { @@ -2383,22 +2383,22 @@ } }, "node_modules/@aws-sdk/client-s3vectors/node_modules/@smithy/service-error-classification": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/@smithy/service-error-classification/-/service-error-classification-4.1.0.tgz", - "integrity": "sha512-UBpNFzBNmS20jJomuYn++Y+soF8rOK9AvIGjS9yGP6uRXF5rP18h4FDUsoNpWTlSsmiJ87e2DpZo9ywzSMH7PQ==", + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/@smithy/service-error-classification/-/service-error-classification-4.2.3.tgz", + "integrity": "sha512-NkxsAxFWwsPsQiwFG2MzJ/T7uIR6AQNh1SzcxSUnmmIqIQMlLRQDKhc17M7IYjiuBXhrQRjQTo3CxX+DobS93g==", "dependencies": { - "@smithy/types": "^4.4.0" + "@smithy/types": "^4.8.0" }, "engines": { "node": ">=18.0.0" } }, "node_modules/@aws-sdk/client-s3vectors/node_modules/@smithy/shared-ini-file-loader": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/@smithy/shared-ini-file-loader/-/shared-ini-file-loader-4.1.0.tgz", - "integrity": "sha512-W0VMlz9yGdQ/0ZAgWICFjFHTVU0YSfGoCVpKaExRM/FDkTeP/yz8OKvjtGjs6oFokCRm0srgj/g4Cg0xuHu8Rw==", + "version": "4.3.3", + "resolved": "https://registry.npmjs.org/@smithy/shared-ini-file-loader/-/shared-ini-file-loader-4.3.3.tgz", + "integrity": "sha512-9f9Ixej0hFhroOK2TxZfUUDR13WVa8tQzhSzPDgXe5jGL3KmaM9s8XN7RQwqtEypI82q9KHnKS71CJ+q/1xLtQ==", "dependencies": { - "@smithy/types": "^4.4.0", + "@smithy/types": "^4.8.0", "tslib": "^2.6.2" }, "engines": { @@ -2406,17 +2406,17 @@ } }, "node_modules/@aws-sdk/client-s3vectors/node_modules/@smithy/signature-v4": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/@smithy/signature-v4/-/signature-v4-5.2.0.tgz", - "integrity": "sha512-ObX1ZqG2DdZQlXx9mLD7yAR8AGb7yXurGm+iWx9x4l1fBZ8CZN2BRT09aSbcXVPZXWGdn5VtMuupjxhOTI2EjA==", + "version": "5.3.3", + "resolved": "https://registry.npmjs.org/@smithy/signature-v4/-/signature-v4-5.3.3.tgz", + "integrity": "sha512-CmSlUy+eEYbIEYN5N3vvQTRfqt0lJlQkaQUIf+oizu7BbDut0pozfDjBGecfcfWf7c62Yis4JIEgqQ/TCfodaA==", "dependencies": { - "@smithy/is-array-buffer": "^4.1.0", - "@smithy/protocol-http": "^5.2.0", - "@smithy/types": "^4.4.0", - "@smithy/util-hex-encoding": "^4.1.0", - "@smithy/util-middleware": "^4.1.0", - "@smithy/util-uri-escape": "^4.1.0", - "@smithy/util-utf8": "^4.1.0", + "@smithy/is-array-buffer": "^4.2.0", + "@smithy/protocol-http": "^5.3.3", + "@smithy/types": "^4.8.0", + "@smithy/util-hex-encoding": "^4.2.0", + "@smithy/util-middleware": "^4.2.3", + "@smithy/util-uri-escape": "^4.2.0", + "@smithy/util-utf8": "^4.2.0", "tslib": "^2.6.2" }, "engines": { @@ -2424,16 +2424,16 @@ } }, "node_modules/@aws-sdk/client-s3vectors/node_modules/@smithy/smithy-client": { - "version": "4.6.0", - "resolved": "https://registry.npmjs.org/@smithy/smithy-client/-/smithy-client-4.6.0.tgz", - "integrity": "sha512-TvlIshqx5PIi0I0AiR+PluCpJ8olVG++xbYkAIGCUkByaMUlfOXLgjQTmYbr46k4wuDe8eHiTIlUflnjK2drPQ==", + "version": "4.9.1", + "resolved": "https://registry.npmjs.org/@smithy/smithy-client/-/smithy-client-4.9.1.tgz", + "integrity": "sha512-Ngb95ryR5A9xqvQFT5mAmYkCwbXvoLavLFwmi7zVg/IowFPCfiqRfkOKnbc/ZRL8ZKJ4f+Tp6kSu6wjDQb8L/g==", "dependencies": { - "@smithy/core": "^3.10.0", - "@smithy/middleware-endpoint": "^4.2.0", - "@smithy/middleware-stack": "^4.1.0", - "@smithy/protocol-http": "^5.2.0", - "@smithy/types": "^4.4.0", - "@smithy/util-stream": "^4.3.0", + "@smithy/core": "^3.17.1", + "@smithy/middleware-endpoint": "^4.3.5", + "@smithy/middleware-stack": "^4.2.3", + "@smithy/protocol-http": "^5.3.3", + "@smithy/types": "^4.8.0", + "@smithy/util-stream": "^4.5.4", "tslib": "^2.6.2" }, "engines": { @@ -2441,9 +2441,9 @@ } }, "node_modules/@aws-sdk/client-s3vectors/node_modules/@smithy/types": { - "version": "4.4.0", - "resolved": "https://registry.npmjs.org/@smithy/types/-/types-4.4.0.tgz", - "integrity": "sha512-4jY91NgZz+ZnSFcVzWwngOW6VuK3gR/ihTwSU1R/0NENe9Jd8SfWgbhDCAGUWL3bI7DiDSW7XF6Ui6bBBjrqXw==", + "version": "4.8.0", + "resolved": "https://registry.npmjs.org/@smithy/types/-/types-4.8.0.tgz", + "integrity": "sha512-QpELEHLO8SsQVtqP+MkEgCYTFW0pleGozfs3cZ183ZBj9z3VC1CX1/wtFMK64p+5bhtZo41SeLK1rBRtd25nHQ==", "dependencies": { "tslib": "^2.6.2" }, @@ -2452,12 +2452,12 @@ } }, "node_modules/@aws-sdk/client-s3vectors/node_modules/@smithy/url-parser": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/@smithy/url-parser/-/url-parser-4.1.0.tgz", - "integrity": "sha512-/LYEIOuO5B2u++tKr1NxNxhZTrr3A63jW8N73YTwVeUyAlbB/YM+hkftsvtKAcMt3ADYo0FsF1GY3anehffSVQ==", + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/@smithy/url-parser/-/url-parser-4.2.3.tgz", + "integrity": "sha512-I066AigYvY3d9VlU3zG9XzZg1yT10aNqvCaBTw9EPgu5GrsEl1aUkcMvhkIXascYH1A8W0LQo3B1Kr1cJNcQEw==", "dependencies": { - "@smithy/querystring-parser": "^4.1.0", - "@smithy/types": "^4.4.0", + "@smithy/querystring-parser": "^4.2.3", + "@smithy/types": "^4.8.0", "tslib": "^2.6.2" }, "engines": { @@ -2465,12 +2465,12 @@ } }, "node_modules/@aws-sdk/client-s3vectors/node_modules/@smithy/util-base64": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/@smithy/util-base64/-/util-base64-4.1.0.tgz", - "integrity": "sha512-RUGd4wNb8GeW7xk+AY5ghGnIwM96V0l2uzvs/uVHf+tIuVX2WSvynk5CxNoBCsM2rQRSZElAo9rt3G5mJ/gktQ==", + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/@smithy/util-base64/-/util-base64-4.3.0.tgz", + "integrity": "sha512-GkXZ59JfyxsIwNTWFnjmFEI8kZpRNIBfxKjv09+nkAWPt/4aGaEWMM04m4sxgNVWkbt2MdSvE3KF/PfX4nFedQ==", "dependencies": { - "@smithy/util-buffer-from": "^4.1.0", - "@smithy/util-utf8": "^4.1.0", + "@smithy/util-buffer-from": "^4.2.0", + "@smithy/util-utf8": "^4.2.0", "tslib": "^2.6.2" }, "engines": { @@ -2478,9 +2478,9 @@ } }, "node_modules/@aws-sdk/client-s3vectors/node_modules/@smithy/util-body-length-browser": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/@smithy/util-body-length-browser/-/util-body-length-browser-4.1.0.tgz", - "integrity": "sha512-V2E2Iez+bo6bUMOTENPr6eEmepdY8Hbs+Uc1vkDKgKNA/brTJqOW/ai3JO1BGj9GbCeLqw90pbbH7HFQyFotGQ==", + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/@smithy/util-body-length-browser/-/util-body-length-browser-4.2.0.tgz", + "integrity": "sha512-Fkoh/I76szMKJnBXWPdFkQJl2r9SjPt3cMzLdOB6eJ4Pnpas8hVoWPYemX/peO0yrrvldgCUVJqOAjUrOLjbxg==", "dependencies": { "tslib": "^2.6.2" }, @@ -2489,9 +2489,9 @@ } }, "node_modules/@aws-sdk/client-s3vectors/node_modules/@smithy/util-body-length-node": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/@smithy/util-body-length-node/-/util-body-length-node-4.1.0.tgz", - "integrity": "sha512-BOI5dYjheZdgR9XiEM3HJcEMCXSoqbzu7CzIgYrx0UtmvtC3tC2iDGpJLsSRFffUpy8ymsg2ARMP5fR8mtuUQQ==", + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/@smithy/util-body-length-node/-/util-body-length-node-4.2.1.tgz", + "integrity": "sha512-h53dz/pISVrVrfxV1iqXlx5pRg3V2YWFcSQyPyXZRrZoZj4R4DeWRDo1a7dd3CPTcFi3kE+98tuNyD2axyZReA==", "dependencies": { "tslib": "^2.6.2" }, @@ -2500,11 +2500,11 @@ } }, "node_modules/@aws-sdk/client-s3vectors/node_modules/@smithy/util-buffer-from": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-4.1.0.tgz", - "integrity": "sha512-N6yXcjfe/E+xKEccWEKzK6M+crMrlwaCepKja0pNnlSkm6SjAeLKKA++er5Ba0I17gvKfN/ThV+ZOx/CntKTVw==", + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-4.2.0.tgz", + "integrity": "sha512-kAY9hTKulTNevM2nlRtxAG2FQ3B2OR6QIrPY3zE5LqJy1oxzmgBGsHLWTcNhWXKchgA0WHW+mZkQrng/pgcCew==", "dependencies": { - "@smithy/is-array-buffer": "^4.1.0", + "@smithy/is-array-buffer": "^4.2.0", "tslib": "^2.6.2" }, "engines": { @@ -2512,9 +2512,9 @@ } }, "node_modules/@aws-sdk/client-s3vectors/node_modules/@smithy/util-config-provider": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/@smithy/util-config-provider/-/util-config-provider-4.1.0.tgz", - "integrity": "sha512-swXz2vMjrP1ZusZWVTB/ai5gK+J8U0BWvP10v9fpcFvg+Xi/87LHvHfst2IgCs1i0v4qFZfGwCmeD/KNCdJZbQ==", + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/@smithy/util-config-provider/-/util-config-provider-4.2.0.tgz", + "integrity": "sha512-YEjpl6XJ36FTKmD+kRJJWYvrHeUvm5ykaUS5xK+6oXffQPHeEM4/nXlZPe+Wu0lsgRUcNZiliYNh/y7q9c2y6Q==", "dependencies": { "tslib": "^2.6.2" }, @@ -2523,14 +2523,13 @@ } }, "node_modules/@aws-sdk/client-s3vectors/node_modules/@smithy/util-defaults-mode-browser": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/@smithy/util-defaults-mode-browser/-/util-defaults-mode-browser-4.1.0.tgz", - "integrity": "sha512-D27cLtJtC4EEeERJXS+JPoogz2tE5zeE3zhWSSu6ER5/wJ5gihUxIzoarDX6K1U27IFTHit5YfHqU4Y9RSGE0w==", + "version": "4.3.4", + "resolved": "https://registry.npmjs.org/@smithy/util-defaults-mode-browser/-/util-defaults-mode-browser-4.3.4.tgz", + "integrity": "sha512-qI5PJSW52rnutos8Bln8nwQZRpyoSRN6k2ajyoUHNMUzmWqHnOJCnDELJuV6m5PML0VkHI+XcXzdB+6awiqYUw==", "dependencies": { - "@smithy/property-provider": "^4.1.0", - "@smithy/smithy-client": "^4.6.0", - "@smithy/types": "^4.4.0", - "bowser": "^2.11.0", + "@smithy/property-provider": "^4.2.3", + "@smithy/smithy-client": "^4.9.1", + "@smithy/types": "^4.8.0", "tslib": "^2.6.2" }, "engines": { @@ -2538,16 +2537,16 @@ } }, "node_modules/@aws-sdk/client-s3vectors/node_modules/@smithy/util-defaults-mode-node": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/@smithy/util-defaults-mode-node/-/util-defaults-mode-node-4.1.0.tgz", - "integrity": "sha512-gnZo3u5dP1o87plKupg39alsbeIY1oFFnCyV2nI/++pL19vTtBLgOyftLEjPjuXmoKn2B2rskX8b7wtC/+3Okg==", + "version": "4.2.6", + "resolved": "https://registry.npmjs.org/@smithy/util-defaults-mode-node/-/util-defaults-mode-node-4.2.6.tgz", + "integrity": "sha512-c6M/ceBTm31YdcFpgfgQAJaw3KbaLuRKnAz91iMWFLSrgxRpYm03c3bu5cpYojNMfkV9arCUelelKA7XQT36SQ==", "dependencies": { - "@smithy/config-resolver": "^4.2.0", - "@smithy/credential-provider-imds": "^4.1.0", - "@smithy/node-config-provider": "^4.2.0", - "@smithy/property-provider": "^4.1.0", - "@smithy/smithy-client": "^4.6.0", - "@smithy/types": "^4.4.0", + "@smithy/config-resolver": "^4.4.0", + "@smithy/credential-provider-imds": "^4.2.3", + "@smithy/node-config-provider": "^4.3.3", + "@smithy/property-provider": "^4.2.3", + "@smithy/smithy-client": "^4.9.1", + "@smithy/types": "^4.8.0", "tslib": "^2.6.2" }, "engines": { @@ -2555,12 +2554,12 @@ } }, "node_modules/@aws-sdk/client-s3vectors/node_modules/@smithy/util-endpoints": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/@smithy/util-endpoints/-/util-endpoints-3.1.0.tgz", - "integrity": "sha512-5LFg48KkunBVGrNs3dnQgLlMXJLVo7k9sdZV5su3rjO3c3DmQ2LwUZI0Zr49p89JWK6sB7KmzyI2fVcDsZkwuw==", + "version": "3.2.3", + "resolved": "https://registry.npmjs.org/@smithy/util-endpoints/-/util-endpoints-3.2.3.tgz", + "integrity": "sha512-aCfxUOVv0CzBIkU10TubdgKSx5uRvzH064kaiPEWfNIvKOtNpu642P4FP1hgOFkjQIkDObrfIDnKMKkeyrejvQ==", "dependencies": { - "@smithy/node-config-provider": "^4.2.0", - "@smithy/types": "^4.4.0", + "@smithy/node-config-provider": "^4.3.3", + "@smithy/types": "^4.8.0", "tslib": "^2.6.2" }, "engines": { @@ -2568,9 +2567,9 @@ } }, "node_modules/@aws-sdk/client-s3vectors/node_modules/@smithy/util-hex-encoding": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/@smithy/util-hex-encoding/-/util-hex-encoding-4.1.0.tgz", - "integrity": "sha512-1LcueNN5GYC4tr8mo14yVYbh/Ur8jHhWOxniZXii+1+ePiIbsLZ5fEI0QQGtbRRP5mOhmooos+rLmVASGGoq5w==", + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/@smithy/util-hex-encoding/-/util-hex-encoding-4.2.0.tgz", + "integrity": "sha512-CCQBwJIvXMLKxVbO88IukazJD9a4kQ9ZN7/UMGBjBcJYvatpWk+9g870El4cB8/EJxfe+k+y0GmR9CAzkF+Nbw==", "dependencies": { "tslib": "^2.6.2" }, @@ -2579,11 +2578,11 @@ } }, "node_modules/@aws-sdk/client-s3vectors/node_modules/@smithy/util-middleware": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/@smithy/util-middleware/-/util-middleware-4.1.0.tgz", - "integrity": "sha512-612onNcKyxhP7/YOTKFTb2F6sPYtMRddlT5mZvYf1zduzaGzkYhpYIPxIeeEwBZFjnvEqe53Ijl2cYEfJ9d6/Q==", + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/@smithy/util-middleware/-/util-middleware-4.2.3.tgz", + "integrity": "sha512-v5ObKlSe8PWUHCqEiX2fy1gNv6goiw6E5I/PN2aXg3Fb/hse0xeaAnSpXDiWl7x6LamVKq7senB+m5LOYHUAHw==", "dependencies": { - "@smithy/types": "^4.4.0", + "@smithy/types": "^4.8.0", "tslib": "^2.6.2" }, "engines": { @@ -2591,12 +2590,12 @@ } }, "node_modules/@aws-sdk/client-s3vectors/node_modules/@smithy/util-retry": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/@smithy/util-retry/-/util-retry-4.1.0.tgz", - "integrity": "sha512-5AGoBHb207xAKSVwaUnaER+L55WFY8o2RhlafELZR3mB0J91fpL+Qn+zgRkPzns3kccGaF2vy0HmNVBMWmN6dA==", + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/@smithy/util-retry/-/util-retry-4.2.3.tgz", + "integrity": "sha512-lLPWnakjC0q9z+OtiXk+9RPQiYPNAovt2IXD3CP4LkOnd9NpUsxOjMx1SnoUVB7Orb7fZp67cQMtTBKMFDvOGg==", "dependencies": { - "@smithy/service-error-classification": "^4.1.0", - "@smithy/types": "^4.4.0", + "@smithy/service-error-classification": "^4.2.3", + "@smithy/types": "^4.8.0", "tslib": "^2.6.2" }, "engines": { @@ -2604,17 +2603,17 @@ } }, "node_modules/@aws-sdk/client-s3vectors/node_modules/@smithy/util-stream": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/@smithy/util-stream/-/util-stream-4.3.0.tgz", - "integrity": "sha512-ZOYS94jksDwvsCJtppHprUhsIscRnCKGr6FXCo3SxgQ31ECbza3wqDBqSy6IsAak+h/oAXb1+UYEBmDdseAjUQ==", + "version": "4.5.4", + "resolved": "https://registry.npmjs.org/@smithy/util-stream/-/util-stream-4.5.4.tgz", + "integrity": "sha512-+qDxSkiErejw1BAIXUFBSfM5xh3arbz1MmxlbMCKanDDZtVEQ7PSKW9FQS0Vud1eI/kYn0oCTVKyNzRlq+9MUw==", "dependencies": { - "@smithy/fetch-http-handler": "^5.2.0", - "@smithy/node-http-handler": "^4.2.0", - "@smithy/types": "^4.4.0", - "@smithy/util-base64": "^4.1.0", - "@smithy/util-buffer-from": "^4.1.0", - "@smithy/util-hex-encoding": "^4.1.0", - "@smithy/util-utf8": "^4.1.0", + "@smithy/fetch-http-handler": "^5.3.4", + "@smithy/node-http-handler": "^4.4.3", + "@smithy/types": "^4.8.0", + "@smithy/util-base64": "^4.3.0", + "@smithy/util-buffer-from": "^4.2.0", + "@smithy/util-hex-encoding": "^4.2.0", + "@smithy/util-utf8": "^4.2.0", "tslib": "^2.6.2" }, "engines": { @@ -2622,9 +2621,9 @@ } }, "node_modules/@aws-sdk/client-s3vectors/node_modules/@smithy/util-uri-escape": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/@smithy/util-uri-escape/-/util-uri-escape-4.1.0.tgz", - "integrity": "sha512-b0EFQkq35K5NHUYxU72JuoheM6+pytEVUGlTwiFxWFpmddA+Bpz3LgsPRIpBk8lnPE47yT7AF2Egc3jVnKLuPg==", + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/@smithy/util-uri-escape/-/util-uri-escape-4.2.0.tgz", + "integrity": "sha512-igZpCKV9+E/Mzrpq6YacdTQ0qTiLm85gD6N/IrmyDvQFA4UnU3d5g3m8tMT/6zG/vVkWSU+VxeUyGonL62DuxA==", "dependencies": { "tslib": "^2.6.2" }, @@ -2633,11 +2632,11 @@ } }, "node_modules/@aws-sdk/client-s3vectors/node_modules/@smithy/util-utf8": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-4.1.0.tgz", - "integrity": "sha512-mEu1/UIXAdNYuBcyEPbjScKi/+MQVXNIuY/7Cm5XLIWe319kDrT5SizBE95jqtmEXoDbGoZxKLCMttdZdqTZKQ==", + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-4.2.0.tgz", + "integrity": "sha512-zBPfuzoI8xyBtR2P6WQj63Rz8i3AmfAaJLuNG8dWsfvPe8lO4aCPYLn879mEgHndZH1zQ2oXmG8O1GGzzaoZiw==", "dependencies": { - "@smithy/util-buffer-from": "^4.1.0", + "@smithy/util-buffer-from": "^4.2.0", "tslib": "^2.6.2" }, "engines": { @@ -9918,6 +9917,14 @@ "node": ">=16.0.0" } }, + "node_modules/@aws/lambda-invoke-store": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/@aws/lambda-invoke-store/-/lambda-invoke-store-0.1.1.tgz", + "integrity": "sha512-RcLam17LdlbSOSp9VxmUu1eI6Mwxp+OwhD2QhiSNmNCzoDb0EeUXTD2n/WbcnrAYMGlmf05th6QYq23VqvJqpA==", + "engines": { + "node": ">=18.0.0" + } + }, "node_modules/@babel/code-frame": { "version": "7.26.2", "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.26.2.tgz", @@ -15702,6 +15709,17 @@ "node": ">=16.0.0" } }, + "node_modules/@smithy/uuid": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@smithy/uuid/-/uuid-1.1.0.tgz", + "integrity": "sha512-4aUIteuyxtBUhVdiQqcDhKFitwfd9hqoSDYY2KRXiWtgoWJ9Bmise+KfEPDiVHWeJepvF8xJO9/9+WDIciMFFw==", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, "node_modules/@tus/file-store": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/@tus/file-store/-/file-store-2.0.0.tgz", @@ -16992,6 +17010,12 @@ "resolved": "https://registry.npmjs.org/@types/js-yaml/-/js-yaml-4.0.5.tgz", "integrity": "sha512-FhpRzf927MNQdRZP0J5DLIdTXhjLYzeUTmLAu69mnVksLH9CJY3IuSeEgbKUki7GQZm0WqDkGzyxju2EZGD2wA==" }, + "node_modules/@types/json-bigint": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@types/json-bigint/-/json-bigint-1.0.4.tgz", + "integrity": "sha512-ydHooXLbOmxBbubnA7Eh+RpBzuaIiQjh8WGJYQB50JFGFrdxW7JzVlyEV7fAXw0T2sqJ1ysTneJbiyNLqZRAag==", + "dev": true + }, "node_modules/@types/json-schema": { "version": "7.0.9", "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.9.tgz", @@ -25480,809 +25504,806 @@ } }, "@aws-sdk/client-s3vectors": { - "version": "3.883.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-s3vectors/-/client-s3vectors-3.883.0.tgz", - "integrity": "sha512-QtGyHqvih3it25HFnxudHg6J6FvlpSQs9aSN2IgpofAQGLENGVtKFtVlFay2KOlUFPh5aL1OOl5ONR70O8q/Eg==", + "version": "3.919.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-s3vectors/-/client-s3vectors-3.919.0.tgz", + "integrity": "sha512-zEGUf1Nhijx5ldZJYSusrXCHMOOCZ53YYif4HYCfveUD3xp0ZmWuRnhlhGU/UDykZEa4OzZx7nxK+4UkcnKhdA==", "requires": { "@aws-crypto/sha256-browser": "5.2.0", "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "3.883.0", - "@aws-sdk/credential-provider-node": "3.883.0", - "@aws-sdk/middleware-host-header": "3.873.0", - "@aws-sdk/middleware-logger": "3.876.0", - "@aws-sdk/middleware-recursion-detection": "3.873.0", - "@aws-sdk/middleware-user-agent": "3.883.0", - "@aws-sdk/region-config-resolver": "3.873.0", - "@aws-sdk/types": "3.862.0", - "@aws-sdk/util-endpoints": "3.879.0", - "@aws-sdk/util-user-agent-browser": "3.873.0", - "@aws-sdk/util-user-agent-node": "3.883.0", - "@smithy/config-resolver": "^4.1.5", - "@smithy/core": "^3.9.2", - "@smithy/fetch-http-handler": "^5.1.1", - "@smithy/hash-node": "^4.0.5", - "@smithy/invalid-dependency": "^4.0.5", - "@smithy/middleware-content-length": "^4.0.5", - "@smithy/middleware-endpoint": "^4.1.21", - "@smithy/middleware-retry": "^4.1.22", - "@smithy/middleware-serde": "^4.0.9", - "@smithy/middleware-stack": "^4.0.5", - "@smithy/node-config-provider": "^4.1.4", - "@smithy/node-http-handler": "^4.1.1", - "@smithy/protocol-http": "^5.1.3", - "@smithy/smithy-client": "^4.5.2", - "@smithy/types": "^4.3.2", - "@smithy/url-parser": "^4.0.5", - "@smithy/util-base64": "^4.0.0", - "@smithy/util-body-length-browser": "^4.0.0", - "@smithy/util-body-length-node": "^4.0.0", - "@smithy/util-defaults-mode-browser": "^4.0.29", - "@smithy/util-defaults-mode-node": "^4.0.29", - "@smithy/util-endpoints": "^3.0.7", - "@smithy/util-middleware": "^4.0.5", - "@smithy/util-retry": "^4.0.7", - "@smithy/util-utf8": "^4.0.0", + "@aws-sdk/core": "3.916.0", + "@aws-sdk/credential-provider-node": "3.919.0", + "@aws-sdk/middleware-host-header": "3.914.0", + "@aws-sdk/middleware-logger": "3.914.0", + "@aws-sdk/middleware-recursion-detection": "3.919.0", + "@aws-sdk/middleware-user-agent": "3.916.0", + "@aws-sdk/region-config-resolver": "3.914.0", + "@aws-sdk/types": "3.914.0", + "@aws-sdk/util-endpoints": "3.916.0", + "@aws-sdk/util-user-agent-browser": "3.914.0", + "@aws-sdk/util-user-agent-node": "3.916.0", + "@smithy/config-resolver": "^4.4.0", + "@smithy/core": "^3.17.1", + "@smithy/fetch-http-handler": "^5.3.4", + "@smithy/hash-node": "^4.2.3", + "@smithy/invalid-dependency": "^4.2.3", + "@smithy/middleware-content-length": "^4.2.3", + "@smithy/middleware-endpoint": "^4.3.5", + "@smithy/middleware-retry": "^4.4.5", + "@smithy/middleware-serde": "^4.2.3", + "@smithy/middleware-stack": "^4.2.3", + "@smithy/node-config-provider": "^4.3.3", + "@smithy/node-http-handler": "^4.4.3", + "@smithy/protocol-http": "^5.3.3", + "@smithy/smithy-client": "^4.9.1", + "@smithy/types": "^4.8.0", + "@smithy/url-parser": "^4.2.3", + "@smithy/util-base64": "^4.3.0", + "@smithy/util-body-length-browser": "^4.2.0", + "@smithy/util-body-length-node": "^4.2.1", + "@smithy/util-defaults-mode-browser": "^4.3.4", + "@smithy/util-defaults-mode-node": "^4.2.6", + "@smithy/util-endpoints": "^3.2.3", + "@smithy/util-middleware": "^4.2.3", + "@smithy/util-retry": "^4.2.3", + "@smithy/util-utf8": "^4.2.0", "tslib": "^2.6.2" }, "dependencies": { "@aws-sdk/client-sso": { - "version": "3.883.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-sso/-/client-sso-3.883.0.tgz", - "integrity": "sha512-Ybjw76yPceEBO7+VLjy5+/Gr0A1UNymSDHda5w8tfsS2iHZt/vuD6wrYpHdLoUx4H5la8ZhwcSfK/+kmE+QLPw==", + "version": "3.919.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-sso/-/client-sso-3.919.0.tgz", + "integrity": "sha512-9DVw/1DCzZ9G7Jofnhpg/XDC3wdJ3NAJdNWY1TrgE5ZcpTM+UTIQMGyaljCv9rgxggutHBgmBI5lP3YMcPk9ZQ==", "requires": { "@aws-crypto/sha256-browser": "5.2.0", "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "3.883.0", - "@aws-sdk/middleware-host-header": "3.873.0", - "@aws-sdk/middleware-logger": "3.876.0", - "@aws-sdk/middleware-recursion-detection": "3.873.0", - "@aws-sdk/middleware-user-agent": "3.883.0", - "@aws-sdk/region-config-resolver": "3.873.0", - "@aws-sdk/types": "3.862.0", - "@aws-sdk/util-endpoints": "3.879.0", - "@aws-sdk/util-user-agent-browser": "3.873.0", - "@aws-sdk/util-user-agent-node": "3.883.0", - "@smithy/config-resolver": "^4.1.5", - "@smithy/core": "^3.9.2", - "@smithy/fetch-http-handler": "^5.1.1", - "@smithy/hash-node": "^4.0.5", - "@smithy/invalid-dependency": "^4.0.5", - "@smithy/middleware-content-length": "^4.0.5", - "@smithy/middleware-endpoint": "^4.1.21", - "@smithy/middleware-retry": "^4.1.22", - "@smithy/middleware-serde": "^4.0.9", - "@smithy/middleware-stack": "^4.0.5", - "@smithy/node-config-provider": "^4.1.4", - "@smithy/node-http-handler": "^4.1.1", - "@smithy/protocol-http": "^5.1.3", - "@smithy/smithy-client": "^4.5.2", - "@smithy/types": "^4.3.2", - "@smithy/url-parser": "^4.0.5", - "@smithy/util-base64": "^4.0.0", - "@smithy/util-body-length-browser": "^4.0.0", - "@smithy/util-body-length-node": "^4.0.0", - "@smithy/util-defaults-mode-browser": "^4.0.29", - "@smithy/util-defaults-mode-node": "^4.0.29", - "@smithy/util-endpoints": "^3.0.7", - "@smithy/util-middleware": "^4.0.5", - "@smithy/util-retry": "^4.0.7", - "@smithy/util-utf8": "^4.0.0", + "@aws-sdk/core": "3.916.0", + "@aws-sdk/middleware-host-header": "3.914.0", + "@aws-sdk/middleware-logger": "3.914.0", + "@aws-sdk/middleware-recursion-detection": "3.919.0", + "@aws-sdk/middleware-user-agent": "3.916.0", + "@aws-sdk/region-config-resolver": "3.914.0", + "@aws-sdk/types": "3.914.0", + "@aws-sdk/util-endpoints": "3.916.0", + "@aws-sdk/util-user-agent-browser": "3.914.0", + "@aws-sdk/util-user-agent-node": "3.916.0", + "@smithy/config-resolver": "^4.4.0", + "@smithy/core": "^3.17.1", + "@smithy/fetch-http-handler": "^5.3.4", + "@smithy/hash-node": "^4.2.3", + "@smithy/invalid-dependency": "^4.2.3", + "@smithy/middleware-content-length": "^4.2.3", + "@smithy/middleware-endpoint": "^4.3.5", + "@smithy/middleware-retry": "^4.4.5", + "@smithy/middleware-serde": "^4.2.3", + "@smithy/middleware-stack": "^4.2.3", + "@smithy/node-config-provider": "^4.3.3", + "@smithy/node-http-handler": "^4.4.3", + "@smithy/protocol-http": "^5.3.3", + "@smithy/smithy-client": "^4.9.1", + "@smithy/types": "^4.8.0", + "@smithy/url-parser": "^4.2.3", + "@smithy/util-base64": "^4.3.0", + "@smithy/util-body-length-browser": "^4.2.0", + "@smithy/util-body-length-node": "^4.2.1", + "@smithy/util-defaults-mode-browser": "^4.3.4", + "@smithy/util-defaults-mode-node": "^4.2.6", + "@smithy/util-endpoints": "^3.2.3", + "@smithy/util-middleware": "^4.2.3", + "@smithy/util-retry": "^4.2.3", + "@smithy/util-utf8": "^4.2.0", "tslib": "^2.6.2" } }, "@aws-sdk/core": { - "version": "3.883.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/core/-/core-3.883.0.tgz", - "integrity": "sha512-FmkqnqBLkXi4YsBPbF6vzPa0m4XKUuvgKDbamfw4DZX2CzfBZH6UU4IwmjNV3ZM38m0xraHarK8KIbGSadN3wg==", - "requires": { - "@aws-sdk/types": "3.862.0", - "@aws-sdk/xml-builder": "3.873.0", - "@smithy/core": "^3.9.2", - "@smithy/node-config-provider": "^4.1.4", - "@smithy/property-provider": "^4.0.5", - "@smithy/protocol-http": "^5.1.3", - "@smithy/signature-v4": "^5.1.3", - "@smithy/smithy-client": "^4.5.2", - "@smithy/types": "^4.3.2", - "@smithy/util-base64": "^4.0.0", - "@smithy/util-body-length-browser": "^4.0.0", - "@smithy/util-middleware": "^4.0.5", - "@smithy/util-utf8": "^4.0.0", - "fast-xml-parser": "5.2.5", + "version": "3.916.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/core/-/core-3.916.0.tgz", + "integrity": "sha512-1JHE5s6MD5PKGovmx/F1e01hUbds/1y3X8rD+Gvi/gWVfdg5noO7ZCerpRsWgfzgvCMZC9VicopBqNHCKLykZA==", + "requires": { + "@aws-sdk/types": "3.914.0", + "@aws-sdk/xml-builder": "3.914.0", + "@smithy/core": "^3.17.1", + "@smithy/node-config-provider": "^4.3.3", + "@smithy/property-provider": "^4.2.3", + "@smithy/protocol-http": "^5.3.3", + "@smithy/signature-v4": "^5.3.3", + "@smithy/smithy-client": "^4.9.1", + "@smithy/types": "^4.8.0", + "@smithy/util-base64": "^4.3.0", + "@smithy/util-middleware": "^4.2.3", + "@smithy/util-utf8": "^4.2.0", "tslib": "^2.6.2" } }, "@aws-sdk/credential-provider-env": { - "version": "3.883.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-env/-/credential-provider-env-3.883.0.tgz", - "integrity": "sha512-Z6tPBXPCodfhIF1rvQKoeRGMkwL6TK0xdl1UoMIA1x4AfBpPICAF77JkFBExk/pdiFYq1d04Qzddd/IiujSlLg==", + "version": "3.916.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-env/-/credential-provider-env-3.916.0.tgz", + "integrity": "sha512-3gDeqOXcBRXGHScc6xb7358Lyf64NRG2P08g6Bu5mv1Vbg9PKDyCAZvhKLkG7hkdfAM8Yc6UJNhbFxr1ud/tCQ==", "requires": { - "@aws-sdk/core": "3.883.0", - "@aws-sdk/types": "3.862.0", - "@smithy/property-provider": "^4.0.5", - "@smithy/types": "^4.3.2", + "@aws-sdk/core": "3.916.0", + "@aws-sdk/types": "3.914.0", + "@smithy/property-provider": "^4.2.3", + "@smithy/types": "^4.8.0", "tslib": "^2.6.2" } }, "@aws-sdk/credential-provider-http": { - "version": "3.883.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-http/-/credential-provider-http-3.883.0.tgz", - "integrity": "sha512-P589ug1lMOOEYLTaQJjSP+Gee34za8Kk2LfteNQfO9SpByHFgGj++Sg8VyIe30eZL8Q+i4qTt24WDCz1c+dgYg==", + "version": "3.916.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-http/-/credential-provider-http-3.916.0.tgz", + "integrity": "sha512-NmooA5Z4/kPFJdsyoJgDxuqXC1C6oPMmreJjbOPqcwo6E/h2jxaG8utlQFgXe5F9FeJsMx668dtxVxSYnAAqHQ==", "requires": { - "@aws-sdk/core": "3.883.0", - "@aws-sdk/types": "3.862.0", - "@smithy/fetch-http-handler": "^5.1.1", - "@smithy/node-http-handler": "^4.1.1", - "@smithy/property-provider": "^4.0.5", - "@smithy/protocol-http": "^5.1.3", - "@smithy/smithy-client": "^4.5.2", - "@smithy/types": "^4.3.2", - "@smithy/util-stream": "^4.2.4", + "@aws-sdk/core": "3.916.0", + "@aws-sdk/types": "3.914.0", + "@smithy/fetch-http-handler": "^5.3.4", + "@smithy/node-http-handler": "^4.4.3", + "@smithy/property-provider": "^4.2.3", + "@smithy/protocol-http": "^5.3.3", + "@smithy/smithy-client": "^4.9.1", + "@smithy/types": "^4.8.0", + "@smithy/util-stream": "^4.5.4", "tslib": "^2.6.2" } }, "@aws-sdk/credential-provider-ini": { - "version": "3.883.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-ini/-/credential-provider-ini-3.883.0.tgz", - "integrity": "sha512-n6z9HTzuDEdugXvPiE/95VJXbF4/gBffdV/SRHDJKtDHaRuvp/gggbfmfVSTFouGVnlKPb2pQWQsW3Nr/Y3Lrw==", + "version": "3.919.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-ini/-/credential-provider-ini-3.919.0.tgz", + "integrity": "sha512-fAWVfh0P54UFbyAK4tmIPh/X3COFAyXYSp8b2Pc1R6GRwDDMvrAigwGJuyZS4BmpPlXij1gB0nXbhM5Yo4MMMA==", "requires": { - "@aws-sdk/core": "3.883.0", - "@aws-sdk/credential-provider-env": "3.883.0", - "@aws-sdk/credential-provider-http": "3.883.0", - "@aws-sdk/credential-provider-process": "3.883.0", - "@aws-sdk/credential-provider-sso": "3.883.0", - "@aws-sdk/credential-provider-web-identity": "3.883.0", - "@aws-sdk/nested-clients": "3.883.0", - "@aws-sdk/types": "3.862.0", - "@smithy/credential-provider-imds": "^4.0.7", - "@smithy/property-provider": "^4.0.5", - "@smithy/shared-ini-file-loader": "^4.0.5", - "@smithy/types": "^4.3.2", + "@aws-sdk/core": "3.916.0", + "@aws-sdk/credential-provider-env": "3.916.0", + "@aws-sdk/credential-provider-http": "3.916.0", + "@aws-sdk/credential-provider-process": "3.916.0", + "@aws-sdk/credential-provider-sso": "3.919.0", + "@aws-sdk/credential-provider-web-identity": "3.919.0", + "@aws-sdk/nested-clients": "3.919.0", + "@aws-sdk/types": "3.914.0", + "@smithy/credential-provider-imds": "^4.2.3", + "@smithy/property-provider": "^4.2.3", + "@smithy/shared-ini-file-loader": "^4.3.3", + "@smithy/types": "^4.8.0", "tslib": "^2.6.2" } }, "@aws-sdk/credential-provider-node": { - "version": "3.883.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-node/-/credential-provider-node-3.883.0.tgz", - "integrity": "sha512-QIUhsatsrwfB9ZsKpmi0EySSfexVP61wgN7hr493DOileh2QsKW4XATEfsWNmx0dj9323Vg1Mix7bXtRfl9cGg==", + "version": "3.919.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-node/-/credential-provider-node-3.919.0.tgz", + "integrity": "sha512-GL5filyxYS+eZq8ZMQnY5hh79Wxor7Rljo0SUJxZVwEj8cf3zY0MMuwoXU1HQrVabvYtkPDOWSreX8GkIBtBCw==", "requires": { - "@aws-sdk/credential-provider-env": "3.883.0", - "@aws-sdk/credential-provider-http": "3.883.0", - "@aws-sdk/credential-provider-ini": "3.883.0", - "@aws-sdk/credential-provider-process": "3.883.0", - "@aws-sdk/credential-provider-sso": "3.883.0", - "@aws-sdk/credential-provider-web-identity": "3.883.0", - "@aws-sdk/types": "3.862.0", - "@smithy/credential-provider-imds": "^4.0.7", - "@smithy/property-provider": "^4.0.5", - "@smithy/shared-ini-file-loader": "^4.0.5", - "@smithy/types": "^4.3.2", + "@aws-sdk/credential-provider-env": "3.916.0", + "@aws-sdk/credential-provider-http": "3.916.0", + "@aws-sdk/credential-provider-ini": "3.919.0", + "@aws-sdk/credential-provider-process": "3.916.0", + "@aws-sdk/credential-provider-sso": "3.919.0", + "@aws-sdk/credential-provider-web-identity": "3.919.0", + "@aws-sdk/types": "3.914.0", + "@smithy/credential-provider-imds": "^4.2.3", + "@smithy/property-provider": "^4.2.3", + "@smithy/shared-ini-file-loader": "^4.3.3", + "@smithy/types": "^4.8.0", "tslib": "^2.6.2" } }, "@aws-sdk/credential-provider-process": { - "version": "3.883.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-process/-/credential-provider-process-3.883.0.tgz", - "integrity": "sha512-m1shbHY/Vppy4EdddG9r8x64TO/9FsCjokp5HbKcZvVoTOTgUJrdT8q2TAQJ89+zYIJDqsKbqfrmfwJ1zOdnGQ==", + "version": "3.916.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-process/-/credential-provider-process-3.916.0.tgz", + "integrity": "sha512-SXDyDvpJ1+WbotZDLJW1lqP6gYGaXfZJrgFSXIuZjHb75fKeNRgPkQX/wZDdUvCwdrscvxmtyJorp2sVYkMcvA==", "requires": { - "@aws-sdk/core": "3.883.0", - "@aws-sdk/types": "3.862.0", - "@smithy/property-provider": "^4.0.5", - "@smithy/shared-ini-file-loader": "^4.0.5", - "@smithy/types": "^4.3.2", + "@aws-sdk/core": "3.916.0", + "@aws-sdk/types": "3.914.0", + "@smithy/property-provider": "^4.2.3", + "@smithy/shared-ini-file-loader": "^4.3.3", + "@smithy/types": "^4.8.0", "tslib": "^2.6.2" } }, "@aws-sdk/credential-provider-sso": { - "version": "3.883.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-sso/-/credential-provider-sso-3.883.0.tgz", - "integrity": "sha512-37ve9Tult08HLXrJFHJM/sGB/vO7wzI6v1RUUfeTiShqx8ZQ5fTzCTNY/duO96jCtCexmFNSycpQzh7lDIf0aA==", + "version": "3.919.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-sso/-/credential-provider-sso-3.919.0.tgz", + "integrity": "sha512-oN1XG/frOc2K2KdVwRQjLTBLM1oSFJLtOhuV/6g9N0ASD+44uVJai1CF9JJv5GjHGV+wsqAt+/Dzde0tZEXirA==", "requires": { - "@aws-sdk/client-sso": "3.883.0", - "@aws-sdk/core": "3.883.0", - "@aws-sdk/token-providers": "3.883.0", - "@aws-sdk/types": "3.862.0", - "@smithy/property-provider": "^4.0.5", - "@smithy/shared-ini-file-loader": "^4.0.5", - "@smithy/types": "^4.3.2", + "@aws-sdk/client-sso": "3.919.0", + "@aws-sdk/core": "3.916.0", + "@aws-sdk/token-providers": "3.919.0", + "@aws-sdk/types": "3.914.0", + "@smithy/property-provider": "^4.2.3", + "@smithy/shared-ini-file-loader": "^4.3.3", + "@smithy/types": "^4.8.0", "tslib": "^2.6.2" } }, "@aws-sdk/credential-provider-web-identity": { - "version": "3.883.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-web-identity/-/credential-provider-web-identity-3.883.0.tgz", - "integrity": "sha512-SL82K9Jb0vpuTadqTO4Fpdu7SKtebZ3Yo4LZvk/U0UauVMlJj5ZTos0mFx1QSMB9/4TpqifYrSZcdnxgYg8Eqw==", + "version": "3.919.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-web-identity/-/credential-provider-web-identity-3.919.0.tgz", + "integrity": "sha512-Wi7RmyWA8kUJ++/8YceC7U5r4LyvOHGCnJLDHliP8rOC8HLdSgxw/Upeq3WmC+RPw1zyGOtEDRS/caop2xLXEA==", "requires": { - "@aws-sdk/core": "3.883.0", - "@aws-sdk/nested-clients": "3.883.0", - "@aws-sdk/types": "3.862.0", - "@smithy/property-provider": "^4.0.5", - "@smithy/types": "^4.3.2", + "@aws-sdk/core": "3.916.0", + "@aws-sdk/nested-clients": "3.919.0", + "@aws-sdk/types": "3.914.0", + "@smithy/property-provider": "^4.2.3", + "@smithy/shared-ini-file-loader": "^4.3.3", + "@smithy/types": "^4.8.0", "tslib": "^2.6.2" } }, "@aws-sdk/middleware-host-header": { - "version": "3.873.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-host-header/-/middleware-host-header-3.873.0.tgz", - "integrity": "sha512-KZ/W1uruWtMOs7D5j3KquOxzCnV79KQW9MjJFZM/M0l6KI8J6V3718MXxFHsTjUE4fpdV6SeCNLV1lwGygsjJA==", + "version": "3.914.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-host-header/-/middleware-host-header-3.914.0.tgz", + "integrity": "sha512-7r9ToySQ15+iIgXMF/h616PcQStByylVkCshmQqcdeynD/lCn2l667ynckxW4+ql0Q+Bo/URljuhJRxVJzydNA==", "requires": { - "@aws-sdk/types": "3.862.0", - "@smithy/protocol-http": "^5.1.3", - "@smithy/types": "^4.3.2", + "@aws-sdk/types": "3.914.0", + "@smithy/protocol-http": "^5.3.3", + "@smithy/types": "^4.8.0", "tslib": "^2.6.2" } }, "@aws-sdk/middleware-logger": { - "version": "3.876.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-logger/-/middleware-logger-3.876.0.tgz", - "integrity": "sha512-cpWJhOuMSyz9oV25Z/CMHCBTgafDCbv7fHR80nlRrPdPZ8ETNsahwRgltXP1QJJ8r3X/c1kwpOR7tc+RabVzNA==", + "version": "3.914.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-logger/-/middleware-logger-3.914.0.tgz", + "integrity": "sha512-/gaW2VENS5vKvJbcE1umV4Ag3NuiVzpsANxtrqISxT3ovyro29o1RezW/Avz/6oJqjnmgz8soe9J1t65jJdiNg==", "requires": { - "@aws-sdk/types": "3.862.0", - "@smithy/types": "^4.3.2", + "@aws-sdk/types": "3.914.0", + "@smithy/types": "^4.8.0", "tslib": "^2.6.2" } }, "@aws-sdk/middleware-recursion-detection": { - "version": "3.873.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-recursion-detection/-/middleware-recursion-detection-3.873.0.tgz", - "integrity": "sha512-OtgY8EXOzRdEWR//WfPkA/fXl0+WwE8hq0y9iw2caNyKPtca85dzrrZWnPqyBK/cpImosrpR1iKMYr41XshsCg==", + "version": "3.919.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-recursion-detection/-/middleware-recursion-detection-3.919.0.tgz", + "integrity": "sha512-q3MAUxLQve4rTfAannUCx2q1kAHkBBsxt6hVUpzi63KC4lBLScc1ltr7TI+hDxlfGRWGo54jRegb2SsY9Jm+Mw==", "requires": { - "@aws-sdk/types": "3.862.0", - "@smithy/protocol-http": "^5.1.3", - "@smithy/types": "^4.3.2", + "@aws-sdk/types": "3.914.0", + "@aws/lambda-invoke-store": "^0.1.1", + "@smithy/protocol-http": "^5.3.3", + "@smithy/types": "^4.8.0", "tslib": "^2.6.2" } }, "@aws-sdk/middleware-user-agent": { - "version": "3.883.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-user-agent/-/middleware-user-agent-3.883.0.tgz", - "integrity": "sha512-q58uLYnGLg7hsnWpdj7Cd1Ulsq1/PUJOHvAfgcBuiDE/+Fwh0DZxZZyjrU+Cr+dbeowIdUaOO8BEDDJ0CUenJw==", + "version": "3.916.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-user-agent/-/middleware-user-agent-3.916.0.tgz", + "integrity": "sha512-mzF5AdrpQXc2SOmAoaQeHpDFsK2GE6EGcEACeNuoESluPI2uYMpuuNMYrUufdnIAIyqgKlis0NVxiahA5jG42w==", "requires": { - "@aws-sdk/core": "3.883.0", - "@aws-sdk/types": "3.862.0", - "@aws-sdk/util-endpoints": "3.879.0", - "@smithy/core": "^3.9.2", - "@smithy/protocol-http": "^5.1.3", - "@smithy/types": "^4.3.2", + "@aws-sdk/core": "3.916.0", + "@aws-sdk/types": "3.914.0", + "@aws-sdk/util-endpoints": "3.916.0", + "@smithy/core": "^3.17.1", + "@smithy/protocol-http": "^5.3.3", + "@smithy/types": "^4.8.0", "tslib": "^2.6.2" } }, "@aws-sdk/nested-clients": { - "version": "3.883.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/nested-clients/-/nested-clients-3.883.0.tgz", - "integrity": "sha512-IhzDM+v0ga53GOOrZ9jmGNr7JU5OR6h6ZK9NgB7GXaa+gsDbqfUuXRwyKDYXldrTXf1sUR3vy1okWDXA7S2ejQ==", + "version": "3.919.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/nested-clients/-/nested-clients-3.919.0.tgz", + "integrity": "sha512-5D9OQsMPkbkp4KHM7JZv/RcGCpr3E1L7XX7U9sCxY+sFGeysltoviTmaIBXsJ2IjAJbBULtf0G/J+2cfH5OP+w==", "requires": { "@aws-crypto/sha256-browser": "5.2.0", "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "3.883.0", - "@aws-sdk/middleware-host-header": "3.873.0", - "@aws-sdk/middleware-logger": "3.876.0", - "@aws-sdk/middleware-recursion-detection": "3.873.0", - "@aws-sdk/middleware-user-agent": "3.883.0", - "@aws-sdk/region-config-resolver": "3.873.0", - "@aws-sdk/types": "3.862.0", - "@aws-sdk/util-endpoints": "3.879.0", - "@aws-sdk/util-user-agent-browser": "3.873.0", - "@aws-sdk/util-user-agent-node": "3.883.0", - "@smithy/config-resolver": "^4.1.5", - "@smithy/core": "^3.9.2", - "@smithy/fetch-http-handler": "^5.1.1", - "@smithy/hash-node": "^4.0.5", - "@smithy/invalid-dependency": "^4.0.5", - "@smithy/middleware-content-length": "^4.0.5", - "@smithy/middleware-endpoint": "^4.1.21", - "@smithy/middleware-retry": "^4.1.22", - "@smithy/middleware-serde": "^4.0.9", - "@smithy/middleware-stack": "^4.0.5", - "@smithy/node-config-provider": "^4.1.4", - "@smithy/node-http-handler": "^4.1.1", - "@smithy/protocol-http": "^5.1.3", - "@smithy/smithy-client": "^4.5.2", - "@smithy/types": "^4.3.2", - "@smithy/url-parser": "^4.0.5", - "@smithy/util-base64": "^4.0.0", - "@smithy/util-body-length-browser": "^4.0.0", - "@smithy/util-body-length-node": "^4.0.0", - "@smithy/util-defaults-mode-browser": "^4.0.29", - "@smithy/util-defaults-mode-node": "^4.0.29", - "@smithy/util-endpoints": "^3.0.7", - "@smithy/util-middleware": "^4.0.5", - "@smithy/util-retry": "^4.0.7", - "@smithy/util-utf8": "^4.0.0", + "@aws-sdk/core": "3.916.0", + "@aws-sdk/middleware-host-header": "3.914.0", + "@aws-sdk/middleware-logger": "3.914.0", + "@aws-sdk/middleware-recursion-detection": "3.919.0", + "@aws-sdk/middleware-user-agent": "3.916.0", + "@aws-sdk/region-config-resolver": "3.914.0", + "@aws-sdk/types": "3.914.0", + "@aws-sdk/util-endpoints": "3.916.0", + "@aws-sdk/util-user-agent-browser": "3.914.0", + "@aws-sdk/util-user-agent-node": "3.916.0", + "@smithy/config-resolver": "^4.4.0", + "@smithy/core": "^3.17.1", + "@smithy/fetch-http-handler": "^5.3.4", + "@smithy/hash-node": "^4.2.3", + "@smithy/invalid-dependency": "^4.2.3", + "@smithy/middleware-content-length": "^4.2.3", + "@smithy/middleware-endpoint": "^4.3.5", + "@smithy/middleware-retry": "^4.4.5", + "@smithy/middleware-serde": "^4.2.3", + "@smithy/middleware-stack": "^4.2.3", + "@smithy/node-config-provider": "^4.3.3", + "@smithy/node-http-handler": "^4.4.3", + "@smithy/protocol-http": "^5.3.3", + "@smithy/smithy-client": "^4.9.1", + "@smithy/types": "^4.8.0", + "@smithy/url-parser": "^4.2.3", + "@smithy/util-base64": "^4.3.0", + "@smithy/util-body-length-browser": "^4.2.0", + "@smithy/util-body-length-node": "^4.2.1", + "@smithy/util-defaults-mode-browser": "^4.3.4", + "@smithy/util-defaults-mode-node": "^4.2.6", + "@smithy/util-endpoints": "^3.2.3", + "@smithy/util-middleware": "^4.2.3", + "@smithy/util-retry": "^4.2.3", + "@smithy/util-utf8": "^4.2.0", "tslib": "^2.6.2" } }, "@aws-sdk/region-config-resolver": { - "version": "3.873.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/region-config-resolver/-/region-config-resolver-3.873.0.tgz", - "integrity": "sha512-q9sPoef+BBG6PJnc4x60vK/bfVwvRWsPgcoQyIra057S/QGjq5VkjvNk6H8xedf6vnKlXNBwq9BaANBXnldUJg==", + "version": "3.914.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/region-config-resolver/-/region-config-resolver-3.914.0.tgz", + "integrity": "sha512-KlmHhRbn1qdwXUdsdrJ7S/MAkkC1jLpQ11n+XvxUUUCGAJd1gjC7AjxPZUM7ieQ2zcb8bfEzIU7al+Q3ZT0u7Q==", "requires": { - "@aws-sdk/types": "3.862.0", - "@smithy/node-config-provider": "^4.1.4", - "@smithy/types": "^4.3.2", - "@smithy/util-config-provider": "^4.0.0", - "@smithy/util-middleware": "^4.0.5", + "@aws-sdk/types": "3.914.0", + "@smithy/config-resolver": "^4.4.0", + "@smithy/types": "^4.8.0", "tslib": "^2.6.2" } }, "@aws-sdk/token-providers": { - "version": "3.883.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/token-providers/-/token-providers-3.883.0.tgz", - "integrity": "sha512-tcj/Z5paGn9esxhmmkEW7gt39uNoIRbXG1UwJrfKu4zcTr89h86PDiIE2nxUO3CMQf1KgncPpr5WouPGzkh/QQ==", + "version": "3.919.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/token-providers/-/token-providers-3.919.0.tgz", + "integrity": "sha512-6aFv4lzXbfbkl0Pv37Us8S/ZkqplOQZIEgQg7bfMru7P96Wv2jVnDGsEc5YyxMnnRyIB90naQ5JgslZ4rkpknw==", "requires": { - "@aws-sdk/core": "3.883.0", - "@aws-sdk/nested-clients": "3.883.0", - "@aws-sdk/types": "3.862.0", - "@smithy/property-provider": "^4.0.5", - "@smithy/shared-ini-file-loader": "^4.0.5", - "@smithy/types": "^4.3.2", + "@aws-sdk/core": "3.916.0", + "@aws-sdk/nested-clients": "3.919.0", + "@aws-sdk/types": "3.914.0", + "@smithy/property-provider": "^4.2.3", + "@smithy/shared-ini-file-loader": "^4.3.3", + "@smithy/types": "^4.8.0", "tslib": "^2.6.2" } }, "@aws-sdk/types": { - "version": "3.862.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.862.0.tgz", - "integrity": "sha512-Bei+RL0cDxxV+lW2UezLbCYYNeJm6Nzee0TpW0FfyTRBhH9C1XQh4+x+IClriXvgBnRquTMMYsmJfvx8iyLKrg==", + "version": "3.914.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.914.0.tgz", + "integrity": "sha512-kQWPsRDmom4yvAfyG6L1lMmlwnTzm1XwMHOU+G5IFlsP4YEaMtXidDzW/wiivY0QFrhfCz/4TVmu0a2aPU57ug==", "requires": { - "@smithy/types": "^4.3.2", + "@smithy/types": "^4.8.0", "tslib": "^2.6.2" } }, "@aws-sdk/util-endpoints": { - "version": "3.879.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/util-endpoints/-/util-endpoints-3.879.0.tgz", - "integrity": "sha512-aVAJwGecYoEmbEFju3127TyJDF9qJsKDUUTRMDuS8tGn+QiWQFnfInmbt+el9GU1gEJupNTXV+E3e74y51fb7A==", + "version": "3.916.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-endpoints/-/util-endpoints-3.916.0.tgz", + "integrity": "sha512-bAgUQwvixdsiGNcuZSDAOWbyHlnPtg8G8TyHD6DTfTmKTHUW6tAn+af/ZYJPXEzXhhpwgJqi58vWnsiDhmr7NQ==", "requires": { - "@aws-sdk/types": "3.862.0", - "@smithy/types": "^4.3.2", - "@smithy/url-parser": "^4.0.5", - "@smithy/util-endpoints": "^3.0.7", + "@aws-sdk/types": "3.914.0", + "@smithy/types": "^4.8.0", + "@smithy/url-parser": "^4.2.3", + "@smithy/util-endpoints": "^3.2.3", "tslib": "^2.6.2" } }, "@aws-sdk/util-user-agent-browser": { - "version": "3.873.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/util-user-agent-browser/-/util-user-agent-browser-3.873.0.tgz", - "integrity": "sha512-AcRdbK6o19yehEcywI43blIBhOCSo6UgyWcuOJX5CFF8k39xm1ILCjQlRRjchLAxWrm0lU0Q7XV90RiMMFMZtA==", + "version": "3.914.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-user-agent-browser/-/util-user-agent-browser-3.914.0.tgz", + "integrity": "sha512-rMQUrM1ECH4kmIwlGl9UB0BtbHy6ZuKdWFrIknu8yGTRI/saAucqNTh5EI1vWBxZ0ElhK5+g7zOnUuhSmVQYUA==", "requires": { - "@aws-sdk/types": "3.862.0", - "@smithy/types": "^4.3.2", + "@aws-sdk/types": "3.914.0", + "@smithy/types": "^4.8.0", "bowser": "^2.11.0", "tslib": "^2.6.2" } }, "@aws-sdk/util-user-agent-node": { - "version": "3.883.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/util-user-agent-node/-/util-user-agent-node-3.883.0.tgz", - "integrity": "sha512-28cQZqC+wsKUHGpTBr+afoIdjS6IoEJkMqcZsmo2Ag8LzmTa6BUWQenFYB0/9BmDy4PZFPUn+uX+rJgWKB+jzA==", + "version": "3.916.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-user-agent-node/-/util-user-agent-node-3.916.0.tgz", + "integrity": "sha512-CwfWV2ch6UdjuSV75ZU99N03seEUb31FIUrXBnwa6oONqj/xqXwrxtlUMLx6WH3OJEE4zI3zt5PjlTdGcVwf4g==", "requires": { - "@aws-sdk/middleware-user-agent": "3.883.0", - "@aws-sdk/types": "3.862.0", - "@smithy/node-config-provider": "^4.1.4", - "@smithy/types": "^4.3.2", + "@aws-sdk/middleware-user-agent": "3.916.0", + "@aws-sdk/types": "3.914.0", + "@smithy/node-config-provider": "^4.3.3", + "@smithy/types": "^4.8.0", "tslib": "^2.6.2" } }, "@aws-sdk/xml-builder": { - "version": "3.873.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/xml-builder/-/xml-builder-3.873.0.tgz", - "integrity": "sha512-kLO7k7cGJ6KaHiExSJWojZurF7SnGMDHXRuQunFnEoD0n1yB6Lqy/S/zHiQ7oJnBhPr9q0TW9qFkrsZb1Uc54w==", + "version": "3.914.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/xml-builder/-/xml-builder-3.914.0.tgz", + "integrity": "sha512-k75evsBD5TcIjedycYS7QXQ98AmOtbnxRJOPtCo0IwYRmy7UvqgS/gBL5SmrIqeV6FDSYRQMgdBxSMp6MLmdew==", "requires": { - "@smithy/types": "^4.3.2", + "@smithy/types": "^4.8.0", + "fast-xml-parser": "5.2.5", "tslib": "^2.6.2" } }, "@smithy/abort-controller": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/@smithy/abort-controller/-/abort-controller-4.1.0.tgz", - "integrity": "sha512-wEhSYznxOmx7EdwK1tYEWJF5+/wmSFsff9BfTOn8oO/+KPl3gsmThrb6MJlWbOC391+Ya31s5JuHiC2RlT80Zg==", + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/@smithy/abort-controller/-/abort-controller-4.2.3.tgz", + "integrity": "sha512-xWL9Mf8b7tIFuAlpjKtRPnHrR8XVrwTj5NPYO/QwZPtc0SDLsPxb56V5tzi5yspSMytISHybifez+4jlrx0vkQ==", "requires": { - "@smithy/types": "^4.4.0", + "@smithy/types": "^4.8.0", "tslib": "^2.6.2" } }, "@smithy/config-resolver": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/@smithy/config-resolver/-/config-resolver-4.2.0.tgz", - "integrity": "sha512-FA10YhPFLy23uxeWu7pOM2ctlw+gzbPMTZQwrZ8FRIfyJ/p8YIVz7AVTB5jjLD+QIerydyKcVMZur8qzzDILAQ==", + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/@smithy/config-resolver/-/config-resolver-4.4.0.tgz", + "integrity": "sha512-Kkmz3Mup2PGp/HNJxhCWkLNdlajJORLSjwkcfrj0E7nu6STAEdcMR1ir5P9/xOmncx8xXfru0fbUYLlZog/cFg==", "requires": { - "@smithy/node-config-provider": "^4.2.0", - "@smithy/types": "^4.4.0", - "@smithy/util-config-provider": "^4.1.0", - "@smithy/util-middleware": "^4.1.0", + "@smithy/node-config-provider": "^4.3.3", + "@smithy/types": "^4.8.0", + "@smithy/util-config-provider": "^4.2.0", + "@smithy/util-endpoints": "^3.2.3", + "@smithy/util-middleware": "^4.2.3", "tslib": "^2.6.2" } }, "@smithy/core": { - "version": "3.10.0", - "resolved": "https://registry.npmjs.org/@smithy/core/-/core-3.10.0.tgz", - "integrity": "sha512-bXyD3Ij6b1qDymEYlEcF+QIjwb9gObwZNaRjETJsUEvSIzxFdynSQ3E4ysY7lUFSBzeWBNaFvX+5A0smbC2q6A==", - "requires": { - "@smithy/middleware-serde": "^4.1.0", - "@smithy/protocol-http": "^5.2.0", - "@smithy/types": "^4.4.0", - "@smithy/util-base64": "^4.1.0", - "@smithy/util-body-length-browser": "^4.1.0", - "@smithy/util-middleware": "^4.1.0", - "@smithy/util-stream": "^4.3.0", - "@smithy/util-utf8": "^4.1.0", - "@types/uuid": "^9.0.1", - "tslib": "^2.6.2", - "uuid": "^9.0.1" + "version": "3.17.1", + "resolved": "https://registry.npmjs.org/@smithy/core/-/core-3.17.1.tgz", + "integrity": "sha512-V4Qc2CIb5McABYfaGiIYLTmo/vwNIK7WXI5aGveBd9UcdhbOMwcvIMxIw/DJj1S9QgOMa/7FBkarMdIC0EOTEQ==", + "requires": { + "@smithy/middleware-serde": "^4.2.3", + "@smithy/protocol-http": "^5.3.3", + "@smithy/types": "^4.8.0", + "@smithy/util-base64": "^4.3.0", + "@smithy/util-body-length-browser": "^4.2.0", + "@smithy/util-middleware": "^4.2.3", + "@smithy/util-stream": "^4.5.4", + "@smithy/util-utf8": "^4.2.0", + "@smithy/uuid": "^1.1.0", + "tslib": "^2.6.2" } }, "@smithy/credential-provider-imds": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/@smithy/credential-provider-imds/-/credential-provider-imds-4.1.0.tgz", - "integrity": "sha512-iVwNhxTsCQTPdp++4C/d9xvaDmuEWhXi55qJobMp9QMaEHRGH3kErU4F8gohtdsawRqnUy/ANylCjKuhcR2mPw==", + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/@smithy/credential-provider-imds/-/credential-provider-imds-4.2.3.tgz", + "integrity": "sha512-hA1MQ/WAHly4SYltJKitEsIDVsNmXcQfYBRv2e+q04fnqtAX5qXaybxy/fhUeAMCnQIdAjaGDb04fMHQefWRhw==", "requires": { - "@smithy/node-config-provider": "^4.2.0", - "@smithy/property-provider": "^4.1.0", - "@smithy/types": "^4.4.0", - "@smithy/url-parser": "^4.1.0", + "@smithy/node-config-provider": "^4.3.3", + "@smithy/property-provider": "^4.2.3", + "@smithy/types": "^4.8.0", + "@smithy/url-parser": "^4.2.3", "tslib": "^2.6.2" } }, "@smithy/fetch-http-handler": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/@smithy/fetch-http-handler/-/fetch-http-handler-5.2.0.tgz", - "integrity": "sha512-VZenjDdVaUGiy3hwQtxm75nhXZrhFG+3xyL93qCQAlYDyhT/jeDWM8/3r5uCFMlTmmyrIjiDyiOynVFchb0BSg==", + "version": "5.3.4", + "resolved": "https://registry.npmjs.org/@smithy/fetch-http-handler/-/fetch-http-handler-5.3.4.tgz", + "integrity": "sha512-bwigPylvivpRLCm+YK9I5wRIYjFESSVwl8JQ1vVx/XhCw0PtCi558NwTnT2DaVCl5pYlImGuQTSwMsZ+pIavRw==", "requires": { - "@smithy/protocol-http": "^5.2.0", - "@smithy/querystring-builder": "^4.1.0", - "@smithy/types": "^4.4.0", - "@smithy/util-base64": "^4.1.0", + "@smithy/protocol-http": "^5.3.3", + "@smithy/querystring-builder": "^4.2.3", + "@smithy/types": "^4.8.0", + "@smithy/util-base64": "^4.3.0", "tslib": "^2.6.2" } }, "@smithy/hash-node": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/@smithy/hash-node/-/hash-node-4.1.0.tgz", - "integrity": "sha512-mXkJQ/6lAXTuoSsEH+d/fHa4ms4qV5LqYoPLYhmhCRTNcMMdg+4Ya8cMgU1W8+OR40eX0kzsExT7fAILqtTl2w==", + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/@smithy/hash-node/-/hash-node-4.2.3.tgz", + "integrity": "sha512-6+NOdZDbfuU6s1ISp3UOk5Rg953RJ2aBLNLLBEcamLjHAg1Po9Ha7QIB5ZWhdRUVuOUrT8BVFR+O2KIPmw027g==", "requires": { - "@smithy/types": "^4.4.0", - "@smithy/util-buffer-from": "^4.1.0", - "@smithy/util-utf8": "^4.1.0", + "@smithy/types": "^4.8.0", + "@smithy/util-buffer-from": "^4.2.0", + "@smithy/util-utf8": "^4.2.0", "tslib": "^2.6.2" } }, "@smithy/invalid-dependency": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/@smithy/invalid-dependency/-/invalid-dependency-4.1.0.tgz", - "integrity": "sha512-4/FcV6aCMzgpM4YyA/GRzTtG28G0RQJcWK722MmpIgzOyfSceWcI9T9c8matpHU9qYYLaWtk8pSGNCLn5kzDRw==", + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/@smithy/invalid-dependency/-/invalid-dependency-4.2.3.tgz", + "integrity": "sha512-Cc9W5DwDuebXEDMpOpl4iERo8I0KFjTnomK2RMdhhR87GwrSmUmwMxS4P5JdRf+LsjOdIqumcerwRgYMr/tZ9Q==", "requires": { - "@smithy/types": "^4.4.0", + "@smithy/types": "^4.8.0", "tslib": "^2.6.2" } }, "@smithy/is-array-buffer": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-4.1.0.tgz", - "integrity": "sha512-ePTYUOV54wMogio+he4pBybe8fwg4sDvEVDBU8ZlHOZXbXK3/C0XfJgUCu6qAZcawv05ZhZzODGUerFBPsPUDQ==", + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-4.2.0.tgz", + "integrity": "sha512-DZZZBvC7sjcYh4MazJSGiWMI2L7E0oCiRHREDzIxi/M2LY79/21iXt6aPLHge82wi5LsuRF5A06Ds3+0mlh6CQ==", "requires": { "tslib": "^2.6.2" } }, "@smithy/middleware-content-length": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/@smithy/middleware-content-length/-/middleware-content-length-4.1.0.tgz", - "integrity": "sha512-x3dgLFubk/ClKVniJu+ELeZGk4mq7Iv0HgCRUlxNUIcerHTLVmq7Q5eGJL0tOnUltY6KFw5YOKaYxwdcMwox/w==", + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/@smithy/middleware-content-length/-/middleware-content-length-4.2.3.tgz", + "integrity": "sha512-/atXLsT88GwKtfp5Jr0Ks1CSa4+lB+IgRnkNrrYP0h1wL4swHNb0YONEvTceNKNdZGJsye+W2HH8W7olbcPUeA==", "requires": { - "@smithy/protocol-http": "^5.2.0", - "@smithy/types": "^4.4.0", + "@smithy/protocol-http": "^5.3.3", + "@smithy/types": "^4.8.0", "tslib": "^2.6.2" } }, "@smithy/middleware-endpoint": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/@smithy/middleware-endpoint/-/middleware-endpoint-4.2.0.tgz", - "integrity": "sha512-J1eCF7pPDwgv7fGwRd2+Y+H9hlIolF3OZ2PjptonzzyOXXGh/1KGJAHpEcY1EX+WLlclKu2yC5k+9jWXdUG4YQ==", + "version": "4.3.5", + "resolved": "https://registry.npmjs.org/@smithy/middleware-endpoint/-/middleware-endpoint-4.3.5.tgz", + "integrity": "sha512-SIzKVTvEudFWJbxAaq7f2GvP3jh2FHDpIFI6/VAf4FOWGFZy0vnYMPSRj8PGYI8Hjt29mvmwSRgKuO3bK4ixDw==", "requires": { - "@smithy/core": "^3.10.0", - "@smithy/middleware-serde": "^4.1.0", - "@smithy/node-config-provider": "^4.2.0", - "@smithy/shared-ini-file-loader": "^4.1.0", - "@smithy/types": "^4.4.0", - "@smithy/url-parser": "^4.1.0", - "@smithy/util-middleware": "^4.1.0", + "@smithy/core": "^3.17.1", + "@smithy/middleware-serde": "^4.2.3", + "@smithy/node-config-provider": "^4.3.3", + "@smithy/shared-ini-file-loader": "^4.3.3", + "@smithy/types": "^4.8.0", + "@smithy/url-parser": "^4.2.3", + "@smithy/util-middleware": "^4.2.3", "tslib": "^2.6.2" } }, "@smithy/middleware-retry": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/@smithy/middleware-retry/-/middleware-retry-4.2.0.tgz", - "integrity": "sha512-raL5oWYf5ALl3jCJrajE8enKJEnV/2wZkKS6mb3ZRY2tg3nj66ssdWy5Ps8E6Yu8Wqh3Tt+Sb9LozjvwZupq+A==", - "requires": { - "@smithy/node-config-provider": "^4.2.0", - "@smithy/protocol-http": "^5.2.0", - "@smithy/service-error-classification": "^4.1.0", - "@smithy/smithy-client": "^4.6.0", - "@smithy/types": "^4.4.0", - "@smithy/util-middleware": "^4.1.0", - "@smithy/util-retry": "^4.1.0", - "@types/uuid": "^9.0.1", - "tslib": "^2.6.2", - "uuid": "^9.0.1" + "version": "4.4.5", + "resolved": "https://registry.npmjs.org/@smithy/middleware-retry/-/middleware-retry-4.4.5.tgz", + "integrity": "sha512-DCaXbQqcZ4tONMvvdz+zccDE21sLcbwWoNqzPLFlZaxt1lDtOE2tlVpRSwcTOJrjJSUThdgEYn7HrX5oLGlK9A==", + "requires": { + "@smithy/node-config-provider": "^4.3.3", + "@smithy/protocol-http": "^5.3.3", + "@smithy/service-error-classification": "^4.2.3", + "@smithy/smithy-client": "^4.9.1", + "@smithy/types": "^4.8.0", + "@smithy/util-middleware": "^4.2.3", + "@smithy/util-retry": "^4.2.3", + "@smithy/uuid": "^1.1.0", + "tslib": "^2.6.2" } }, "@smithy/middleware-serde": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/@smithy/middleware-serde/-/middleware-serde-4.1.0.tgz", - "integrity": "sha512-CtLFYlHt7c2VcztyVRc+25JLV4aGpmaSv9F1sPB0AGFL6S+RPythkqpGDa2XBQLJQooKkjLA1g7Xe4450knShg==", + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/@smithy/middleware-serde/-/middleware-serde-4.2.3.tgz", + "integrity": "sha512-8g4NuUINpYccxiCXM5s1/V+uLtts8NcX4+sPEbvYQDZk4XoJfDpq5y2FQxfmUL89syoldpzNzA0R9nhzdtdKnQ==", "requires": { - "@smithy/protocol-http": "^5.2.0", - "@smithy/types": "^4.4.0", + "@smithy/protocol-http": "^5.3.3", + "@smithy/types": "^4.8.0", "tslib": "^2.6.2" } }, "@smithy/middleware-stack": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/@smithy/middleware-stack/-/middleware-stack-4.1.0.tgz", - "integrity": "sha512-91Fuw4IKp0eK8PNhMXrHRcYA1jvbZ9BJGT91wwPy3bTQT8mHTcQNius/EhSQTlT9QUI3Ki1wjHeNXbWK0tO8YQ==", + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/@smithy/middleware-stack/-/middleware-stack-4.2.3.tgz", + "integrity": "sha512-iGuOJkH71faPNgOj/gWuEGS6xvQashpLwWB1HjHq1lNNiVfbiJLpZVbhddPuDbx9l4Cgl0vPLq5ltRfSaHfspA==", "requires": { - "@smithy/types": "^4.4.0", + "@smithy/types": "^4.8.0", "tslib": "^2.6.2" } }, "@smithy/node-config-provider": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/@smithy/node-config-provider/-/node-config-provider-4.2.0.tgz", - "integrity": "sha512-8/fpilqKurQ+f8nFvoFkJ0lrymoMJ+5/CQV5IcTv/MyKhk2Q/EFYCAgTSWHD4nMi9ux9NyBBynkyE9SLg2uSLA==", + "version": "4.3.3", + "resolved": "https://registry.npmjs.org/@smithy/node-config-provider/-/node-config-provider-4.3.3.tgz", + "integrity": "sha512-NzI1eBpBSViOav8NVy1fqOlSfkLgkUjUTlohUSgAEhHaFWA3XJiLditvavIP7OpvTjDp5u2LhtlBhkBlEisMwA==", "requires": { - "@smithy/property-provider": "^4.1.0", - "@smithy/shared-ini-file-loader": "^4.1.0", - "@smithy/types": "^4.4.0", + "@smithy/property-provider": "^4.2.3", + "@smithy/shared-ini-file-loader": "^4.3.3", + "@smithy/types": "^4.8.0", "tslib": "^2.6.2" } }, "@smithy/node-http-handler": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-4.2.0.tgz", - "integrity": "sha512-G4NV70B4hF9vBrUkkvNfWO6+QR4jYjeO4tc+4XrKCb4nPYj49V9Hu8Ftio7Mb0/0IlFyEOORudHrm+isY29nCA==", + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-4.4.3.tgz", + "integrity": "sha512-MAwltrDB0lZB/H6/2M5PIsISSwdI5yIh6DaBB9r0Flo9nx3y0dzl/qTMJPd7tJvPdsx6Ks/cwVzheGNYzXyNbQ==", "requires": { - "@smithy/abort-controller": "^4.1.0", - "@smithy/protocol-http": "^5.2.0", - "@smithy/querystring-builder": "^4.1.0", - "@smithy/types": "^4.4.0", + "@smithy/abort-controller": "^4.2.3", + "@smithy/protocol-http": "^5.3.3", + "@smithy/querystring-builder": "^4.2.3", + "@smithy/types": "^4.8.0", "tslib": "^2.6.2" } }, "@smithy/property-provider": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/@smithy/property-provider/-/property-provider-4.1.0.tgz", - "integrity": "sha512-eksMjMHUlG5PwOUWO3k+rfLNOPVPJ70mUzyYNKb5lvyIuAwS4zpWGsxGiuT74DFWonW0xRNy+jgzGauUzX7SyA==", + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/@smithy/property-provider/-/property-provider-4.2.3.tgz", + "integrity": "sha512-+1EZ+Y+njiefCohjlhyOcy1UNYjT+1PwGFHCxA/gYctjg3DQWAU19WigOXAco/Ql8hZokNehpzLd0/+3uCreqQ==", "requires": { - "@smithy/types": "^4.4.0", + "@smithy/types": "^4.8.0", "tslib": "^2.6.2" } }, "@smithy/protocol-http": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/@smithy/protocol-http/-/protocol-http-5.2.0.tgz", - "integrity": "sha512-bwjlh5JwdOQnA01be+5UvHK4HQz4iaRKlVG46hHSJuqi0Ribt3K06Z1oQ29i35Np4G9MCDgkOGcHVyLMreMcbg==", + "version": "5.3.3", + "resolved": "https://registry.npmjs.org/@smithy/protocol-http/-/protocol-http-5.3.3.tgz", + "integrity": "sha512-Mn7f/1aN2/jecywDcRDvWWWJF4uwg/A0XjFMJtj72DsgHTByfjRltSqcT9NyE9RTdBSN6X1RSXrhn/YWQl8xlw==", "requires": { - "@smithy/types": "^4.4.0", + "@smithy/types": "^4.8.0", "tslib": "^2.6.2" } }, "@smithy/querystring-builder": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/@smithy/querystring-builder/-/querystring-builder-4.1.0.tgz", - "integrity": "sha512-JqTWmVIq4AF8R8OK/2cCCiQo5ZJ0SRPsDkDgLO5/3z8xxuUp1oMIBBjfuueEe+11hGTZ6rRebzYikpKc6yQV9Q==", + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/@smithy/querystring-builder/-/querystring-builder-4.2.3.tgz", + "integrity": "sha512-LOVCGCmwMahYUM/P0YnU/AlDQFjcu+gWbFJooC417QRB/lDJlWSn8qmPSDp+s4YVAHOgtgbNG4sR+SxF/VOcJQ==", "requires": { - "@smithy/types": "^4.4.0", - "@smithy/util-uri-escape": "^4.1.0", + "@smithy/types": "^4.8.0", + "@smithy/util-uri-escape": "^4.2.0", "tslib": "^2.6.2" } }, "@smithy/querystring-parser": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/@smithy/querystring-parser/-/querystring-parser-4.1.0.tgz", - "integrity": "sha512-VgdHhr8YTRsjOl4hnKFm7xEMOCRTnKw3FJ1nU+dlWNhdt/7eEtxtkdrJdx7PlRTabdANTmvyjE4umUl9cK4awg==", + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/@smithy/querystring-parser/-/querystring-parser-4.2.3.tgz", + "integrity": "sha512-cYlSNHcTAX/wc1rpblli3aUlLMGgKZ/Oqn8hhjFASXMCXjIqeuQBei0cnq2JR8t4RtU9FpG6uyl6PxyArTiwKA==", "requires": { - "@smithy/types": "^4.4.0", + "@smithy/types": "^4.8.0", "tslib": "^2.6.2" } }, "@smithy/service-error-classification": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/@smithy/service-error-classification/-/service-error-classification-4.1.0.tgz", - "integrity": "sha512-UBpNFzBNmS20jJomuYn++Y+soF8rOK9AvIGjS9yGP6uRXF5rP18h4FDUsoNpWTlSsmiJ87e2DpZo9ywzSMH7PQ==", + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/@smithy/service-error-classification/-/service-error-classification-4.2.3.tgz", + "integrity": "sha512-NkxsAxFWwsPsQiwFG2MzJ/T7uIR6AQNh1SzcxSUnmmIqIQMlLRQDKhc17M7IYjiuBXhrQRjQTo3CxX+DobS93g==", "requires": { - "@smithy/types": "^4.4.0" + "@smithy/types": "^4.8.0" } }, "@smithy/shared-ini-file-loader": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/@smithy/shared-ini-file-loader/-/shared-ini-file-loader-4.1.0.tgz", - "integrity": "sha512-W0VMlz9yGdQ/0ZAgWICFjFHTVU0YSfGoCVpKaExRM/FDkTeP/yz8OKvjtGjs6oFokCRm0srgj/g4Cg0xuHu8Rw==", + "version": "4.3.3", + "resolved": "https://registry.npmjs.org/@smithy/shared-ini-file-loader/-/shared-ini-file-loader-4.3.3.tgz", + "integrity": "sha512-9f9Ixej0hFhroOK2TxZfUUDR13WVa8tQzhSzPDgXe5jGL3KmaM9s8XN7RQwqtEypI82q9KHnKS71CJ+q/1xLtQ==", "requires": { - "@smithy/types": "^4.4.0", + "@smithy/types": "^4.8.0", "tslib": "^2.6.2" } }, "@smithy/signature-v4": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/@smithy/signature-v4/-/signature-v4-5.2.0.tgz", - "integrity": "sha512-ObX1ZqG2DdZQlXx9mLD7yAR8AGb7yXurGm+iWx9x4l1fBZ8CZN2BRT09aSbcXVPZXWGdn5VtMuupjxhOTI2EjA==", + "version": "5.3.3", + "resolved": "https://registry.npmjs.org/@smithy/signature-v4/-/signature-v4-5.3.3.tgz", + "integrity": "sha512-CmSlUy+eEYbIEYN5N3vvQTRfqt0lJlQkaQUIf+oizu7BbDut0pozfDjBGecfcfWf7c62Yis4JIEgqQ/TCfodaA==", "requires": { - "@smithy/is-array-buffer": "^4.1.0", - "@smithy/protocol-http": "^5.2.0", - "@smithy/types": "^4.4.0", - "@smithy/util-hex-encoding": "^4.1.0", - "@smithy/util-middleware": "^4.1.0", - "@smithy/util-uri-escape": "^4.1.0", - "@smithy/util-utf8": "^4.1.0", + "@smithy/is-array-buffer": "^4.2.0", + "@smithy/protocol-http": "^5.3.3", + "@smithy/types": "^4.8.0", + "@smithy/util-hex-encoding": "^4.2.0", + "@smithy/util-middleware": "^4.2.3", + "@smithy/util-uri-escape": "^4.2.0", + "@smithy/util-utf8": "^4.2.0", "tslib": "^2.6.2" } }, "@smithy/smithy-client": { - "version": "4.6.0", - "resolved": "https://registry.npmjs.org/@smithy/smithy-client/-/smithy-client-4.6.0.tgz", - "integrity": "sha512-TvlIshqx5PIi0I0AiR+PluCpJ8olVG++xbYkAIGCUkByaMUlfOXLgjQTmYbr46k4wuDe8eHiTIlUflnjK2drPQ==", + "version": "4.9.1", + "resolved": "https://registry.npmjs.org/@smithy/smithy-client/-/smithy-client-4.9.1.tgz", + "integrity": "sha512-Ngb95ryR5A9xqvQFT5mAmYkCwbXvoLavLFwmi7zVg/IowFPCfiqRfkOKnbc/ZRL8ZKJ4f+Tp6kSu6wjDQb8L/g==", "requires": { - "@smithy/core": "^3.10.0", - "@smithy/middleware-endpoint": "^4.2.0", - "@smithy/middleware-stack": "^4.1.0", - "@smithy/protocol-http": "^5.2.0", - "@smithy/types": "^4.4.0", - "@smithy/util-stream": "^4.3.0", + "@smithy/core": "^3.17.1", + "@smithy/middleware-endpoint": "^4.3.5", + "@smithy/middleware-stack": "^4.2.3", + "@smithy/protocol-http": "^5.3.3", + "@smithy/types": "^4.8.0", + "@smithy/util-stream": "^4.5.4", "tslib": "^2.6.2" } }, "@smithy/types": { - "version": "4.4.0", - "resolved": "https://registry.npmjs.org/@smithy/types/-/types-4.4.0.tgz", - "integrity": "sha512-4jY91NgZz+ZnSFcVzWwngOW6VuK3gR/ihTwSU1R/0NENe9Jd8SfWgbhDCAGUWL3bI7DiDSW7XF6Ui6bBBjrqXw==", + "version": "4.8.0", + "resolved": "https://registry.npmjs.org/@smithy/types/-/types-4.8.0.tgz", + "integrity": "sha512-QpELEHLO8SsQVtqP+MkEgCYTFW0pleGozfs3cZ183ZBj9z3VC1CX1/wtFMK64p+5bhtZo41SeLK1rBRtd25nHQ==", "requires": { "tslib": "^2.6.2" } }, "@smithy/url-parser": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/@smithy/url-parser/-/url-parser-4.1.0.tgz", - "integrity": "sha512-/LYEIOuO5B2u++tKr1NxNxhZTrr3A63jW8N73YTwVeUyAlbB/YM+hkftsvtKAcMt3ADYo0FsF1GY3anehffSVQ==", + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/@smithy/url-parser/-/url-parser-4.2.3.tgz", + "integrity": "sha512-I066AigYvY3d9VlU3zG9XzZg1yT10aNqvCaBTw9EPgu5GrsEl1aUkcMvhkIXascYH1A8W0LQo3B1Kr1cJNcQEw==", "requires": { - "@smithy/querystring-parser": "^4.1.0", - "@smithy/types": "^4.4.0", + "@smithy/querystring-parser": "^4.2.3", + "@smithy/types": "^4.8.0", "tslib": "^2.6.2" } }, "@smithy/util-base64": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/@smithy/util-base64/-/util-base64-4.1.0.tgz", - "integrity": "sha512-RUGd4wNb8GeW7xk+AY5ghGnIwM96V0l2uzvs/uVHf+tIuVX2WSvynk5CxNoBCsM2rQRSZElAo9rt3G5mJ/gktQ==", + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/@smithy/util-base64/-/util-base64-4.3.0.tgz", + "integrity": "sha512-GkXZ59JfyxsIwNTWFnjmFEI8kZpRNIBfxKjv09+nkAWPt/4aGaEWMM04m4sxgNVWkbt2MdSvE3KF/PfX4nFedQ==", "requires": { - "@smithy/util-buffer-from": "^4.1.0", - "@smithy/util-utf8": "^4.1.0", + "@smithy/util-buffer-from": "^4.2.0", + "@smithy/util-utf8": "^4.2.0", "tslib": "^2.6.2" } }, "@smithy/util-body-length-browser": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/@smithy/util-body-length-browser/-/util-body-length-browser-4.1.0.tgz", - "integrity": "sha512-V2E2Iez+bo6bUMOTENPr6eEmepdY8Hbs+Uc1vkDKgKNA/brTJqOW/ai3JO1BGj9GbCeLqw90pbbH7HFQyFotGQ==", + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/@smithy/util-body-length-browser/-/util-body-length-browser-4.2.0.tgz", + "integrity": "sha512-Fkoh/I76szMKJnBXWPdFkQJl2r9SjPt3cMzLdOB6eJ4Pnpas8hVoWPYemX/peO0yrrvldgCUVJqOAjUrOLjbxg==", "requires": { "tslib": "^2.6.2" } }, "@smithy/util-body-length-node": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/@smithy/util-body-length-node/-/util-body-length-node-4.1.0.tgz", - "integrity": "sha512-BOI5dYjheZdgR9XiEM3HJcEMCXSoqbzu7CzIgYrx0UtmvtC3tC2iDGpJLsSRFffUpy8ymsg2ARMP5fR8mtuUQQ==", + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/@smithy/util-body-length-node/-/util-body-length-node-4.2.1.tgz", + "integrity": "sha512-h53dz/pISVrVrfxV1iqXlx5pRg3V2YWFcSQyPyXZRrZoZj4R4DeWRDo1a7dd3CPTcFi3kE+98tuNyD2axyZReA==", "requires": { "tslib": "^2.6.2" } }, "@smithy/util-buffer-from": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-4.1.0.tgz", - "integrity": "sha512-N6yXcjfe/E+xKEccWEKzK6M+crMrlwaCepKja0pNnlSkm6SjAeLKKA++er5Ba0I17gvKfN/ThV+ZOx/CntKTVw==", + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-4.2.0.tgz", + "integrity": "sha512-kAY9hTKulTNevM2nlRtxAG2FQ3B2OR6QIrPY3zE5LqJy1oxzmgBGsHLWTcNhWXKchgA0WHW+mZkQrng/pgcCew==", "requires": { - "@smithy/is-array-buffer": "^4.1.0", + "@smithy/is-array-buffer": "^4.2.0", "tslib": "^2.6.2" } }, "@smithy/util-config-provider": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/@smithy/util-config-provider/-/util-config-provider-4.1.0.tgz", - "integrity": "sha512-swXz2vMjrP1ZusZWVTB/ai5gK+J8U0BWvP10v9fpcFvg+Xi/87LHvHfst2IgCs1i0v4qFZfGwCmeD/KNCdJZbQ==", + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/@smithy/util-config-provider/-/util-config-provider-4.2.0.tgz", + "integrity": "sha512-YEjpl6XJ36FTKmD+kRJJWYvrHeUvm5ykaUS5xK+6oXffQPHeEM4/nXlZPe+Wu0lsgRUcNZiliYNh/y7q9c2y6Q==", "requires": { "tslib": "^2.6.2" } }, "@smithy/util-defaults-mode-browser": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/@smithy/util-defaults-mode-browser/-/util-defaults-mode-browser-4.1.0.tgz", - "integrity": "sha512-D27cLtJtC4EEeERJXS+JPoogz2tE5zeE3zhWSSu6ER5/wJ5gihUxIzoarDX6K1U27IFTHit5YfHqU4Y9RSGE0w==", + "version": "4.3.4", + "resolved": "https://registry.npmjs.org/@smithy/util-defaults-mode-browser/-/util-defaults-mode-browser-4.3.4.tgz", + "integrity": "sha512-qI5PJSW52rnutos8Bln8nwQZRpyoSRN6k2ajyoUHNMUzmWqHnOJCnDELJuV6m5PML0VkHI+XcXzdB+6awiqYUw==", "requires": { - "@smithy/property-provider": "^4.1.0", - "@smithy/smithy-client": "^4.6.0", - "@smithy/types": "^4.4.0", - "bowser": "^2.11.0", + "@smithy/property-provider": "^4.2.3", + "@smithy/smithy-client": "^4.9.1", + "@smithy/types": "^4.8.0", "tslib": "^2.6.2" } }, "@smithy/util-defaults-mode-node": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/@smithy/util-defaults-mode-node/-/util-defaults-mode-node-4.1.0.tgz", - "integrity": "sha512-gnZo3u5dP1o87plKupg39alsbeIY1oFFnCyV2nI/++pL19vTtBLgOyftLEjPjuXmoKn2B2rskX8b7wtC/+3Okg==", + "version": "4.2.6", + "resolved": "https://registry.npmjs.org/@smithy/util-defaults-mode-node/-/util-defaults-mode-node-4.2.6.tgz", + "integrity": "sha512-c6M/ceBTm31YdcFpgfgQAJaw3KbaLuRKnAz91iMWFLSrgxRpYm03c3bu5cpYojNMfkV9arCUelelKA7XQT36SQ==", "requires": { - "@smithy/config-resolver": "^4.2.0", - "@smithy/credential-provider-imds": "^4.1.0", - "@smithy/node-config-provider": "^4.2.0", - "@smithy/property-provider": "^4.1.0", - "@smithy/smithy-client": "^4.6.0", - "@smithy/types": "^4.4.0", + "@smithy/config-resolver": "^4.4.0", + "@smithy/credential-provider-imds": "^4.2.3", + "@smithy/node-config-provider": "^4.3.3", + "@smithy/property-provider": "^4.2.3", + "@smithy/smithy-client": "^4.9.1", + "@smithy/types": "^4.8.0", "tslib": "^2.6.2" } }, "@smithy/util-endpoints": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/@smithy/util-endpoints/-/util-endpoints-3.1.0.tgz", - "integrity": "sha512-5LFg48KkunBVGrNs3dnQgLlMXJLVo7k9sdZV5su3rjO3c3DmQ2LwUZI0Zr49p89JWK6sB7KmzyI2fVcDsZkwuw==", + "version": "3.2.3", + "resolved": "https://registry.npmjs.org/@smithy/util-endpoints/-/util-endpoints-3.2.3.tgz", + "integrity": "sha512-aCfxUOVv0CzBIkU10TubdgKSx5uRvzH064kaiPEWfNIvKOtNpu642P4FP1hgOFkjQIkDObrfIDnKMKkeyrejvQ==", "requires": { - "@smithy/node-config-provider": "^4.2.0", - "@smithy/types": "^4.4.0", + "@smithy/node-config-provider": "^4.3.3", + "@smithy/types": "^4.8.0", "tslib": "^2.6.2" } }, "@smithy/util-hex-encoding": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/@smithy/util-hex-encoding/-/util-hex-encoding-4.1.0.tgz", - "integrity": "sha512-1LcueNN5GYC4tr8mo14yVYbh/Ur8jHhWOxniZXii+1+ePiIbsLZ5fEI0QQGtbRRP5mOhmooos+rLmVASGGoq5w==", + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/@smithy/util-hex-encoding/-/util-hex-encoding-4.2.0.tgz", + "integrity": "sha512-CCQBwJIvXMLKxVbO88IukazJD9a4kQ9ZN7/UMGBjBcJYvatpWk+9g870El4cB8/EJxfe+k+y0GmR9CAzkF+Nbw==", "requires": { "tslib": "^2.6.2" } }, "@smithy/util-middleware": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/@smithy/util-middleware/-/util-middleware-4.1.0.tgz", - "integrity": "sha512-612onNcKyxhP7/YOTKFTb2F6sPYtMRddlT5mZvYf1zduzaGzkYhpYIPxIeeEwBZFjnvEqe53Ijl2cYEfJ9d6/Q==", + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/@smithy/util-middleware/-/util-middleware-4.2.3.tgz", + "integrity": "sha512-v5ObKlSe8PWUHCqEiX2fy1gNv6goiw6E5I/PN2aXg3Fb/hse0xeaAnSpXDiWl7x6LamVKq7senB+m5LOYHUAHw==", "requires": { - "@smithy/types": "^4.4.0", + "@smithy/types": "^4.8.0", "tslib": "^2.6.2" } }, "@smithy/util-retry": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/@smithy/util-retry/-/util-retry-4.1.0.tgz", - "integrity": "sha512-5AGoBHb207xAKSVwaUnaER+L55WFY8o2RhlafELZR3mB0J91fpL+Qn+zgRkPzns3kccGaF2vy0HmNVBMWmN6dA==", + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/@smithy/util-retry/-/util-retry-4.2.3.tgz", + "integrity": "sha512-lLPWnakjC0q9z+OtiXk+9RPQiYPNAovt2IXD3CP4LkOnd9NpUsxOjMx1SnoUVB7Orb7fZp67cQMtTBKMFDvOGg==", "requires": { - "@smithy/service-error-classification": "^4.1.0", - "@smithy/types": "^4.4.0", + "@smithy/service-error-classification": "^4.2.3", + "@smithy/types": "^4.8.0", "tslib": "^2.6.2" } }, "@smithy/util-stream": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/@smithy/util-stream/-/util-stream-4.3.0.tgz", - "integrity": "sha512-ZOYS94jksDwvsCJtppHprUhsIscRnCKGr6FXCo3SxgQ31ECbza3wqDBqSy6IsAak+h/oAXb1+UYEBmDdseAjUQ==", + "version": "4.5.4", + "resolved": "https://registry.npmjs.org/@smithy/util-stream/-/util-stream-4.5.4.tgz", + "integrity": "sha512-+qDxSkiErejw1BAIXUFBSfM5xh3arbz1MmxlbMCKanDDZtVEQ7PSKW9FQS0Vud1eI/kYn0oCTVKyNzRlq+9MUw==", "requires": { - "@smithy/fetch-http-handler": "^5.2.0", - "@smithy/node-http-handler": "^4.2.0", - "@smithy/types": "^4.4.0", - "@smithy/util-base64": "^4.1.0", - "@smithy/util-buffer-from": "^4.1.0", - "@smithy/util-hex-encoding": "^4.1.0", - "@smithy/util-utf8": "^4.1.0", + "@smithy/fetch-http-handler": "^5.3.4", + "@smithy/node-http-handler": "^4.4.3", + "@smithy/types": "^4.8.0", + "@smithy/util-base64": "^4.3.0", + "@smithy/util-buffer-from": "^4.2.0", + "@smithy/util-hex-encoding": "^4.2.0", + "@smithy/util-utf8": "^4.2.0", "tslib": "^2.6.2" } }, "@smithy/util-uri-escape": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/@smithy/util-uri-escape/-/util-uri-escape-4.1.0.tgz", - "integrity": "sha512-b0EFQkq35K5NHUYxU72JuoheM6+pytEVUGlTwiFxWFpmddA+Bpz3LgsPRIpBk8lnPE47yT7AF2Egc3jVnKLuPg==", + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/@smithy/util-uri-escape/-/util-uri-escape-4.2.0.tgz", + "integrity": "sha512-igZpCKV9+E/Mzrpq6YacdTQ0qTiLm85gD6N/IrmyDvQFA4UnU3d5g3m8tMT/6zG/vVkWSU+VxeUyGonL62DuxA==", "requires": { "tslib": "^2.6.2" } }, "@smithy/util-utf8": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-4.1.0.tgz", - "integrity": "sha512-mEu1/UIXAdNYuBcyEPbjScKi/+MQVXNIuY/7Cm5XLIWe319kDrT5SizBE95jqtmEXoDbGoZxKLCMttdZdqTZKQ==", + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-4.2.0.tgz", + "integrity": "sha512-zBPfuzoI8xyBtR2P6WQj63Rz8i3AmfAaJLuNG8dWsfvPe8lO4aCPYLn879mEgHndZH1zQ2oXmG8O1GGzzaoZiw==", "requires": { - "@smithy/util-buffer-from": "^4.1.0", + "@smithy/util-buffer-from": "^4.2.0", "tslib": "^2.6.2" } }, @@ -31962,6 +31983,11 @@ } } }, + "@aws/lambda-invoke-store": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/@aws/lambda-invoke-store/-/lambda-invoke-store-0.1.1.tgz", + "integrity": "sha512-RcLam17LdlbSOSp9VxmUu1eI6Mwxp+OwhD2QhiSNmNCzoDb0EeUXTD2n/WbcnrAYMGlmf05th6QYq23VqvJqpA==" + }, "@babel/code-frame": { "version": "7.26.2", "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.26.2.tgz", @@ -35971,6 +35997,14 @@ } } }, + "@smithy/uuid": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@smithy/uuid/-/uuid-1.1.0.tgz", + "integrity": "sha512-4aUIteuyxtBUhVdiQqcDhKFitwfd9hqoSDYY2KRXiWtgoWJ9Bmise+KfEPDiVHWeJepvF8xJO9/9+WDIciMFFw==", + "requires": { + "tslib": "^2.6.2" + } + }, "@tus/file-store": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/@tus/file-store/-/file-store-2.0.0.tgz", @@ -37014,6 +37048,12 @@ "resolved": "https://registry.npmjs.org/@types/js-yaml/-/js-yaml-4.0.5.tgz", "integrity": "sha512-FhpRzf927MNQdRZP0J5DLIdTXhjLYzeUTmLAu69mnVksLH9CJY3IuSeEgbKUki7GQZm0WqDkGzyxju2EZGD2wA==" }, + "@types/json-bigint": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@types/json-bigint/-/json-bigint-1.0.4.tgz", + "integrity": "sha512-ydHooXLbOmxBbubnA7Eh+RpBzuaIiQjh8WGJYQB50JFGFrdxW7JzVlyEV7fAXw0T2sqJ1ysTneJbiyNLqZRAag==", + "dev": true + }, "@types/json-schema": { "version": "7.0.9", "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.9.tgz", diff --git a/package.json b/package.json index d61461628..979a048ba 100644 --- a/package.json +++ b/package.json @@ -30,7 +30,7 @@ "dependencies": { "@aws-sdk/client-ecs": "^3.795.0", "@aws-sdk/client-s3": "3.654.0", - "@aws-sdk/client-s3vectors": "^3.883.0", + "@aws-sdk/client-s3vectors": "^3.919.0", "@aws-sdk/lib-storage": "3.654.0", "@aws-sdk/s3-request-presigner": "3.654.0", "@fastify/accepts": "^4.3.0", @@ -72,6 +72,7 @@ "ioredis": "^5.2.4", "ip-address": "^10.0.1", "jose": "^6.0.10", + "json-bigint": "^1.0.0", "knex": "^3.1.0", "lru-cache": "^10.2.0", "md5-file": "^5.0.0", @@ -98,6 +99,7 @@ "@types/glob": "^8.1.0", "@types/jest": "^29.2.1", "@types/js-yaml": "^4.0.5", + "@types/json-bigint": "^1.0.4", "@types/multistream": "^4.1.3", "@types/mustache": "^4.2.2", "@types/node": "^22.18.8", diff --git a/src/config.ts b/src/config.ts index 1c817e80b..13ad291c5 100644 --- a/src/config.ts +++ b/src/config.ts @@ -189,7 +189,7 @@ type StorageConfigType = { icebergS3DeleteEnabled: boolean vectorEnabled: boolean - vectorBucketS3?: string + vectorS3Buckets: string[] vectorBucketRegion?: string vectorMaxBucketsCount: number vectorMaxIndexesCount: number @@ -534,7 +534,7 @@ export function getConfig(options?: { reload?: boolean }): StorageConfigType { icebergS3DeleteEnabled: getOptionalConfigFromEnv('ICEBERG_S3_DELETE_ENABLED') === 'true', vectorEnabled: getOptionalConfigFromEnv('VECTOR_ENABLED') === 'true', - vectorBucketS3: getOptionalConfigFromEnv('VECTOR_BUCKET_S3') || undefined, + vectorS3Buckets: getOptionalConfigFromEnv('VECTOR_S3_BUCKETS')?.trim()?.split(',') || [], vectorBucketRegion: getOptionalConfigFromEnv('VECTOR_BUCKET_REGION') || undefined, vectorMaxBucketsCount: parseInt(getOptionalConfigFromEnv('VECTOR_MAX_BUCKETS') || '10', 10), vectorMaxIndexesCount: parseInt(getOptionalConfigFromEnv('VECTOR_MAX_INDEXES') || '20', 10), diff --git a/src/http/plugins/vector.ts b/src/http/plugins/vector.ts index a46fa09f3..5c0df7cf3 100644 --- a/src/http/plugins/vector.ts +++ b/src/http/plugins/vector.ts @@ -1,6 +1,6 @@ import fastifyPlugin from 'fastify-plugin' import { FastifyInstance } from 'fastify' -import { getTenantConfig } from '@internal/database' +import { getTenantConfig, multitenantKnex } from '@internal/database' import { createS3VectorClient, KnexVectorMetadataDB, @@ -9,6 +9,7 @@ import { } from '@storage/protocols/vector' import { getConfig } from '../../config' import { ERRORS } from '@internal/errors' +import { KnexShardStoreFactory, ShardCatalog, SingleShard } from '@internal/sharding' declare module 'fastify' { interface FastifyRequest { @@ -21,10 +22,10 @@ const s3VectorAdapter = new S3Vector(s3VectorClient) export const s3vector = fastifyPlugin(async function (fastify: FastifyInstance) { fastify.addHook('preHandler', async (req) => { - const { isMultitenant, vectorBucketS3, vectorMaxBucketsCount, vectorMaxIndexesCount } = + const { isMultitenant, vectorS3Buckets, vectorMaxBucketsCount, vectorMaxIndexesCount } = getConfig() - if (!vectorBucketS3) { + if (!vectorS3Buckets || vectorS3Buckets.length === 0) { throw ERRORS.FeatureNotEnabled('vector', 'Vector service not configured') } @@ -39,9 +40,15 @@ export const s3vector = fastifyPlugin(async function (fastify: FastifyInstance) const db = req.db.pool.acquire() const store = new KnexVectorMetadataDB(db) - req.s3Vector = new VectorStoreManager(s3VectorAdapter, store, { + const shard = isMultitenant + ? new ShardCatalog(new KnexShardStoreFactory(multitenantKnex)) + : new SingleShard({ + shardKey: vectorS3Buckets[0], + capacity: 10000, + }) + + req.s3Vector = new VectorStoreManager(s3VectorAdapter, store, shard, { tenantId: req.tenantId, - vectorBucketName: vectorBucketS3, maxBucketCount: maxBucketCount, maxIndexCount: maxIndexCount, }) diff --git a/src/http/routes/iceberg/bucket.ts b/src/http/routes/iceberg/bucket.ts index 6c9bdfb93..a984f1674 100644 --- a/src/http/routes/iceberg/bucket.ts +++ b/src/http/routes/iceberg/bucket.ts @@ -31,13 +31,6 @@ const listBucketsQuerySchema = { }, } as const -const successResponseSchema = { - type: 'object', - properties: { - message: { type: 'string', examples: ['Successfully deleted'] }, - }, -} - interface deleteBucketRequestInterface extends AuthenticatedRequest { Params: FromSchema } @@ -54,11 +47,11 @@ export default async function routes(fastify: FastifyInstance) { fastify.delete( '/bucket/:bucketId', { - schema: createDefaultSchema(successResponseSchema, { + schema: { params: deleteBucketParamsSchema, summary: 'Delete an analytics bucket', tags: ['bucket'], - }), + }, config: { operation: { type: ROUTE_OPERATIONS.DELETE_BUCKET }, }, diff --git a/src/http/routes/iceberg/table.ts b/src/http/routes/iceberg/table.ts index bb77cd7b0..b6373aaae 100644 --- a/src/http/routes/iceberg/table.ts +++ b/src/http/routes/iceberg/table.ts @@ -4,6 +4,7 @@ import { FromSchema } from 'json-schema-to-ts' import { ERRORS } from '@internal/errors' import { CreateTableRequest } from '@storage/protocols/iceberg/catalog/rest-catalog-client' import { ROUTE_OPERATIONS } from '../operations' +import JSONBigint from 'json-bigint' const createTableSchema = { body: { @@ -234,18 +235,24 @@ const commitTransactionSchema = { items: { type: 'object', description: 'A requirement assertion', + required: ['type'], properties: { - name: { + type: { type: 'string', - description: 'Name of the requirement (e.g. assert-ref-snapshot-id)', - examples: ['assert-ref-snapshot-id'], + description: + 'Type of the requirement (e.g. assert-ref-snapshot-id, assert-table-uuid)', + examples: ['assert-ref-snapshot-id', 'assert-table-uuid'], }, + // allow arbitrary additional args specific to the requirement + ref: { type: 'string' }, + // 'snapshot-id': { type: 'number', format: 'int64', bigint: true }, + uuid: { type: 'string' }, args: { type: 'object', - description: 'Arguments for the requirement', additionalProperties: true, }, }, + additionalProperties: true, }, }, updates: { @@ -254,18 +261,54 @@ const commitTransactionSchema = { items: { type: 'object', description: 'A single update operation', + required: ['action'], properties: { - name: { + action: { type: 'string', - description: 'Name of the update operation (e.g. add-column)', - examples: ['add-column'], + description: 'Action to perform (e.g. add-snapshot, set-snapshot-ref)', + examples: ['add-snapshot', 'set-snapshot-ref'], }, + snapshot: { + type: 'object', + properties: { + // 'snapshot-id': { type: 'string', format: 'int64', bigint: true }, + // 'parent-snapshot-id': { + // type: 'integer', + // format: 'int64', + // bigint: true, + // nullable: true, + // }, + 'sequence-number': { type: 'integer' }, + 'timestamp-ms': { type: 'integer' }, + 'manifest-list': { type: 'string' }, + summary: { + type: 'object', + additionalProperties: true, + properties: { + operation: { type: 'string' }, + 'added-files-size': { type: 'string' }, + 'added-data-files': { type: 'string' }, + 'added-records': { type: 'string' }, + 'total-delete-files': { type: 'string' }, + 'total-records': { type: 'string' }, + 'total-position-deletes': { type: 'string' }, + 'total-equality-deletes': { type: 'string' }, + }, + }, + 'schema-id': { type: 'integer' }, + }, + additionalProperties: true, + }, + // Fields for set-snapshot-ref or similar actions + 'ref-name': { type: 'string' }, + type: { type: 'string' }, + // 'snapshot-id': { type: 'integer', format: 'int64', bigint: true }, args: { type: 'object', - description: 'Arguments for the update operation', additionalProperties: true, }, }, + additionalProperties: true, }, }, }, @@ -298,11 +341,30 @@ interface commitTableRequest extends AuthenticatedRequest { Body: FromSchema<(typeof commitTransactionSchema)['body']> } +const BigIntSerializer = JSONBigint({ + strict: true, + useNativeBigInt: true, +}) + export default async function routes(fastify: FastifyInstance) { + // Make sure big ints responses are serialized correctly as integers and not strings + fastify.setSerializerCompiler(() => { + return BigIntSerializer.stringify + }) + fastify.post( '/:prefix/namespaces/:namespace/tables', { - schema: { ...createTableSchema, tags: ['iceberg'] }, + schema: { + ...createTableSchema, + response: { + 200: { + type: 'object', + additionalProperties: true, + }, + }, + tags: ['iceberg'], + }, }, async (request, response) => { if (!request.icebergCatalog) { @@ -325,7 +387,16 @@ export default async function routes(fastify: FastifyInstance) { config: { operation: { type: ROUTE_OPERATIONS.ICEBERG_LIST_TABLES }, }, - schema: { ...listTableSchema, tags: ['iceberg'] }, + schema: { + ...listTableSchema, + response: { + 200: { + type: 'object', + additionalProperties: true, + }, + }, + tags: ['iceberg'], + }, }, async (request, response) => { if (!request.icebergCatalog) { @@ -349,7 +420,16 @@ export default async function routes(fastify: FastifyInstance) { config: { operation: { type: ROUTE_OPERATIONS.ICEBERG_LOAD_TABLE }, }, - schema: { ...loadTableSchema, tags: ['iceberg'] }, + schema: { + ...loadTableSchema, + response: { + 200: { + type: 'object', + additionalProperties: true, + }, + }, + tags: ['iceberg'], + }, exposeHeadRoute: false, }, async (request, response) => { @@ -373,7 +453,16 @@ export default async function routes(fastify: FastifyInstance) { config: { operation: { type: ROUTE_OPERATIONS.ICEBERG_TABLE_EXISTS }, }, - schema: { ...loadTableSchema, tags: ['iceberg'] }, + schema: { + ...loadTableSchema, + response: { + 200: { + type: 'object', + additionalProperties: true, + }, + }, + tags: ['iceberg'], + }, }, async (request, response) => { if (!request.icebergCatalog) { @@ -424,27 +513,64 @@ export default async function routes(fastify: FastifyInstance) { ) }) - fastify.post( - '/:prefix/namespaces/:namespace/tables/:table', - { - config: { - operation: { type: ROUTE_OPERATIONS.ICEBERG_COMMIT_TABLE }, - }, - schema: { ...commitTransactionSchema, tags: ['iceberg'] }, - }, - async (request, response) => { - if (!request.icebergCatalog) { - throw ERRORS.FeatureNotEnabled('icebergCatalog', 'iceberg_catalog') + fastify.register(async (fastify) => { + fastify.addContentTypeParser('application/json', {}, (_request, payload: unknown, done) => { + try { + if (typeof payload === 'string') return done(null, JSONBigint.parse(payload)) + if (Buffer.isBuffer(payload)) return done(null, JSONBigint.parse(payload.toString('utf8'))) + if (payload && typeof (payload as any).on === 'function') { + const chunks: Buffer[] = [] + ;(payload as NodeJS.ReadableStream).on('data', (c) => + chunks.push(Buffer.isBuffer(c) ? c : Buffer.from(String(c))) + ) + ;(payload as NodeJS.ReadableStream).on('end', () => { + try { + done(null, JSONBigint.parse(Buffer.concat(chunks).toString('utf8'))) + } catch (err) { + done(err as Error) + } + }) + ;(payload as NodeJS.ReadableStream).on('error', (err) => done(err as Error)) + return + } + done(null, payload) + } catch (err) { + done(err as Error) } + }) - const result = await request.icebergCatalog.updateTable({ - ...request.body, - namespace: request.params.namespace, - table: request.params.table, - warehouse: request.params.prefix, - }) + fastify.post( + '/:prefix/namespaces/:namespace/tables/:table', + { + config: { + operation: { type: ROUTE_OPERATIONS.ICEBERG_COMMIT_TABLE }, + }, + schema: { + ...commitTransactionSchema, + response: { + 200: { + type: 'object', + additionalProperties: true, + }, + }, + tags: ['iceberg'], + }, + }, + async (request, response) => { + if (!request.icebergCatalog) { + throw ERRORS.FeatureNotEnabled('icebergCatalog', 'iceberg_catalog') + } - return response.send(result) - } - ) + const result = await request.icebergCatalog.updateTable({ + namespace: request.params.namespace, + table: request.params.table, + warehouse: request.params.prefix, + requirements: request.body.requirements, + updates: request.body.updates, + }) + + return response.send(result) + } + ) + }) } diff --git a/src/http/routes/vector/index.ts b/src/http/routes/vector/index.ts index a93300d51..fd19d5731 100644 --- a/src/http/routes/vector/index.ts +++ b/src/http/routes/vector/index.ts @@ -72,6 +72,7 @@ export default async function routes(fastify: FastifyInstance) { fastify.register(deleteVectors) fastify.register(listVectors) fastify.register(getVectors) + setErrorHandler(fastify, { respectStatusCode: true, }) diff --git a/src/http/routes/vector/put-vectors.ts b/src/http/routes/vector/put-vectors.ts index 786a69b25..78ae31a6f 100644 --- a/src/http/routes/vector/put-vectors.ts +++ b/src/http/routes/vector/put-vectors.ts @@ -33,7 +33,9 @@ const putVector = { }, metadata: { type: 'object', - additionalProperties: { type: 'string' }, + additionalProperties: { + oneOf: [{ type: 'string' }, { type: 'boolean' }, { type: 'number' }], + }, }, key: { type: 'string' }, }, diff --git a/src/internal/errors/codes.ts b/src/internal/errors/codes.ts index 79dd05c3a..7f947dd92 100644 --- a/src/internal/errors/codes.ts +++ b/src/internal/errors/codes.ts @@ -50,6 +50,7 @@ export enum ErrorCode { S3VectorBucketNotEmpty = 'VectorBucketNotEmpty', S3VectorMaxBucketsExceeded = 'S3VectorMaxBucketsExceeded', S3VectorMaxIndexesExceeded = 'S3VectorMaxIndexesExceeded', + S3VectorNoAvailableShard = 'S3VectorNoAvailableShard', } export const ERRORS = { @@ -471,6 +472,13 @@ export const ERRORS = { message: `Maximum number of indexes exceeded. Max allowed is ${maxIndexes}. Contact support to increase your limit.`, }) }, + S3VectorNoAvailableShard() { + return new StorageBackendError({ + code: ErrorCode.S3VectorNoAvailableShard, + httpStatusCode: 500, + message: `No available shards are available to host the vector index. Please try again later.`, + }) + }, } export function isStorageError(errorType: ErrorCode, error: any): error is StorageBackendError { diff --git a/src/internal/hashing/index.ts b/src/internal/hashing/index.ts new file mode 100644 index 000000000..df225df91 --- /dev/null +++ b/src/internal/hashing/index.ts @@ -0,0 +1 @@ +export * from './string-to-int' diff --git a/src/internal/hashing/string-to-int.ts b/src/internal/hashing/string-to-int.ts new file mode 100644 index 000000000..0105e3b56 --- /dev/null +++ b/src/internal/hashing/string-to-int.ts @@ -0,0 +1,9 @@ +export function hashStringToInt(str: string): number { + let hash = 5381 + let i = -1 + while (i < str.length - 1) { + i += 1 + hash = (hash * 33) ^ str.charCodeAt(i) + } + return hash >>> 0 +} diff --git a/src/internal/monitoring/logger.ts b/src/internal/monitoring/logger.ts index dcd320b64..f85312d39 100644 --- a/src/internal/monitoring/logger.ts +++ b/src/internal/monitoring/logger.ts @@ -1,4 +1,4 @@ -import pino, { BaseLogger } from 'pino' +import pino, { BaseLogger, Logger } from 'pino' import { getConfig } from '../../config' import { FastifyReply, FastifyRequest } from 'fastify' import { URL } from 'node:url' @@ -48,7 +48,11 @@ export const baseLogger = pino({ timestamp: pino.stdTimeFunctions.isoTime, }) -export const logger = baseLogger.child({ region }) +export let logger = baseLogger.child({ region }) + +export function setLogger(newLogger: Logger) { + logger = newLogger +} export interface RequestLog { type: 'request' diff --git a/src/internal/queue/event.ts b/src/internal/queue/event.ts index 39f2b79b6..ec7d33e2f 100644 --- a/src/internal/queue/event.ts +++ b/src/internal/queue/event.ts @@ -134,6 +134,18 @@ export class Event> { return that.invoke() } + static invokeOrSend>( + this: StaticThis, + payload: Omit, + options?: SendOptions + ) { + if (!payload.$version) { + ;(payload as T['payload']).$version = this.version + } + const that = new this(payload) + return that.invokeOrSend(options) + } + static handle( job: Job['payload']> | Job['payload']>[], opts?: { signal?: AbortSignal } @@ -179,6 +191,27 @@ export class Event> { ) } + async invokeOrSend(sendOptions?: SendOptions): Promise { + const constructor = this.constructor as typeof Event + + if (!constructor.allowSync) { + throw ERRORS.InternalError(undefined, 'Cannot send this event synchronously') + } + + try { + await this.invoke() + } catch (e) { + logSchema.error(logger, '[Queue] Error invoking event synchronously, sending to queue', { + type: 'queue', + project: this.payload.tenant?.ref, + error: e, + metadata: JSON.stringify(this.payload), + }) + + return this.send(sendOptions) + } + } + async invoke(): Promise { const constructor = this.constructor as typeof Event @@ -198,7 +231,7 @@ export class Event> { }) } - async send(): Promise { + async send(customSendOptions?: SendOptions): Promise { const constructor = this.constructor as typeof Event const shouldSend = await constructor.shouldSend(this.payload) @@ -245,7 +278,10 @@ export class Event> { ...this.payload, $version: constructor.version, }, - options: sendOptions, + options: { + ...sendOptions, + ...customSendOptions, + }, }) QueueJobScheduled.inc({ diff --git a/src/internal/sharding/architecture.md b/src/internal/sharding/architecture.md new file mode 100644 index 000000000..c8ece87ec --- /dev/null +++ b/src/internal/sharding/architecture.md @@ -0,0 +1,471 @@ +# Sharding Implementation + +This document explains how the sharding system works, how it allocates slots, manages reservations, and handles slot reuse. +This document was generated with Calude Code. A human did architecture & implementation. + +## Overview + +The sharding system provides a way to distribute resources (like vector indexes or Iceberg tables) across multiple physical shards while maintaining a logical namespace for tenants. It ensures that resources are evenly distributed and that capacity is managed efficiently. + +## Core Concepts + +### 1. Shards + +A **shard** is a physical storage location (e.g., an S3 bucket) that can hold multiple resources. Each shard has: + +- A `kind` (e.g., "vector" or "iceberg") +- A `shard_key` (the physical identifier, like "vector-shard-01") +- A `capacity` (maximum number of slots) +- A `next_slot` counter (tracks the next slot number to mint) +- A `status` ("active", "draining", or "disabled") + +### 2. Slots + +A **slot** is a numbered position within a shard. Slots are **sparse** - they only exist as rows in `shard_slots` when they've been allocated at least once. + +Each slot can be in one of three states: + +1. **Free**: `resource_id IS NULL` AND no active pending reservation exists +2. **Leased**: `resource_id IS NULL` AND an active pending reservation exists (temporarily held during creation) +3. **Confirmed**: `resource_id IS NOT NULL` (contains an active resource) + +### 3. Reservations + +A **reservation** is a time-bound claim on a slot. It's used to coordinate the two-phase process of creating a resource: + +1. **Reserve**: Claim a slot and insert metadata in the database +2. **Confirm**: Create the actual resource (e.g., S3 Vector index) and mark the reservation as confirmed + +Reservations have a `lease_expires_at` timestamp to prevent orphaned slots if a process crashes mid-creation. + +## The Reservation Flow + +### Phase 1: Reserve a Slot + +When a tenant wants to create a new resource (e.g., a vector index), the system: + +1. **Acquire advisory lock** on the logical key to prevent concurrent reservations +2. **Check for existing reservation** by `(kind, logical_key)` + - If pending or confirmed → return the existing reservation + - If cancelled or expired → delete it and continue +3. **Select a shard** using fullest first strategy +4. **Reserve a slot** on that shard (see "Slot Allocation" below) +5. **Create reservation record** in `shard_reservation` table +6. **Return reservation details** to the caller + +### Phase 2: Confirm the Reservation + +After the physical resource is created: + +1. **Update `shard_slots`**: Set `resource_id` +2. **Update `shard_reservation`**: Set status to `'confirmed'` + +This is done atomically to ensure consistency. + +### Phase 3: Cleanup on Failure + +If resource creation fails: + +1. **Cancel the reservation**: Set status to `'cancelled'` + +The slot becomes **free** again and can be reused immediately (since there's no active pending reservation). + +## Slot Allocation Algorithm + +When `reserveOneSlotOnShard(shardId, tenantId)` is called: + +### Step 1: Try to Claim a Free Existing Slot + +```sql +WITH pick AS ( + SELECT slot_no + FROM shard_slots ss + WHERE ss.shard_id = ? + AND ss.resource_id IS NULL -- Not occupied by a resource + AND NOT EXISTS ( + SELECT 1 FROM shard_reservation sr + WHERE sr.shard_id = ss.shard_id + AND sr.slot_no = ss.slot_no + AND sr.status = 'pending' + AND sr.lease_expires_at > now() + ) + ORDER BY slot_no + LIMIT 1 + FOR UPDATE SKIP LOCKED -- Lock-free concurrent access +) +UPDATE shard_slots s + SET tenant_id = ? +FROM pick +WHERE s.shard_id = ? AND s.slot_no = pick.slot_no +RETURNING s.slot_no; +``` + +**Why this works:** + +- Free slots have `resource_id` as `NULL` and no active pending reservation +- These slots were previously used but are now available for reuse +- `FOR UPDATE SKIP LOCKED` allows concurrent processes to skip locked rows and find other free slots + +### Step 2: Mint a Fresh Slot (if no free slots found) + +```sql +WITH ok AS ( + SELECT id, capacity, next_slot + FROM shard + WHERE id = ? AND status = 'active' +), +bumped AS ( + UPDATE shard + SET next_slot = ok.next_slot + 1 + FROM ok + WHERE shard.id = ok.id + AND ok.next_slot < ok.capacity -- Enforce capacity limit + RETURNING ok.next_slot AS slot_no +) +SELECT slot_no FROM bumped; +``` + +Then insert a new row in `shard_slots`: + +```sql +INSERT INTO shard_slots (shard_id, slot_no, tenant_id) +VALUES (?, ?, ?); +``` + +**Why we need `next_slot`:** + +- It ensures we never create duplicate slot numbers +- It tracks how many slots have ever been created (even if some are now free) +- It enforces the shard's capacity limit + +## How Reservations Prevent Double Allocation + +The `shard_reservation` table serves as the **source of truth** for active leases. Here's how the system prevents double allocation: + +### The Problem: Resource Creation is Not Atomic + +Creating a resource involves multiple steps: + +1. Reserve a slot in the database +2. Create the actual resource (e.g., call S3 Vectors API) +3. Update status to confirmed + +If a process crashes between steps 1 and 2, we'd have a slot that's "allocated" in the database but has no actual resource. + +### The Solution: Time-Bound Reservation Records + +1. **During allocation**: Create a reservation record with status `'pending'` and a `lease_expires_at` timestamp +2. **On success**: Set `resource_id` in the slot, mark reservation as `'confirmed'` +3. **On failure**: Mark reservation as `'cancelled'` (slot becomes free again) +4. **On timeout**: A background job marks expired reservations as `'expired'` automatically + +### The State Machine + +``` +FREE LEASED CONFIRMED +(no resource_id, (no resource_id, (resource_id set, + no active pending) pending reservation exists) confirmed reservation) + │ │ │ + ├────reserve slot───────────►│ │ + │ (create pending resv) │ │ + │ ├──confirm───────────────────►│ + │ │ (set resource_id, │ + │ │ mark resv confirmed) │ + │ │ │ + │◄───cancel/expire───────────┤ │ + │ (mark resv cancelled) │ │ + │ │ │ + │◄─────────────────delete resource────────────────────────┤ + │ (clear resource_id) │ +``` + +**How the system prevents double allocation:** + +- A slot is only considered "free" if there's no active pending reservation for it +- The `NOT EXISTS` query checks for pending reservations with unexpired leases +- Multiple processes cannot create duplicate pending reservations due to unique constraint on `(kind, logical_key)` +- The reservation table provides a full audit trail of all attempts + +### Example: Concurrent Allocations + +``` +Process A Process B +──────────────────────────────────────────────────── +1. Reserve slot 5 + INSERT INTO shard_reservation + (status='pending', slot_no=5) + +2. Start creating resource... + 3. Try to reserve slot 5 + WHERE NOT EXISTS ( + pending reservation for slot 5 + ) + → SKIP (pending exists!) + +3a. If success: + SET resource_id = 'res-A' + UPDATE reservation status='confirmed' + +3b. If failure: + UPDATE reservation status='cancelled' + 4. Try again, now succeeds: + WHERE NOT EXISTS ( + active pending for slot 5 + ) + → OK (cancelled, not pending!) + INSERT new reservation for slot 5 +``` + +Without checking for active pending reservations, both processes could claim the same slot simultaneously. + +### Database Design + +The relationship is: + +- `shard_reservation` has a foreign key to `shard_slots` via `(shard_id, slot_no)` +- This means **reservations point to slots** (not vice versa) +- A slot can have multiple reservation records over time (audit trail) +- Only one pending reservation per slot at a time is allowed by the NOT EXISTS check + +## Handling Old Reservations During Slot Reuse + +### The Challenge + +When a slot is freed (after a resource is deleted), we have: + +- A row in `shard_slots` with `resource_id = NULL` (slot is free) +- A row in `shard_reservation` with status `'confirmed'` (from the previous allocation) + +The slot is considered free because the NOT EXISTS check only looks for **pending** reservations with valid leases. However, when we try to create a new reservation for the same slot, we'd hit a unique constraint violation on `(shard_id, slot_no)` because the old confirmed reservation still exists. + +### The Solution: `deleteStaleReservationsForSlot()` + +Before inserting a new reservation, we call: + +```typescript +await store.deleteStaleReservationsForSlot(shardId, slotNo) +``` + +This deletes any old reservation rows (cancelled, expired, or confirmed) for the same `(shard_id, slot_no)` pair, allowing the slot to be reused with a new reservation. + +**Why we need this:** + +- Old reservations (cancelled, expired, or confirmed from freed slots) would prevent creating new reservations for the same slot +- Without cleanup, we'd get a unique violation on the `(shard_id, slot_no)` constraint +- Deleting old reservations before inserting enables slot reuse while preventing conflicts +- Only pending reservations are preserved (active leases in progress) + +## Tenant ID Tracking + +Both `shard_slots` and `shard_reservation` tables include a `tenant_id` column for: + +- **Usage accounting**: Count slots per tenant +- **Quota enforcement**: Limit resources per tenant +- **Billing**: Track resource consumption + +The `tenant_id` is: + +- Set in both `shard_slots` and `shard_reservation` when reserving a slot +- Preserved in `shard_slots` when confirming +- Cleared from `shard_slots` when freeing a slot for reuse + +### Available Strategies + +#### 1. Fill-First Strategy (Default) + +Prioritizes the shard with the **least free capacity** (most full first). This minimizes the number of active shards by filling them sequentially. + +**Implementation**: Uses a single efficient SQL query: + +```sql +SELECT s.*, + GREATEST( + (s.capacity - s.next_slot) + + COALESCE(( + SELECT COUNT(*) + FROM shard_slots sl + WHERE sl.shard_id = s.id + AND sl.resource_id IS NULL + AND NOT EXISTS ( + SELECT 1 FROM shard_reservation sr + WHERE sr.shard_id = sl.shard_id + AND sr.slot_no = sl.slot_no + AND sr.status = 'pending' + AND sr.lease_expires_at > now() + ) + ), 0), + 0 + ) AS free_capacity +FROM shard s +WHERE s.kind = ? AND s.status = 'active' +HAVING free_capacity > 0 +ORDER BY free_capacity ASC, s.shard_key ASC +LIMIT 1; +``` + +**Good for:** + +- Reducing operational costs (fewer active shards to manage) +- Low contention environments +- When deterministic placement isn't required +- Consolidating resources into fewer physical locations + +**Trade-offs:** + +- No consistency across retries (a resource could map to different shards if capacity changes) +- All new allocations go to the same shard until it fills up (potential write hotspot) + +## Capacity Management + +Each shard has a `capacity` limit. When a shard reaches capacity: + +1. `reserveOneSlotOnShard()` returns `null` +2. The shard selector may return a different shard (depending on strategy) +3. If no shards have available capacity, allocation fails with `NoActiveShardError` + +**Capacity calculation:** + +``` +available = (capacity - next_slot) + count(free_slots) +``` + +Where: + +- `capacity - next_slot` = slots that were never minted +- `count(free_slots)` = previously used slots that are now free + +## Lease Expiration + +A background job periodically calls `expireLeases()` to: + +1. Find reservations where `status = 'pending' AND lease_expires_at < now()` +2. Mark reservations as expired: `UPDATE shard_reservation SET status = 'expired'` + +Once marked as expired, slots automatically become free (since the NOT EXISTS check only looks for pending reservations with valid leases). This ensures that crashed processes don't permanently leak slots. + +## Consistency Guarantees + +### Advisory Locks + +The system uses PostgreSQL advisory locks (`pg_advisory_xact_lock`) to serialize operations on the same logical resource, preventing race conditions. + +**Scope**: Per logical resource (tenant + bucket + resource name) +**Protection**: Prevents duplicate reservations for the same logical resource + +### Transactions + +All multi-step operations are wrapped in database transactions with serializable isolation level to ensure consistency. + +### Atomic Confirmations + +The `confirmReservation()` method uses a CTE (Common Table Expression) to atomically: + +- Check the reservation is still valid +- Update the slot +- Update the reservation status + +If any step fails, the entire operation rolls back. + +### Handling Concurrent Shard Selection + +**The Problem**: Without locking, two processes might: + +1. Read the same `next_slot` value (uncommitted data) +2. Both try to allocate the same slot number +3. Violate fill-first ordering by reading stale capacity + +**The Solution**: `FOR UPDATE` on Shard Selection + +The shard selection query uses `FOR UPDATE` to lock shard rows: + +```sql +WITH candidates AS ( + SELECT s.*, + FROM shard s + WHERE s.kind = ? AND s.status = 'active' + FOR UPDATE -- ← Serialize selection on same shard +) +SELECT * FROM candidates +WHERE free_capacity > 0 +ORDER BY free_capacity ASC +LIMIT 1; +``` + +**How it works**: + +``` +Process A Process B +──────────────────────────────────────────────────── +SELECT ... FOR UPDATE +Locks shard-1 row +Returns shard-1 + SELECT ... FOR UPDATE + ⏸ Waits for shard-1 lock +reserveSlot(1) → SUCCESS +Updates next_slot: 0 → 1 +Commit → releases lock + ✅ Lock acquired! + Reads next_slot = 1 (committed) + Returns shard-1 + reserveSlot(1) → SUCCESS (slot 1) +``` + +**Why `FOR UPDATE` (not `SKIP LOCKED`)**: + +- ✅ **Serializes selection**: Processes wait for correct capacity +- ✅ **Maintains fill-first**: All processes try fullest shard first +- ✅ **Prevents stale reads**: Always see committed `next_slot` value +- ✅ **No race on next_slot**: Can't allocate duplicate slot numbers + +**Why NOT `SKIP LOCKED`**: + +- ❌ Would violate fill-first: Process B skips to shard-2 while shard-1 has space +- ❌ Would spread load prematurely: Defeats purpose of fill-first strategy + +**Lock Scope**: + +- Locks only shard rows (not slots) +- Lock duration: ~1ms (during SELECT only) +- Released on transaction commit +- Affects only processes selecting the **same** kind (e.g., all "vector" allocations) + +**Additional Protection**: `FOR UPDATE SKIP LOCKED` on Slots + +The `reserveOneSlotOnShard` query uses `FOR UPDATE SKIP LOCKED` on **slot rows**: + +```sql +SELECT ss.slot_no +FROM shard_slots ss +WHERE ss.shard_id = ? AND ss.resource_id IS NULL +FOR UPDATE SKIP LOCKED -- ← Skip locked slots +``` + +This allows **concurrent reservations on different slots** within the same shard. + +## Error Handling + +### UniqueViolationError + +If `insertReservation()` fails with a unique constraint violation: + +1. Check if another process created a valid reservation concurrently +2. If yes, return that reservation (idempotent) +3. If no, clear the lease and fail + +### Serialization Errors + +The vector metadata DB has retry logic for serialization errors: + +- Max 3 retries with exponential backoff +- Only retries on PostgreSQL error code `40001` (serialization failure) + +## Summary + +The sharding system provides: + +- **Efficient allocation**: Reuses free slots before minting new ones +- **Consistency**: Advisory locks and transactions prevent race conditions +- **Fault tolerance**: Lease expiration prevents permanent slot leaks +- **Scalability**: Sparse slot storage and lock-free concurrent access +- **Multi-tenancy**: Tenant tracking for accounting and quota enforcement +- **Audit trail**: Full history of all reservation attempts in `shard_reservation` table diff --git a/src/internal/sharding/errors.ts b/src/internal/sharding/errors.ts new file mode 100644 index 000000000..3606065b8 --- /dev/null +++ b/src/internal/sharding/errors.ts @@ -0,0 +1,34 @@ +export class NoActiveShardError extends Error { + constructor(kind: string) { + super(`No active shards for kind=${kind}`) + this.name = 'NoActiveShardError' + } +} + +export class NoCapacityError extends Error { + constructor() { + super('No capacity left on any active shard') + this.name = 'NoCapacityError' + } +} + +export class ReservationNotFoundError extends Error { + constructor() { + super('Reservation not found') + this.name = 'ReservationNotFoundError' + } +} + +export class InvalidReservationStatusError extends Error { + constructor(status: string) { + super(`Reservation status is ${status}`) + this.name = 'InvalidReservationStatusError' + } +} + +export class ExpiredReservationError extends Error { + constructor() { + super('Reservation lease expired or slot no longer held') + this.name = 'ExpiredReservationError' + } +} diff --git a/src/internal/sharding/index.ts b/src/internal/sharding/index.ts new file mode 100644 index 000000000..85ab02384 --- /dev/null +++ b/src/internal/sharding/index.ts @@ -0,0 +1,5 @@ +export * from './store' +export * from './sharder' +export * from './strategy/catalog' +export * from './strategy/single-shard' +export * from './knex' diff --git a/src/internal/sharding/knex.ts b/src/internal/sharding/knex.ts new file mode 100644 index 000000000..7717f5089 --- /dev/null +++ b/src/internal/sharding/knex.ts @@ -0,0 +1,396 @@ +import { Knex } from 'knex' +import { + ReservationRow, + ResourceKind, + ShardRow, + ShardStatus, + ShardStore, + ShardStoreFactory, + UniqueViolationError, +} from './store' +import { hashStringToInt } from '@internal/hashing' + +export class KnexShardStoreFactory implements ShardStoreFactory { + constructor(private knex: Knex) {} + async withTransaction(fn: (store: ShardStore) => Promise): Promise { + try { + return await this.knex.transaction(async (trx) => { + return fn(new KnexShardStore(trx)) + }) + } catch (error) { + throw error + } + } + autocommit(): ShardStore { + return new KnexShardStore(this.knex) + } +} + +class KnexShardStore implements ShardStore { + constructor(private db: Knex | Knex.Transaction) {} + + private q(sql: string, params?: any[]) { + return this.db.raw(sql, params as any) + } + + async advisoryLockByString(key: string): Promise { + const id = hashStringToInt(key) + await this.q(`SELECT pg_advisory_xact_lock(?::bigint)`, [id]) + } + + async findShardByResourceId(tenantId: string, resourceId: string): Promise { + const result = await this.db + .select('s.shard_key', 's.id') + .from('shard_slots as ss') + .join('shard as s', 's.id', 'ss.shard_id') + .where('ss.resource_id', resourceId) + .where('ss.tenant_id', tenantId) + .first() + + return result ?? null + } + + async getOrInsertShard( + kind: ResourceKind, + shardKey: string, + capacity: number, + status: ShardStatus + ): Promise { + const inserted = await this.db('shard') + .insert({ kind, shard_key: shardKey, capacity, status, next_slot: 0 }) + .onConflict(['kind', 'shard_key']) + .ignore() + .returning('*') + if (inserted[0]) return inserted[0] + const row = await this.db('shard').where({ kind, shard_key: shardKey }).first() + if (!row) throw new Error('Failed to fetch shard after idempotent insert') + return row + } + + async setShardStatus(shardId: string | number, status: ShardStatus): Promise { + await this.db('shard').update({ status }).where({ id: shardId }) + } + + async listActiveShards(kind: ResourceKind): Promise { + return this.db('shard').select('*').where({ kind, status: 'active' }) + } + + async findShardWithLeastFreeCapacity(kind: ResourceKind): Promise { + const result = await this.q<{ rows: ShardRow[] }>( + ` + WITH candidates AS ( + SELECT s.*, + GREATEST( + (s.capacity - s.next_slot) + + COALESCE(( + SELECT COUNT(*) + FROM shard_slots sl + WHERE sl.shard_id = s.id + AND sl.resource_id IS NULL + AND NOT EXISTS ( + SELECT 1 FROM shard_reservation sr + WHERE sr.shard_id = sl.shard_id + AND sr.slot_no = sl.slot_no + AND sr.status = 'pending' + AND sr.lease_expires_at > now() + ) + ), 0), + 0 + ) AS free_capacity + FROM shard s + WHERE s.kind = ? AND s.status = 'active' + FOR UPDATE + ) + SELECT * + FROM candidates + WHERE free_capacity > 0 + ORDER BY free_capacity ASC, shard_key ASC + LIMIT 1; + `, + [kind] + ) + + return result.rows[0] ?? null + } + + async findReservationByKindKey( + kind: ResourceKind, + resourceId: string + ): Promise { + return ( + (await this.db('shard_reservation') + .select('shard_reservation.*', 'shard.shard_key as shard_key') + // @ts-expect-error join column added dynamically + .where({ 'shard_reservation.kind': kind, resource_id: resourceId }) + .join('shard', 'shard.id', 'shard_reservation.shard_id') + .first()) ?? null + ) + } + + async fetchReservationById(id: string): Promise { + return ( + (await this.db('shard_reservation').select('*').where({ id }).first()) ?? null + ) + } + + /** + * Reserve one slot on a shard (single-table, no freelist): + * 1) Try to claim a previously used-but-now-free row. + * 2) If none, mint a fresh slot number by bumping shard.next_slot (bounded by capacity) and insert row. + * + * A slot is "free" if: + * - resource_id IS NULL (not confirmed) + * - AND no active pending reservation exists for it + */ + async reserveOneSlotOnShard(shardId: string | number, tenantId: string): Promise { + // 1) Try to claim a free existing row + const claimed = await this.q<{ rows: { slot_no: number }[] }>( + ` + WITH pick AS ( + SELECT ss.slot_no + FROM shard_slots ss + WHERE ss.shard_id = ? + AND ss.resource_id IS NULL + AND NOT EXISTS ( + SELECT 1 FROM shard_reservation sr + WHERE sr.shard_id = ss.shard_id + AND sr.slot_no = ss.slot_no + AND sr.status = 'pending' + AND sr.lease_expires_at > now() + ) + ORDER BY ss.slot_no + LIMIT 1 + FOR UPDATE SKIP LOCKED + ) + UPDATE shard_slots s + SET tenant_id = ? + FROM pick + WHERE s.shard_id = ? AND s.slot_no = pick.slot_no + RETURNING s.slot_no; + `, + [shardId, tenantId, shardId] + ) + if (claimed.rows.length) return claimed.rows[0].slot_no + + // 2) Mint a fresh slot_no by bumping shard.next_slot (bounded by capacity) + const minted = await this.q<{ rows: { slot_no: number }[] }>( + ` + WITH ok AS ( + SELECT id, capacity, next_slot + FROM shard + WHERE id = ? AND status = 'active' + ), + bumped AS ( + UPDATE shard + SET next_slot = ok.next_slot + 1 + FROM ok + WHERE shard.id = ok.id + AND ok.next_slot < ok.capacity + RETURNING ok.next_slot AS slot_no + ) + SELECT slot_no FROM bumped; + `, + [shardId] + ) + const slotNo = minted.rows[0]?.slot_no ?? null + if (slotNo == null) return null // at capacity or shard not active + + // Create the slot row + try { + await this.db('shard_slots').insert({ + shard_id: shardId, + slot_no: slotNo, + tenant_id: tenantId, + }) + return slotNo + } catch (e: any) { + if (e?.code === '23505') { + // Extremely rare race if another tx inserted the same slot first. Let caller try another shard/attempt. + return null + } + throw e + } + } + + async insertReservation(data: { + id: string + kind: ResourceKind + resourceId: string + tenantId: string + shardId: string | number + shardKey: string + slotNo: number + leaseMs: number + }): Promise<{ lease_expires_at: string }> { + try { + const row = await this.db('shard_reservation') + .insert({ + id: data.id, + kind: data.kind, + resource_id: data.resourceId, + tenant_id: data.tenantId, + shard_id: data.shardId, + slot_no: data.slotNo, + status: 'pending', + lease_expires_at: (this.db as any).raw(`now() + interval '${data.leaseMs} milliseconds'`), + }) + .returning(['lease_expires_at']) + + return row[0] + } catch (e: any) { + if (e?.code === '23505') throw new UniqueViolationError() + throw e + } + } + + /** Confirm atomically: pending+lease valid → mark slot resource_id + shard_reservation confirmed */ + async confirmReservation( + reservationId: string, + resourceId: string, + tenantId: string + ): Promise { + const res = await this.q( + ` + WITH ok AS ( + SELECT r.shard_id, r.slot_no + FROM shard_reservation r + WHERE r.id = ? + AND r.status = 'pending' + AND r.tenant_id = ? + AND r.lease_expires_at > now() + ), + upd_slots AS ( + UPDATE shard_slots s + SET resource_id = ? + FROM ok + WHERE s.shard_id = ok.shard_id + AND s.slot_no = ok.slot_no + RETURNING 1 + ) + UPDATE shard_reservation r + SET status = 'confirmed' + WHERE r.id = ? + AND EXISTS (SELECT 1 FROM upd_slots) + RETURNING 1; + `, + [reservationId, tenantId, resourceId, reservationId] + ) + return (res as any).rowCount ?? (res as any).rows.length + } + + async updateReservationStatus( + id: string, + status: 'confirmed' | 'cancelled' | 'expired' + ): Promise { + await this.db('shard_reservation') + .update({ status }) + .where({ id }) + .andWhere('status', '<>', status) + } + + async deleteReservation(id: string): Promise { + await this.db('shard_reservation').where({ id }).del() + } + + async deleteStaleReservationsForSlot(shardId: string | number, slotNo: number): Promise { + // Delete any old reservations for this slot (cancelled, expired, or confirmed) + // This allows the slot to be reused with a new reservation + await this.db('shard_reservation') + .where({ shard_id: shardId, slot_no: slotNo }) + .whereIn('status', ['cancelled', 'expired', 'confirmed']) + .del() + } + + async loadExpiredPendingReservations(): Promise { + return this.db('shard_reservation') + .select('*') + .where({ status: 'pending' }) + .andWhere('lease_expires_at', '<', this.db.fn.now()) + } + + async markReservationsExpired(ids: string[]): Promise { + if (!ids.length) return + await this.db('shard_reservation').update({ status: 'expired' }).whereIn('id', ids) + } + + async freeByLocation(shardId: string | number, slotNo: number): Promise { + // On delete of a confirmed resource, mark its row as reusable (clear resource_id and tenant_id) + await this.q( + ` + WITH shard_slots as ( + UPDATE shard_slots + SET resource_id = null, tenant_id = null + WHERE shard_id = ? AND slot_no = ? + RETURNING shard_id, slot_no + ), + deleted_reservations as ( + DELETE FROM shard_reservation + WHERE shard_id = ? AND slot_no = ? + ) + SELECT 1; + `, + [shardId, slotNo, shardId, slotNo] + ) + } + + async freeByResource(shardId: string | number, resourceId: string, tenantId: string) { + await this.q( + ` + WITH shard_slots AS ( + UPDATE shard_slots + SET resource_id = null, tenant_id = null + WHERE shard_id = ? AND resource_id = ? AND tenant_id = ? + RETURNING shard_id, slot_no + ), + deleted_reservations AS ( + DELETE FROM shard_reservation + WHERE shard_id = ? + AND resource_id = ? + AND tenant_id = ? + ) + SELECT 1; + `, + [shardId, resourceId, tenantId, shardId, resourceId, tenantId] + ) + } + + async shardStats(kind?: ResourceKind) { + const res = await this.q( + ` + SELECT s.id AS shard_id, s.shard_key, s.capacity, s.next_slot, + -- confirmed allocations + (SELECT COUNT(*) FROM shard_slots sl WHERE sl.shard_id = s.id AND sl.resource_id IS NOT NULL) AS used, + -- remaining capacity = (unused unminted capacity) + (existing free rows) + GREATEST( + (s.capacity - s.next_slot) + + COALESCE(( + SELECT COUNT(*) + FROM shard_slots sl + WHERE sl.shard_id = s.id + AND sl.resource_id IS NULL + AND NOT EXISTS ( + SELECT 1 FROM shard_reservation sr + WHERE sr.shard_id = sl.shard_id + AND sr.slot_no = sl.slot_no + AND sr.status = 'pending' + AND sr.lease_expires_at > now() + ) + ), 0), + 0 + ) AS free + FROM shard s + ${kind ? `WHERE s.kind = ?` : ``} + ORDER BY s.kind, s.shard_key; + `, + kind ? [kind] : [] + ) + + return (res as any).rows.map((r: any) => ({ + shardId: String(r.shard_id), + shardKey: r.shard_key, + capacity: Number(r.capacity), + used: Number(r.used), + free: Number(r.free), + })) + } +} diff --git a/src/internal/sharding/sharder.ts b/src/internal/sharding/sharder.ts new file mode 100644 index 000000000..e808e0e95 --- /dev/null +++ b/src/internal/sharding/sharder.ts @@ -0,0 +1,41 @@ +import { ResourceKind, ShardRow, ShardStatus } from './store' + +export interface ShardResource { + kind: ResourceKind + tenantId: string + bucketName: string + logicalName: string +} + +export interface Sharder { + createShard(opts: { + kind: ResourceKind + shardKey: string + capacity?: number + status?: ShardStatus + }): Promise + + setShardStatus(shardId: string | number, status: ShardStatus): Promise + + reserve( + opts: ShardResource & { + kind: ResourceKind + tenantId: string + bucketName: string + logicalName: string + } + ): Promise<{ + reservationId: string + shardId: string + shardKey: string + slotNo: number + leaseExpiresAt: string + }> + confirm(reservationId: string, resource: ShardResource): Promise + cancel(reservationId: string): Promise + expireLeases(): Promise + freeByLocation(shardId: string | number, slotNo: number): Promise + freeByResource(shardId: string | number, resource: ShardResource): Promise + shardStats(kind?: ResourceKind): Promise + findShardByResourceId(param: ShardResource): Promise +} diff --git a/src/internal/sharding/store.ts b/src/internal/sharding/store.ts new file mode 100644 index 000000000..317e9b115 --- /dev/null +++ b/src/internal/sharding/store.ts @@ -0,0 +1,104 @@ +export type ResourceKind = 'vector' | 'iceberg' +export type ShardStatus = 'active' | 'draining' | 'disabled' +export type ReservationStatus = 'pending' | 'confirmed' | 'expired' | 'cancelled' + +export type ShardRow = { + id: string + kind: ResourceKind + shard_key: string + capacity: number + next_slot: number + status: ShardStatus + created_at: string +} + +export type ReservationRow = { + id: string + kind: ResourceKind + resource_id: string + shard_id: string + shard_key: string + slot_no: number + status: ReservationStatus + tenant_id: string + lease_expires_at: string + created_at: string +} + +/** Factory that opens a transaction and passes a store bound to that tx */ +export interface ShardStoreFactory { + withTransaction(fn: (store: ShardStore) => Promise): Promise + /** Optional: an autocommit store for read-only helpers */ + autocommit(): ShardStore +} + +export class UniqueViolationError extends Error { + constructor(message = 'unique_violation') { + super(message) + this.name = 'UniqueViolationError' + } +} + +export interface ShardStoreFactory { + withTransaction(fn: (store: ShardStore) => Promise): Promise + autocommit(): ShardStore // optional, for reads/one-off writes +} + +/** Every method uses the bound tx/connection internally */ +export interface ShardStore { + // Locks + advisoryLockByString(key: string): Promise + + // Shards + getOrInsertShard( + kind: ResourceKind, + shardKey: string, + capacity: number, + status: ShardStatus + ): Promise + setShardStatus(shardId: string | number, status: ShardStatus): Promise + listActiveShards(kind: ResourceKind): Promise + findShardWithLeastFreeCapacity(kind: ResourceKind): Promise + + // Reservations + findReservationByKindKey(kind: ResourceKind, resourceId: string): Promise + fetchReservationById(id: string): Promise + + // Sparse allocation (single-table, no freelist) + // No longer needs reservationId parameter since slots don't track it + reserveOneSlotOnShard(shardId: string | number, tenantId: string): Promise + + insertReservation(data: { + id: string + kind: ResourceKind + resourceId: string + tenantId: string + shardId: string | number + shardKey: string + slotNo: number + leaseMs: number + }): Promise<{ lease_expires_at: string }> + + /** Atomic confirm (checks: pending + lease valid) */ + confirmReservation(reservationId: string, resourceId: string, tenantId: string): Promise + + updateReservationStatus(id: string, status: 'confirmed' | 'cancelled' | 'expired'): Promise + deleteReservation(id: string): Promise + deleteStaleReservationsForSlot(shardId: string | number, slotNo: number): Promise + + // Expiry + loadExpiredPendingReservations(): Promise + markReservationsExpired(ids: string[]): Promise + + // Free-by-location after delete + freeByLocation(shardId: string | number, slotNo: number): Promise + freeByResource(shardId: string | number, resourceId: string, tenantId: string): Promise + findShardByResourceId(tenantId: string, resourceId: string): Promise + + // Stats + shardStats( + kind?: ResourceKind + ): Promise< + Array<{ shardId: string; shardKey: string; capacity: number; used: number; free: number }> + > +} diff --git a/src/internal/sharding/strategy/catalog.ts b/src/internal/sharding/strategy/catalog.ts new file mode 100644 index 000000000..8bad01b25 --- /dev/null +++ b/src/internal/sharding/strategy/catalog.ts @@ -0,0 +1,335 @@ +import { randomUUID } from 'crypto' +import { + ExpiredReservationError, + InvalidReservationStatusError, + NoActiveShardError, + ReservationNotFoundError, +} from '@internal/sharding/errors' + +import { + ResourceKind, + ShardRow, + ShardStatus, + ShardStoreFactory, + UniqueViolationError, +} from '../store' +import { Sharder, ShardResource } from '../sharder' + +/** + * Represents the configuration options for a shard in a distributed system or database. + * + * @interface ShardOptions + * + * @property {ResourceKind} kind - The type of resource the shard is managing or related to. + * @property {string} shardKey - A unique identifier used to determine data placement within the shard. + * @property {number} [capacity] - Optional. The storage or operational capacity allocated to the shard. + * @property {ShardStatus} [status] - Optional. The current operational status of the shard. + */ +interface ShardOptions { + kind: ResourceKind + shardKey: string + capacity?: number + status?: ShardStatus +} + +/** + * Represents a catalog that manages shards and provides functionality for allocation, reservation, and management. + * This class uses transactions for consistent state management and interacts with a shard storage system via a factory. + */ +export class ShardCatalog implements Sharder { + constructor(private factory: ShardStoreFactory) {} + + /** + /** + * Creates a new shard or retrieves an existing one based on the provided options. + * + * @param opts - The options to configure the shard. Includes properties like kind, shardKey, capacity, and status. + * @return A promise that resolves to the created or retrieved shard row. + */ + async createShard(opts: ShardOptions): Promise { + const capacity = opts.capacity ?? 10_000 + + return this.factory.withTransaction(async (store) => { + return await store.getOrInsertShard( + opts.kind, + opts.shardKey, + capacity, + opts.status ?? 'active' + ) + }) + } + + /** + /** + * Creates multiple shards based on the provided shard options. + * + * @param opts - An array of shard options to configure each shard. + * @return A promise that resolves to an array of created shards. + */ + createShards(opts: ShardOptions[]) { + return Promise.all(opts.map((o) => this.createShard(o))) + } + + /** + /** + * Updates the status of a shard. + * + * @param shardId - The unique identifier of the shard to update. + * @param status - The new status to set for the shard. + * @return A promise that resolves when the status is updated. + */ + async setShardStatus(shardId: string | number, status: ShardStatus) { + return this.factory.withTransaction((store) => store.setShardStatus(shardId, status)) + } + + /** + * Reserves a slot on a shard for a specific resource. + * If a reservation already exists for the same resource, returns the existing reservation. + * Uses advisory locking to prevent race conditions. + * + * @param opts - The reservation options. + * @param opts.kind - The type of resource being reserved. + * @param opts.tenantId - The ID of the tenant making the reservation. + * @param opts.bucketName - The name of the bucket containing the resource. + * @param opts.logicalName - The logical name of the resource. + * @param opts.leaseMs - Optional. The lease duration in milliseconds (default: 60000ms). + * @return A promise that resolves to an object containing reservation details. + * @return return.reservationId - The unique identifier for the reservation. + * @return return.shardId - The ID of the shard where the slot was reserved. + * @return return.shardKey - The key of the shard where the slot was reserved. + * @return return.slotNo - The slot number that was reserved. + * @return return.leaseExpiresAt - The ISO timestamp when the lease expires. + * @throws NoActiveShardError if no active shard is available for the resource kind. + */ + async reserve(opts: { + kind: ResourceKind + tenantId: string + bucketName: string + logicalName: string + leaseMs?: number + }): Promise<{ + reservationId: string + shardId: string + shardKey: string + slotNo: number + leaseExpiresAt: string + }> { + const leaseMs = opts.leaseMs ?? 60_000 + const resourceId = `${opts.kind}::${opts.bucketName}::${opts.logicalName}` + + return this.factory.withTransaction(async (store) => { + await store.advisoryLockByString(resourceId) + + const existing = await store.findReservationByKindKey(opts.kind, resourceId) + + if (existing) { + if (existing.status === 'pending' || existing.status === 'confirmed') { + return { + shardKey: existing.shard_key, + reservationId: existing.id, + shardId: String(existing.shard_id), + slotNo: Number(existing.slot_no), + leaseExpiresAt: existing.lease_expires_at, + } + } + + // If cancelled or expired, delete it so we can create a new reservation + if (existing.status === 'cancelled' || existing.status === 'expired') { + await store.deleteReservation(existing.id) + } + } + + const reservationId = randomUUID() + + // Select shard using FOR UPDATE to serialize selection and ensure + // we read the committed next_slot value + const shard = await store.findShardWithLeastFreeCapacity(opts.kind) + if (!shard) { + throw new NoActiveShardError(opts.kind) + } + + // Reserve a slot on the selected shard + // FOR UPDATE ensures no two processes reserve on the same shard simultaneously + const slotNo = await store.reserveOneSlotOnShard(shard.id, opts.tenantId) + if (slotNo == null) { + // This should be very rare since FOR UPDATE serializes selection + // Only happens if shard fills up between selection and reservation + throw new NoActiveShardError(opts.kind) + } + + await store.deleteStaleReservationsForSlot(shard.id, slotNo) + + try { + const { lease_expires_at } = await store.insertReservation({ + id: reservationId, + kind: opts.kind, + resourceId: resourceId, + tenantId: opts.tenantId, + shardId: shard.id, + shardKey: shard.shard_key, + slotNo, + leaseMs, + }) + + return { + reservationId, + shardId: String(shard.id), + shardKey: shard.shard_key, + slotNo, + leaseExpiresAt: lease_expires_at, + } + } catch (e) { + if (e instanceof UniqueViolationError) { + const row = await store.findReservationByKindKey(opts.kind, resourceId) + if (row && (row.status === 'pending' || row.status === 'confirmed')) { + return { + reservationId: row.id, + shardId: String(row.shard_id), + shardKey: row.shard_key, + slotNo: Number(row.slot_no), + leaseExpiresAt: row.lease_expires_at, + } + } + } + throw e + } + }) + } + + /** + * Confirms a pending reservation and associates it with a resource. + * If the reservation has expired, frees the slot and throws an error. + * + * @param reservationId - The unique identifier of the reservation to confirm. + * @param resource - The resource details to associate with the reservation. + * @param resource.kind - The type of resource. + * @param resource.tenantId - The ID of the tenant owning the resource. + * @param resource.bucketName - The name of the bucket containing the resource. + * @param resource.logicalName - The logical name of the resource. + * @return A promise that resolves when the reservation is confirmed. + * @throws ReservationNotFoundError if the reservation does not exist. + * @throws InvalidReservationStatusError if the reservation is not in a pending state. + * @throws ExpiredReservationError if the reservation lease has expired. + */ + async confirm( + reservationId: string, + resource: { + kind: ResourceKind + tenantId: string + bucketName: string + logicalName: string + } + ): Promise { + await this.factory.withTransaction(async (store) => { + const resv = await store.fetchReservationById(reservationId) + if (!resv) throw new ReservationNotFoundError() + if (resv.status === 'confirmed') return + + const resourceId = `${resource.kind}::${resource.bucketName}::${resource.logicalName}` + + const ok = await store.confirmReservation(reservationId, resourceId, resource.tenantId) + + if (!ok) { + const fresh = await store.fetchReservationById(reservationId) + + if (!fresh) { + throw new ReservationNotFoundError() + } + + if (fresh.status !== 'pending') { + throw new InvalidReservationStatusError(fresh.status) + } + + await this.freeByLocation(fresh.shard_id, fresh.slot_no) + + throw new ExpiredReservationError() + } + }) + } + + /** + * Cancels a pending reservation. + * If the reservation does not exist, the operation completes silently. + * + * @param reservationId - The unique identifier of the reservation to cancel. + * @return A promise that resolves when the reservation is cancelled. + */ + async cancel(reservationId: string): Promise { + await this.factory.withTransaction(async (store) => { + const resv = await store.fetchReservationById(reservationId) + if (!resv) return + await store.updateReservationStatus(reservationId, 'cancelled') + }) + } + + /** + * Expires all pending reservations whose lease has expired. + * + * @return A promise that resolves to the number of reservations that were expired. + */ + async expireLeases(): Promise { + return this.factory.withTransaction(async (store) => { + const expired = await store.loadExpiredPendingReservations() + if (!expired.length) return 0 + await store.markReservationsExpired(expired.map((r) => r.id)) + return expired.length + }) + } + + /** + * Frees a slot on a shard by its location (shard ID and slot number). + * + * @param shardId - The unique identifier of the shard. + * @param slotNo - The slot number to free. + * @return A promise that resolves when the slot is freed. + */ + freeByLocation(shardId: string | number, slotNo: number) { + return this.factory.autocommit().freeByLocation(shardId, slotNo) + } + + /** + * Frees a slot on a shard by resource identifier. + * + * @param shardId - The unique identifier of the shard. + * @param resource - The resource details to identify the slot. + * @param resource.kind - The type of resource. + * @param resource.bucketName - The name of the bucket containing the resource. + * @param resource.logicalName - The logical name of the resource. + * @param resource.tenantId - The ID of the tenant owning the resource. + * @return A promise that resolves when the slot is freed. + */ + freeByResource(shardId: string | number, resource: ShardResource): Promise { + const resourceId = `${resource.kind}::${resource.bucketName}::${resource.logicalName}` + return this.factory.autocommit().freeByResource(shardId, resourceId, resource.tenantId) + } + + /** + * Retrieves statistics for shards, optionally filtered by resource kind. + * + * @param kind - Optional. The resource kind to filter statistics by. + * @return A promise that resolves to an array of shard statistics. + */ + shardStats(kind?: ResourceKind) { + return this.factory.autocommit().shardStats(kind) + } + + /** + * Finds the shard associated with a specific resource. + * + * @param param - The resource identifier parameters. + * @param param.kind - The type of resource. + * @param param.tenantId - The ID of the tenant owning the resource. + * @param param.logicalName - The logical name of the resource. + * @param param.bucketName - The name of the bucket containing the resource. + * @return A promise that resolves to the shard row if found, or null if not found. + */ + async findShardByResourceId(param: { + kind: string + tenantId: string + logicalName: string + bucketName: string + }) { + const resourceId = `${param.kind}::${param.bucketName}::${param.logicalName}` + return this.factory.autocommit().findShardByResourceId(param.tenantId, resourceId) + } +} diff --git a/src/internal/sharding/strategy/single-shard.ts b/src/internal/sharding/strategy/single-shard.ts new file mode 100644 index 000000000..4c3aa963d --- /dev/null +++ b/src/internal/sharding/strategy/single-shard.ts @@ -0,0 +1,84 @@ +import { Sharder, ShardResource } from '../sharder' +import { ResourceKind, ShardRow, ShardStatus } from '@internal/sharding/store' + +export class SingleShard implements Sharder { + constructor( + protected readonly singleShard: { + shardKey: string + capacity: number + } + ) {} + + freeByResource(): Promise { + return Promise.resolve() + } + + cancel(): Promise { + return Promise.resolve(undefined) + } + + confirm(): Promise { + return Promise.resolve(undefined) + } + + createShard(opts: { + kind: ResourceKind + shardKey: string + capacity?: number + status?: ShardStatus + }): Promise { + return Promise.resolve({ + shard_key: opts.shardKey, + capacity: opts.capacity || this.singleShard.capacity, + kind: opts.kind, + id: this.singleShard.shardKey, + status: 'active', + next_slot: 1, + created_at: new Date().toISOString(), + }) + } + + expireLeases(): Promise { + return Promise.resolve(0) + } + + findShardByResourceId(param: ShardResource): Promise { + return Promise.resolve({ + id: this.singleShard.shardKey, + kind: param.kind, + shard_key: this.singleShard.shardKey, + capacity: this.singleShard.capacity, + status: 'active', + next_slot: 1, + created_at: new Date().toISOString(), + }) + } + + freeByLocation(): Promise { + return Promise.resolve(undefined) + } + + reserve(): Promise<{ + reservationId: string + shardId: string + shardKey: string + slotNo: number + leaseExpiresAt: string + }> { + return Promise.resolve({ + leaseExpiresAt: '', + reservationId: '', + shardId: this.singleShard.shardKey, + shardKey: this.singleShard.shardKey, + slotNo: 0, + }) + } + + setShardStatus(): Promise { + return Promise.resolve(undefined) + } + + shardStats(): Promise { + return Promise.resolve(undefined) + } +} diff --git a/src/start/server.ts b/src/start/server.ts index f1f38e93a..66e81f2f1 100644 --- a/src/start/server.ts +++ b/src/start/server.ts @@ -5,7 +5,12 @@ import { IncomingMessage, Server, ServerResponse } from 'node:http' import build from '../app' import buildAdmin from '../admin-app' import { getConfig } from '../config' -import { listenForTenantUpdate, PubSub, TenantConnection } from '@internal/database' +import { + listenForTenantUpdate, + multitenantKnex, + PubSub, + TenantConnection, +} from '@internal/database' import { logger, logSchema } from '@internal/monitoring' import { Queue } from '@internal/queue' import { registerWorkers } from '@storage/events' @@ -18,6 +23,7 @@ import { startAsyncMigrations, } from '@internal/database/migrations' import { Cluster } from '@internal/cluster/cluster' +import { KnexShardStoreFactory, ShardCatalog } from '@internal/sharding' const shutdownSignal = new AsyncAbortController() @@ -47,12 +53,26 @@ main() * Start Storage API server */ async function main() { - const { databaseURL, isMultitenant, pgQueueEnable, dbMigrationFreezeAt } = getConfig() + const { databaseURL, isMultitenant, pgQueueEnable, dbMigrationFreezeAt, vectorS3Buckets } = + getConfig() + + // Sharding for special buckets (vectors, analytics) + const sharding = new ShardCatalog(new KnexShardStoreFactory(multitenantKnex)) // Migrations if (isMultitenant) { await runMultitenantMigrations() await listenForTenantUpdate(PubSub) + + // Create shards for vector S3 buckets + await sharding.createShards( + vectorS3Buckets?.map((s) => ({ + shardKey: s, + kind: 'vector', + capacity: 10000, + status: 'active', + })) + ) } else { await runMigrationsOnTenant({ databaseUrl: databaseURL, diff --git a/src/start/worker.ts b/src/start/worker.ts index 2ea35ec75..a74900aad 100644 --- a/src/start/worker.ts +++ b/src/start/worker.ts @@ -1,5 +1,5 @@ import { Queue } from '@internal/queue' -import { logger, logSchema } from '@internal/monitoring' +import { logger, logSchema, setLogger } from '@internal/monitoring' import { listenForTenantUpdate, PubSub } from '@internal/database' import { AsyncAbortController } from '@internal/concurrency' import { registerWorkers } from '@storage/events' @@ -8,6 +8,9 @@ import { getConfig } from '../config' import adminApp from '../admin-app' import { bindShutdownSignals, createServerClosedPromise, shutdown } from './shutdown' +const workerLogger = logger.child({ service: 'worker' }) +setLogger(workerLogger) + const shutdownSignal = new AsyncAbortController() bindShutdownSignals(shutdownSignal) diff --git a/src/storage/database/knex.ts b/src/storage/database/knex.ts index 50b1d27d7..7a8a5f9a2 100644 --- a/src/storage/database/knex.ts +++ b/src/storage/database/knex.ts @@ -20,9 +20,10 @@ import { import { DatabaseError } from 'pg' import { TenantConnection } from '@internal/database' import { DbQueryPerformance } from '@internal/monitoring/metrics' -import { isUuid } from '../limits' +import { hashStringToInt } from '@internal/hashing' import { DBMigration, tenantHasMigrations } from '@internal/database/migrations' import { getConfig } from '../../config' +import { isUuid } from '../limits' const { isMultitenant } = getConfig() @@ -1133,13 +1134,3 @@ export class DBError extends StorageBackendError implements RenderableError { } } } - -export default function hashStringToInt(str: string): number { - let hash = 5381 - let i = -1 - while (i < str.length - 1) { - i += 1 - hash = (hash * 33) ^ str.charCodeAt(i) - } - return hash >>> 0 -} diff --git a/src/storage/events/index.ts b/src/storage/events/index.ts index b5a872298..dac72a76e 100644 --- a/src/storage/events/index.ts +++ b/src/storage/events/index.ts @@ -13,5 +13,4 @@ export * from './migrations/reset-migrations' export * from './jwks/jwks-create-signing-secret' export * from './pgboss/upgrade-v10' export * from './pgboss/move-jobs' -export * from './vectors/reconcile' export * from './workers' diff --git a/src/storage/events/lifecycle/bucket-deleted.ts b/src/storage/events/lifecycle/bucket-deleted.ts index 6309df577..669d58ee6 100644 --- a/src/storage/events/lifecycle/bucket-deleted.ts +++ b/src/storage/events/lifecycle/bucket-deleted.ts @@ -44,5 +44,10 @@ export class BucketDeleted extends BaseEvent { if (resources.namespaces > 0 || resources.tables > 0) { throw ERRORS.BucketNotEmpty(job.data.bucketId) } + + await metastore.dropCatalog({ + tenantId: job.data.tenant.ref, + bucketId: job.data.bucketId, + }) } } diff --git a/src/storage/events/vectors/reconcile.ts b/src/storage/events/vectors/reconcile.ts deleted file mode 100644 index a416aaea7..000000000 --- a/src/storage/events/vectors/reconcile.ts +++ /dev/null @@ -1,46 +0,0 @@ -import { BaseEvent } from '../base-event' -import { getTenantConfig } from '@internal/database' -import { JobWithMetadata, Queue, SendOptions, WorkOptions } from 'pg-boss' -import { BasePayload } from '@internal/queue' -import { logger, logSchema } from '@internal/monitoring' - -interface ResetMigrationsPayload extends BasePayload { - tenantId: string -} - -export class VectorReconcile extends BaseEvent { - static queueName = 'vector-reconcile' - - static getQueueOptions(): Queue { - return { - name: this.queueName, - policy: 'exactly_once', - } as const - } - - static getWorkerOptions(): WorkOptions { - return { - includeMetadata: true, - } - } - - static getSendOptions(payload: ResetMigrationsPayload): SendOptions { - return { - expireInHours: 2, - singletonKey: payload.tenantId, - retryLimit: 3, - retryDelay: 5, - priority: 10, - } - } - - static async handle(job: JobWithMetadata) { - const tenantId = job.data.tenant.ref - const tenant = await getTenantConfig(tenantId) - - logSchema.info(logger, `[Migrations] resetting migrations for ${tenantId}`, { - type: 'migrations', - project: tenantId, - }) - } -} diff --git a/src/storage/limits.ts b/src/storage/limits.ts index 562f8e816..864840f60 100644 --- a/src/storage/limits.ts +++ b/src/storage/limits.ts @@ -3,7 +3,7 @@ import { getFileSizeLimit as getFileSizeLimitForTenant, getFeatures, } from '../internal/database/tenant' -import { ERRORS } from '../internal/errors' +import { ERRORS } from '@internal/errors' const { isMultitenant, imageTransformationEnabled, icebergBucketDetectionSuffix } = getConfig() diff --git a/src/storage/protocols/iceberg/catalog/rest-catalog-client.ts b/src/storage/protocols/iceberg/catalog/rest-catalog-client.ts index 47c924761..c94e1c59e 100644 --- a/src/storage/protocols/iceberg/catalog/rest-catalog-client.ts +++ b/src/storage/protocols/iceberg/catalog/rest-catalog-client.ts @@ -1,6 +1,7 @@ import axios, { AxiosError, AxiosInstance, InternalAxiosRequestConfig } from 'axios' import { ErrorCode, ERRORS, StorageBackendError } from '@internal/errors' import { signRequest } from 'aws-sigv4-sign' +import JSONBigint from 'json-bigint' export interface GetConfigRequest { tenantId?: string @@ -268,6 +269,9 @@ export interface TableMetadata { schemas?: Schema[] /** The ID of the current schema in the `schemas` array. */ 'current-schema-id'?: number + + 'current-snapshot-id'?: number + /** The last column ID assigned (for tracking new columns). */ 'last-column-id'?: number /** All known partition specs for the table. */ @@ -344,6 +348,25 @@ export class RestCatalogClient { return this.auth.authorize(req) }) + // request interceptor, preventing the response the default behaviour of parsing the response with JSON.parse + this.httpClient.interceptors.request.use((request) => { + request.transformRequest = [ + (data) => { + return data ? JSONBigint.stringify(data) : data + }, + ] + request.transformResponse = [(data) => data] + return request + }) + + // response interceptor parsing the response data with JSONbigint, and returning the response + this.httpClient.interceptors.response.use((response) => { + if (response.data) { + response.data = JSONBigint.parse(response.data) + } + return response + }) + this.httpClient.interceptors.response.use( // On 2xx responses, just pass through (response) => response, @@ -562,7 +585,13 @@ export class RestCatalogClient { snapshots: params.snapshots, }, }) - .then((response) => response.data) + .then((response) => { + // console.log({ + // url: response.request., + // headers: response.request.headers, + // }) + return response.data + }) .catch((error) => { if (error instanceof AxiosError) { console.error('Error fetching configuration:', error.response?.data) @@ -696,7 +725,7 @@ export class SignV4Auth { { method: req.method?.toUpperCase(), headers: req.headers, - body: req.data ? JSON.stringify(req.data) : undefined, + body: req.data ? JSONBigint.stringify(req.data) : undefined, }, { service: 's3tables', @@ -704,6 +733,20 @@ export class SignV4Auth { } ) + // Replace the original console.log with this curl command generator + // console.log( + // ` + // curl -X ${req.method?.toUpperCase() || 'GET'} \\ + // ${Array.from(signedReq.headers.entries()) + // .map(([name, value]) => ` -H '${name}: ${value}' \\`) + // .join('\n')} + // "${(req.baseURL || '') + req.url + (queryString ? `?${queryString}` : '')}"${ + // req.data ? ` \\\n -d '${JSON.stringify(req.data)}'` : '' + // } + // `.trim() + // ) + + // Keep the original code for setting headers signedReq.headers.forEach((headerValue, headerName) => { req.headers.set(headerName, headerValue as string, true) }) diff --git a/src/storage/protocols/iceberg/knex.ts b/src/storage/protocols/iceberg/knex.ts index 977d72fa3..b5e5d12a6 100644 --- a/src/storage/protocols/iceberg/knex.ts +++ b/src/storage/protocols/iceberg/knex.ts @@ -54,6 +54,7 @@ export interface Metastore { assignNamespace(params: AssignInterfaceParams): Promise listNamespaces(params: ListNamespaceParams): Promise dropNamespace(params: DropNamespaceParams): Promise + dropCatalog(params: { tenantId?: string; bucketId: string }): Promise createTable(params: CreateTableParams): Promise dropTable(params: DropTableRequest): Promise findTableByLocation(params: { tenantId?: string; location: string }): Promise @@ -92,6 +93,21 @@ export class KnexMetastore implements Metastore { private readonly ops: { schema: string; multiTenant?: boolean } ) {} + async dropCatalog(params: { tenantId?: string | undefined; bucketId: string }): Promise { + if (!this.ops.multiTenant) { + return Promise.resolve(false) + } + + await this.db + .withSchema(this.ops.schema) + .table('iceberg_catalogs') + .where('tenant_id', params.tenantId) + .andWhere('id', params.bucketId) + .del() + + return true + } + listTables(param: { tenantId: string pageSize: number | undefined @@ -129,7 +145,6 @@ export class KnexMetastore implements Metastore { if (this.ops.multiTenant) { countNamespaces.andWhere('tenant_id', params.tenantId) - countNamespaces.select('tenant_id') } const countTables = this.db @@ -141,7 +156,6 @@ export class KnexMetastore implements Metastore { if (this.ops.multiTenant) { countTables.andWhere('tenant_id', params.tenantId) - countTables.select('tenant_id') } const resultQuery = this.db diff --git a/src/storage/protocols/vector/adapter/s3-vector.ts b/src/storage/protocols/vector/adapter/s3-vector.ts index b73b46e42..6b81cd53b 100644 --- a/src/storage/protocols/vector/adapter/s3-vector.ts +++ b/src/storage/protocols/vector/adapter/s3-vector.ts @@ -1,4 +1,6 @@ import { + AccessDeniedException, + ConflictException, CreateIndexCommand, CreateIndexCommandInput, CreateIndexCommandOutput, @@ -14,6 +16,7 @@ import { ListVectorsCommand, ListVectorsInput, ListVectorsOutput, + NotFoundException, PutVectorsCommand, PutVectorsInput, PutVectorsOutput, @@ -23,6 +26,7 @@ import { S3VectorsClient, } from '@aws-sdk/client-s3vectors' import { getConfig } from '../../../../config' +import { ERRORS } from '@internal/errors' export interface VectorStore { createVectorIndex(command: CreateIndexCommandInput): Promise @@ -51,36 +55,76 @@ export class S3Vector implements VectorStore { constructor(protected readonly s3VectorClient: S3VectorsClient) {} getVectors(getVectorsInput: GetVectorsCommandInput): Promise { - return this.s3VectorClient.send(new GetVectorsCommand(getVectorsInput)) + return this.handleError( + () => this.s3VectorClient.send(new GetVectorsCommand(getVectorsInput)), + { type: 'vector', name: getVectorsInput.indexName || 'unknown' } + ) } deleteVectors(deleteVectorsInput: DeleteVectorsInput): Promise { - return this.s3VectorClient.send(new DeleteVectorsCommand(deleteVectorsInput)) + return this.handleError( + () => this.s3VectorClient.send(new DeleteVectorsCommand(deleteVectorsInput)), + { type: 'vector', name: deleteVectorsInput.indexName || 'unknown' } + ) } queryVectors(queryInput: QueryVectorsInput): Promise { - return this.s3VectorClient.send(new QueryVectorsCommand(queryInput)) + return this.handleError(() => this.s3VectorClient.send(new QueryVectorsCommand(queryInput)), { + type: 'vector-index', + name: queryInput.indexName || 'unknown', + }) } async listVectors(command: ListVectorsInput): Promise { - return this.s3VectorClient.send(new ListVectorsCommand(command)) + return this.handleError(() => this.s3VectorClient.send(new ListVectorsCommand(command)), { + type: 'vector-index', + name: command.indexName || 'unknown', + }) } putVectors(command: PutVectorsInput): Promise { - const input = new PutVectorsCommand(command) - - return this.s3VectorClient.send(input) + return this.handleError(() => this.s3VectorClient.send(new PutVectorsCommand(command)), { + type: 'vector', + name: command.indexName || 'unknown', + }) } deleteVectorIndex(param: DeleteIndexCommandInput): Promise { - const command = new DeleteIndexCommand(param) - - return this.s3VectorClient.send(command) + return this.handleError(() => this.s3VectorClient.send(new DeleteIndexCommand(param)), { + type: 'vector-index', + name: param.indexName || 'unknown', + }) } - createVectorIndex(command: CreateIndexCommandInput): Promise { - const createIndexCommand = new CreateIndexCommand(command) + async createVectorIndex(command: CreateIndexCommandInput): Promise { + return this.handleError(() => this.s3VectorClient.send(new CreateIndexCommand(command)), { + type: 'vector-index', + name: command.indexName || 'unknown', + }) + } - return this.s3VectorClient.send(createIndexCommand) + protected async handleError( + fn: () => Promise, + resource: { type: string; name: string } + ): Promise { + try { + return await fn() + } catch (e) { + if (e instanceof ConflictException) { + throw ERRORS.S3VectorConflictException(resource.type, resource.name) + } + + if (e instanceof AccessDeniedException) { + throw ERRORS.AccessDenied( + 'Access denied to S3 Vector service. Please check your permissions.' + ) + } + + if (e instanceof NotFoundException) { + throw ERRORS.S3VectorNotFoundException(resource.type, e.message) + } + + throw e + } } } diff --git a/src/storage/protocols/vector/knex.ts b/src/storage/protocols/vector/knex.ts index e3a33cd3c..6928de40f 100644 --- a/src/storage/protocols/vector/knex.ts +++ b/src/storage/protocols/vector/knex.ts @@ -5,6 +5,7 @@ import { VectorBucket } from '@storage/schemas' import { ListVectorBucketsInput } from '@aws-sdk/client-s3vectors' import { DatabaseError } from 'pg' import { wait } from '@internal/concurrency' +import { hashStringToInt } from '@internal/hashing' type DBVectorIndex = VectorIndex & { id: string; created_at: Date; updated_at: Date } @@ -42,6 +43,8 @@ export interface VectorMetadataDB { config?: Knex.TransactionConfig ): Promise + lockResource(resourceType: 'bucket' | 'index', resourceId: string): Promise + findVectorBucket(vectorBucketName: string): Promise createVectorBucket(bucketName: string): Promise deleteVectorBucket(bucketName: string, vectorIndexName: string): Promise @@ -59,6 +62,11 @@ export interface VectorMetadataDB { export class KnexVectorMetadataDB implements VectorMetadataDB { constructor(protected readonly knex: Knex) {} + lockResource(resourceType: 'bucket' | 'index', resourceId: string): Promise { + const lockId = hashStringToInt(`vector:${resourceType}:${resourceId}`) + return this.knex.raw('SELECT pg_advisory_xact_lock(?::bigint)', [lockId]) + } + async countIndexes(bucketId: string): Promise { const row = await this.knex .withSchema('storage') @@ -193,21 +201,33 @@ export class KnexVectorMetadataDB implements VectorMetadataDB { return index } - createVectorIndex(data: CreateVectorIndexParams) { - return this.knex - .withSchema('storage') - .table('vector_indexes') - .insert({ - bucket_id: data.vectorBucketName, - data_type: data.dataType, - name: data.indexName, - dimension: data.dimension, - distance_metric: data.distanceMetric, - metadata_configuration: data.metadataConfiguration, - }) + async createVectorIndex(data: CreateVectorIndexParams) { + try { + return await this.knex + .withSchema('storage') + .table('vector_indexes') + .insert({ + bucket_id: data.vectorBucketName, + data_type: data.dataType, + name: data.indexName, + dimension: data.dimension, + distance_metric: data.distanceMetric, + metadata_configuration: data.metadataConfiguration, + }) + } catch (e) { + if (e instanceof Error && e instanceof DatabaseError) { + if (e.code === '23505') { + throw ERRORS.S3VectorConflictException('vector index', data.indexName) + } + } + throw e + } } - async withTransaction(fn: (db: KnexVectorMetadataDB) => T): Promise { + async withTransaction( + fn: (db: KnexVectorMetadataDB) => T, + config?: Knex.TransactionConfig + ): Promise { const maxRetries = 3 let attempt = 0 let lastError: Error | undefined = undefined @@ -216,8 +236,9 @@ export class KnexVectorMetadataDB implements VectorMetadataDB { try { return await this.knex.transaction(async (trx) => { const trxDb = new KnexVectorMetadataDB(trx) - return fn(trxDb) - }) + const result = await fn(trxDb) + return result + }, config) } catch (error) { attempt++ @@ -233,7 +254,7 @@ export class KnexVectorMetadataDB implements VectorMetadataDB { } } - throw ERRORS.TransactionError('Transaction failed after maximum retries', lastError) + throw ERRORS.TransactionError(`Transaction failed after maximum ${attempt} retries`, lastError) } deleteVectorIndex(bucketName: string, vectorIndexName: string): Promise { diff --git a/src/storage/protocols/vector/vector-store.ts b/src/storage/protocols/vector/vector-store.ts index f35a6589a..5e57f5836 100644 --- a/src/storage/protocols/vector/vector-store.ts +++ b/src/storage/protocols/vector/vector-store.ts @@ -18,10 +18,11 @@ import { import { VectorMetadataDB } from './knex' import { VectorStore } from './adapter/s3-vector' import { ERRORS } from '@internal/errors' +import { Sharder } from '@internal/sharding/sharder' +import { logger, logSchema } from '@internal/monitoring' interface VectorStoreConfig { tenantId: string - vectorBucketName: string maxBucketCount: number maxIndexCount: number } @@ -30,6 +31,7 @@ export class VectorStoreManager { constructor( protected readonly vectorStore: VectorStore, protected readonly db: VectorMetadataDB, + protected readonly sharding: Sharder, protected readonly config: VectorStoreConfig ) {} @@ -127,8 +129,12 @@ export class VectorStoreManager { indexName: this.getIndexName(command.indexName), } - await this.db.withTransaction( - async (tx) => { + let shardReservation: { reservationId: string; shardKey: string; shardId: string } | undefined + + try { + await this.db.withTransaction(async (tx) => { + await tx.lockResource('bucket', command.vectorBucketName!) + const indexCount = await tx.countIndexes(command.vectorBucketName!) if (indexCount >= this.config.maxIndexCount) { @@ -144,21 +150,67 @@ export class VectorStoreManager { vectorBucketName: command.vectorBucketName!, }) + shardReservation = await this.sharding.reserve({ + kind: 'vector', + bucketName: command.vectorBucketName!, + tenantId: this.config.tenantId, + logicalName: command.indexName!, + }) + + if (!shardReservation) { + throw ERRORS.S3VectorNoAvailableShard() + } + try { + if ( + createIndexInput.metadataConfiguration && + createIndexInput.metadataConfiguration.nonFilterableMetadataKeys && + createIndexInput.metadataConfiguration.nonFilterableMetadataKeys.length === 0 + ) { + delete createIndexInput.metadataConfiguration + } + await this.vectorStore.createVectorIndex({ ...createIndexInput, - vectorBucketName: this.config.vectorBucketName, + vectorBucketName: shardReservation.shardKey, + }) + + await this.sharding.confirm(shardReservation.reservationId, { + kind: 'vector', + bucketName: command.vectorBucketName!, + tenantId: this.config.tenantId, + logicalName: command.indexName!, }) } catch (e) { + logSchema.error(logger, 'Vector index creation failed', { + type: 'vector', + error: e, + project: this.config.tenantId, + }) if (e instanceof ConflictException) { + await this.sharding.confirm(shardReservation.reservationId, { + kind: 'vector', + bucketName: command.vectorBucketName!, + tenantId: this.config.tenantId, + logicalName: command.indexName!, + }) return } throw e } - }, - { isolationLevel: 'serializable' } - ) + }) + } catch (error) { + logSchema.error(logger, 'Create vector index transaction failed', { + type: 'vector', + error: error, + project: this.config.tenantId, + }) + if (shardReservation) { + await this.sharding.cancel(shardReservation.reservationId) + } + throw error + } } async deleteIndex(command: DeleteIndexInput): Promise { @@ -175,10 +227,28 @@ export class VectorStoreManager { const vectorIndexName = this.getIndexName(command.indexName) await this.db.withTransaction(async (tx) => { + const shard = await this.sharding.findShardByResourceId({ + kind: 'vector', + tenantId: this.config.tenantId, + logicalName: command.indexName!, + bucketName: command.vectorBucketName!, + }) + + if (!shard) { + throw ERRORS.S3VectorNoAvailableShard() + } + await tx.deleteVectorIndex(command.vectorBucketName!, command.indexName!) + await this.sharding.freeByResource(shard.id, { + kind: 'vector', + tenantId: this.config.tenantId, + bucketName: command.vectorBucketName!, + logicalName: command.indexName!, + }) + await this.vectorStore.deleteVectorIndex({ - vectorBucketName: this.config.vectorBucketName, + vectorBucketName: shard.shard_key, indexName: vectorIndexName, }) }) @@ -239,11 +309,23 @@ export class VectorStoreManager { throw ERRORS.MissingParameter('vectorBucketName') } - await this.db.findVectorIndexForBucket(command.vectorBucketName, command.indexName) + const [shard] = await Promise.all([ + this.sharding.findShardByResourceId({ + kind: 'vector', + tenantId: this.config.tenantId, + logicalName: command.indexName!, + bucketName: command.vectorBucketName!, + }), + this.db.findVectorIndexForBucket(command.vectorBucketName, command.indexName), + ]) + + if (!shard) { + throw ERRORS.S3VectorNoAvailableShard() + } const putVectorsInput = { ...command, - vectorBucketName: this.config.vectorBucketName, + vectorBucketName: shard.shard_key, indexName: this.getIndexName(command.indexName), } await this.vectorStore.putVectors(putVectorsInput) @@ -258,11 +340,23 @@ export class VectorStoreManager { throw ERRORS.MissingParameter('vectorBucketName') } - await this.db.findVectorIndexForBucket(command.vectorBucketName, command.indexName) + const [shard] = await Promise.all([ + this.sharding.findShardByResourceId({ + kind: 'vector', + tenantId: this.config.tenantId, + logicalName: command.indexName!, + bucketName: command.vectorBucketName!, + }), + this.db.findVectorIndexForBucket(command.vectorBucketName, command.indexName), + ]) + + if (!shard) { + throw ERRORS.S3VectorNoAvailableShard() + } const deleteVectorsInput = { ...command, - vectorBucketName: this.config.vectorBucketName, + vectorBucketName: shard.shard_key, indexName: this.getIndexName(command.indexName), } @@ -278,11 +372,23 @@ export class VectorStoreManager { throw ERRORS.MissingParameter('vectorBucketName') } - await this.db.findVectorIndexForBucket(command.vectorBucketName, command.indexName) + const [shard] = await Promise.all([ + this.sharding.findShardByResourceId({ + kind: 'vector', + tenantId: this.config.tenantId, + logicalName: command.indexName!, + bucketName: command.vectorBucketName!, + }), + this.db.findVectorIndexForBucket(command.vectorBucketName, command.indexName), + ]) + + if (!shard) { + throw ERRORS.S3VectorNoAvailableShard() + } const listVectorsInput = { ...command, - vectorBucketName: this.config.vectorBucketName, + vectorBucketName: shard.shard_key, indexName: this.getIndexName(command.indexName), } @@ -303,11 +409,23 @@ export class VectorStoreManager { throw ERRORS.MissingParameter('vectorBucketName') } - await this.db.findVectorIndexForBucket(command.vectorBucketName, command.indexName) + const [shard] = await Promise.all([ + this.sharding.findShardByResourceId({ + kind: 'vector', + tenantId: this.config.tenantId, + logicalName: command.indexName!, + bucketName: command.vectorBucketName!, + }), + this.db.findVectorIndexForBucket(command.vectorBucketName, command.indexName), + ]) + + if (!shard) { + throw ERRORS.S3VectorNoAvailableShard() + } const queryInput = { ...command, - vectorBucketName: this.config.vectorBucketName, + vectorBucketName: shard.shard_key, indexName: this.getIndexName(command.indexName), } return this.vectorStore.queryVectors(queryInput) @@ -322,11 +440,23 @@ export class VectorStoreManager { throw ERRORS.MissingParameter('vectorBucketName') } - await this.db.findVectorIndexForBucket(command.vectorBucketName, command.indexName) + const [shard] = await Promise.all([ + this.sharding.findShardByResourceId({ + kind: 'vector', + tenantId: this.config.tenantId, + logicalName: command.indexName!, + bucketName: command.vectorBucketName!, + }), + this.db.findVectorIndexForBucket(command.vectorBucketName, command.indexName), + ]) + + if (!shard) { + throw ERRORS.S3VectorNoAvailableShard() + } const getVectorsInput = { ...command, - vectorBucketName: this.config.vectorBucketName, + vectorBucketName: shard.shard_key, indexName: this.getIndexName(command.indexName), } diff --git a/src/storage/storage.ts b/src/storage/storage.ts index e2b5fcad2..f8f28e9bc 100644 --- a/src/storage/storage.ts +++ b/src/storage/storage.ts @@ -12,7 +12,6 @@ import { import { getConfig } from '../config' import { ObjectStorage } from './object' import { InfoRenderer } from '@storage/renderer/info' -import { logger, logSchema } from '@internal/monitoring' import { StorageObjectLocator } from '@storage/locator' import { BucketCreatedEvent, BucketDeleted } from '@storage/events' import { tenantHasMigrations } from '@internal/database/migrations' @@ -245,7 +244,7 @@ export class Storage { return this.db.withTransaction(async (db) => { const deleted = await db.deleteAnalyticsBucket(id) - await BucketDeleted.invoke({ + await BucketDeleted.invokeOrSend({ bucketId: id, type: 'ANALYTICS', tenant: { diff --git a/src/test/sharding.test.ts b/src/test/sharding.test.ts new file mode 100644 index 000000000..9ebfdbb7a --- /dev/null +++ b/src/test/sharding.test.ts @@ -0,0 +1,778 @@ +'use strict' + +import { Knex } from 'knex' +import { KnexShardStoreFactory, ShardCatalog } from '@internal/sharding' +import { useStorage } from './utils/storage' +import { + ExpiredReservationError, + NoActiveShardError, + ReservationNotFoundError, +} from '@internal/sharding/errors' +import { multitenantKnex } from '@internal/database' +import { randomUUID } from 'crypto' +import { runMultitenantMigrations } from '@internal/database/migrations' + +describe('Sharding System', () => { + const storageTest = useStorage() + let db: Knex + let storeFactory: KnexShardStoreFactory + let catalog: ShardCatalog + + beforeAll(async () => { + db = multitenantKnex + storeFactory = new KnexShardStoreFactory(db) + catalog = new ShardCatalog(storeFactory) + + await runMultitenantMigrations() + }) + + afterAll(async () => { + await storageTest.database.connection.dispose() + }) + + beforeEach(async () => { + // Clean up sharding tables before each test + await db('shard_reservation').delete() + await db('shard_slots').delete() + await db('shard').delete() + }) + + describe('Shard Management', () => { + it('should create a shard successfully', async () => { + const shard = await catalog.createShard({ + kind: 'vector', + shardKey: 'test-shard-1', + capacity: 100, + status: 'active', + }) + + expect(shard).toBeDefined() + expect(shard.shard_key).toBe('test-shard-1') + expect(shard.kind).toBe('vector') + expect(shard.capacity).toBe(100) + expect(shard.status).toBe('active') + }) + + it('should be idempotent when creating the same shard twice', async () => { + const shard1 = await catalog.createShard({ + kind: 'vector', + shardKey: 'test-shard-1', + capacity: 100, + }) + + const shard2 = await catalog.createShard({ + kind: 'vector', + shardKey: 'test-shard-1', + capacity: 100, + }) + + expect(shard1.id).toBe(shard2.id) + expect(shard1.shard_key).toBe(shard2.shard_key) + }) + + it('should create multiple shards in batch', async () => { + const shards = await catalog.createShards([ + { kind: 'vector', shardKey: 'shard-1', capacity: 100 }, + { kind: 'vector', shardKey: 'shard-2', capacity: 200 }, + { kind: 'vector', shardKey: 'shard-3', capacity: 300 }, + ]) + + expect(shards).toHaveLength(3) + expect(shards[0].shard_key).toBe('shard-1') + expect(shards[1].shard_key).toBe('shard-2') + expect(shards[2].shard_key).toBe('shard-3') + }) + + it('should update shard status', async () => { + const shard = await catalog.createShard({ + kind: 'vector', + shardKey: 'test-shard-1', + capacity: 100, + }) + + await catalog.setShardStatus(shard.id, 'draining') + + const store = storeFactory.autocommit() + const shards = await store.listActiveShards('vector') + expect(shards).toHaveLength(0) // draining shards are not active + }) + + it('should get shard stats', async () => { + await catalog.createShard({ + kind: 'vector', + shardKey: 'test-shard-1', + capacity: 100, + }) + + const stats = await catalog.shardStats('vector') + + expect(stats).toHaveLength(1) + expect(stats[0].shardKey).toBe('test-shard-1') + expect(stats[0].capacity).toBe(100) + expect(stats[0].used).toBe(0) + expect(stats[0].free).toBe(100) + }) + }) + + describe('Reservation Flow', () => { + beforeEach(async () => { + await catalog.createShard({ + kind: 'vector', + shardKey: 'test-shard-1', + capacity: 100, + }) + }) + + it('should reserve a slot successfully', async () => { + const reservation = await catalog.reserve({ + kind: 'vector', + tenantId: 'tenant-1', + bucketName: 'bucket-1', + logicalName: 'index-1', + }) + + expect(reservation).toBeDefined() + expect(reservation.reservationId).toBeDefined() + expect(reservation.shardKey).toBe('test-shard-1') + expect(reservation.slotNo).toBe(0) + expect(reservation.leaseExpiresAt).toBeDefined() + }) + + it('should be idempotent - return existing reservation', async () => { + const res1 = await catalog.reserve({ + kind: 'vector', + tenantId: 'tenant-1', + bucketName: 'bucket-1', + logicalName: 'index-1', + }) + + const res2 = await catalog.reserve({ + kind: 'vector', + tenantId: 'tenant-1', + bucketName: 'bucket-1', + logicalName: 'index-1', + }) + + expect(res1.reservationId).toBe(res2.reservationId) + expect(res1.slotNo).toBe(res2.slotNo) + }) + + it('should confirm a reservation', async () => { + const reservation = await catalog.reserve({ + kind: 'vector', + tenantId: 'tenant-1', + bucketName: 'bucket-1', + logicalName: 'index-1', + }) + + await catalog.confirm(reservation.reservationId, { + kind: 'vector', + tenantId: 'tenant-1', + bucketName: 'bucket-1', + logicalName: 'index-1', + }) + + // Check that slot is now confirmed + const slots = await db('shard_slots').where({ slot_no: reservation.slotNo }).first() + + expect(slots.resource_id).toBe('vector::bucket-1::index-1') + }) + + it('should cancel a reservation', async () => { + const reservation = await catalog.reserve({ + kind: 'vector', + tenantId: 'tenant-1', + bucketName: 'bucket-1', + logicalName: 'index-1', + }) + + await catalog.cancel(reservation.reservationId) + + const resv = await db('shard_reservation').where({ id: reservation.reservationId }).first() + + expect(resv.status).toBe('cancelled') + }) + + it('should throw error when confirming non-existent reservation', async () => { + await expect( + catalog.confirm(randomUUID(), { + kind: 'vector', + tenantId: 'tenant-1', + bucketName: 'bucket-1', + logicalName: 'index-1', + }) + ).rejects.toThrow(ReservationNotFoundError) + }) + + it('should throw error when confirming expired reservation', async () => { + const reservation = await catalog.reserve({ + kind: 'vector', + tenantId: 'tenant-1', + bucketName: 'bucket-1', + logicalName: 'index-1', + leaseMs: 1, // 1ms lease + }) + + // Wait for lease to expire + await new Promise((resolve) => setTimeout(resolve, 10)) + + await expect( + catalog.confirm(reservation.reservationId, { + kind: 'vector', + tenantId: 'tenant-1', + bucketName: 'bucket-1', + logicalName: 'index-1', + }) + ).rejects.toThrow(ExpiredReservationError) + }) + + it('should free a slot by location', async () => { + const reservation = await catalog.reserve({ + kind: 'vector', + tenantId: 'tenant-1', + bucketName: 'bucket-1', + logicalName: 'index-1', + }) + + await catalog.confirm(reservation.reservationId, { + kind: 'vector', + tenantId: 'tenant-1', + bucketName: 'bucket-1', + logicalName: 'index-1', + }) + + await catalog.freeByLocation(reservation.shardId, reservation.slotNo) + + const slot = await db('shard_slots').where({ slot_no: reservation.slotNo }).first() + + expect(slot.resource_id).toBeNull() + expect(slot.tenant_id).toBeNull() + }) + + it('should delete cancelled reservation and reuse slot', async () => { + const res1 = await catalog.reserve({ + kind: 'vector', + tenantId: 'tenant-1', + bucketName: 'bucket-1', + logicalName: 'index-1', + }) + + await catalog.cancel(res1.reservationId) + + // Try to reserve again with same logical key + const res2 = await catalog.reserve({ + kind: 'vector', + tenantId: 'tenant-1', + bucketName: 'bucket-1', + logicalName: 'index-1', + }) + + expect(res2.reservationId).not.toBe(res1.reservationId) + expect(res2.slotNo).toBe(res1.slotNo) // Same slot reused + }) + + it('should handle UniqueViolationError and return existing reservation', async () => { + // This test simulates the race condition where: + // 1. Process A checks for existing reservation (findReservationByKindKey) - finds nothing + // 2. Process B inserts a reservation for the same logical key + // 3. Process A tries to insert - gets UniqueViolationError on (kind, logical_key) + // 4. Process A catches the error, queries again, and returns the reservation from Process B + + // To simulate this, we'll use two concurrent reserve calls + // Due to advisory locks, only one will proceed at a time, but there's still a tiny + // window for race conditions. We'll make this more deterministic by: + // 1. Starting two concurrent reserve operations + // 2. One will succeed and insert the reservation + // 3. The other might hit UniqueViolationError if it checked before the first inserted + // but tries to insert after + + // Start two concurrent reservations for the same logical resource + const promises = [ + catalog.reserve({ + kind: 'vector', + tenantId: 'tenant-1', + bucketName: 'bucket-1', + logicalName: 'index-race', + }), + catalog.reserve({ + kind: 'vector', + tenantId: 'tenant-1', + bucketName: 'bucket-1', + logicalName: 'index-race', + }), + ] + + const results = await Promise.all(promises) + + // Both should succeed and return the same reservation (idempotent) + expect(results[0].reservationId).toBe(results[1].reservationId) + expect(results[0].slotNo).toBe(results[1].slotNo) + expect(results[0].shardKey).toBe(results[1].shardKey) + + // Verify only one reservation was created + const reservations = await db('shard_reservation') + .where({ kind: 'vector', resource_id: 'vector::bucket-1::index-race' }) + .select('*') + + expect(reservations).toHaveLength(1) + }) + }) + + describe('Slot Allocation', () => { + beforeEach(async () => { + await catalog.createShard({ + kind: 'vector', + shardKey: 'test-shard-1', + capacity: 5, + }) + }) + + it('should allocate sequential slot numbers', async () => { + const reservations = [] + for (let i = 0; i < 5; i++) { + const res = await catalog.reserve({ + kind: 'vector', + tenantId: 'tenant-1', + bucketName: 'bucket-1', + logicalName: `index-${i}`, + }) + reservations.push(res) + } + + expect(reservations[0].slotNo).toBe(0) + expect(reservations[1].slotNo).toBe(1) + expect(reservations[2].slotNo).toBe(2) + expect(reservations[3].slotNo).toBe(3) + expect(reservations[4].slotNo).toBe(4) + }) + + it('should reuse freed slots before minting new ones', async () => { + const res1 = await catalog.reserve({ + kind: 'vector', + tenantId: 'tenant-1', + bucketName: 'bucket-1', + logicalName: 'index-1', + }) + + await catalog.confirm(res1.reservationId, { + kind: 'vector', + tenantId: 'tenant-1', + bucketName: 'bucket-1', + logicalName: 'index-1', + }) + + // Free slot 0 + await catalog.freeByLocation(res1.shardId, res1.slotNo) + + // Reserve another - should reuse slot 0 + const res2 = await catalog.reserve({ + kind: 'vector', + tenantId: 'tenant-1', + bucketName: 'bucket-1', + logicalName: 'index-2', + }) + + expect(res2.slotNo).toBe(0) + }) + + it('should throw error when shard is at capacity', async () => { + // Reserve all 5 slots + for (let i = 0; i < 5; i++) { + await catalog.reserve({ + kind: 'vector', + tenantId: 'tenant-1', + bucketName: 'bucket-1', + logicalName: `index-${i}`, + }) + } + + // Try to reserve one more + await expect( + catalog.reserve({ + kind: 'vector', + tenantId: 'tenant-1', + bucketName: 'bucket-1', + logicalName: 'index-6', + }) + ).rejects.toThrow(NoActiveShardError) + }) + + it('should track tenant_id in slots', async () => { + const res = await catalog.reserve({ + kind: 'vector', + tenantId: 'tenant-1', + bucketName: 'bucket-1', + logicalName: 'index-1', + }) + + const slot = await db('shard_slots').where({ slot_no: res.slotNo }).first() + + expect(slot.tenant_id).toBe('tenant-1') + }) + }) + + describe('Lease Expiration', () => { + beforeEach(async () => { + await catalog.createShard({ + kind: 'vector', + shardKey: 'test-shard-1', + capacity: 100, + }) + }) + + it('should expire leases past their expiry time', async () => { + await catalog.reserve({ + kind: 'vector', + tenantId: 'tenant-1', + bucketName: 'bucket-1', + logicalName: 'index-1', + leaseMs: 1, // 1ms lease + }) + + // Wait for lease to expire + await new Promise((resolve) => setTimeout(resolve, 10)) + + const expired = await catalog.expireLeases() + + expect(expired).toBe(1) + + // Check reservation is marked expired + const resv = await db('shard_reservation').first() + expect(resv.status).toBe('expired') + }) + + it('should allow reusing slot after lease expiration', async () => { + const res1 = await catalog.reserve({ + kind: 'vector', + tenantId: 'tenant-1', + bucketName: 'bucket-1', + logicalName: 'index-1', + leaseMs: 1, + }) + + await new Promise((resolve) => setTimeout(resolve, 10)) + await catalog.expireLeases() + + // Reserve with different logical key should reuse the slot + const res2 = await catalog.reserve({ + kind: 'vector', + tenantId: 'tenant-1', + bucketName: 'bucket-1', + logicalName: 'index-2', + }) + + expect(res2.slotNo).toBe(res1.slotNo) + }) + }) + + describe('Shard Selectors', () => { + describe('FillFirstShardSelector', () => { + beforeEach(async () => { + catalog = new ShardCatalog(storeFactory) + + // Create shards with different capacities + await catalog.createShards([ + { kind: 'vector', shardKey: 'shard-1', capacity: 10 }, + { kind: 'vector', shardKey: 'shard-2', capacity: 20 }, + { kind: 'vector', shardKey: 'shard-3', capacity: 30 }, + ]) + }) + + it('should fill shard-1 first (least free capacity)', async () => { + const res = await catalog.reserve({ + kind: 'vector', + tenantId: 'tenant-1', + bucketName: 'bucket-1', + logicalName: 'index-1', + }) + + expect(res.shardKey).toBe('shard-1') + }) + + it('should fill shards sequentially', async () => { + // Fill shard-1 completely (10 slots) + for (let i = 0; i < 10; i++) { + await catalog.reserve({ + kind: 'vector', + tenantId: 'tenant-1', + bucketName: 'bucket-1', + logicalName: `index-${i}`, + }) + } + + // Next reservation should go to shard-2 + const res = await catalog.reserve({ + kind: 'vector', + tenantId: 'tenant-1', + bucketName: 'bucket-1', + logicalName: 'index-11', + }) + + expect(res.shardKey).toBe('shard-2') + }) + }) + }) + + describe('Concurrency Tests', () => { + beforeEach(async () => { + catalog = new ShardCatalog(storeFactory) + + await catalog.createShard({ + kind: 'vector', + shardKey: 'test-shard-1', + capacity: 100, + }) + }) + + it('should handle concurrent reservations without conflicts', async () => { + const promises = [] + for (let i = 0; i < 20; i++) { + promises.push( + catalog.reserve({ + kind: 'vector', + tenantId: `tenant-${i}`, + bucketName: 'bucket-1', + logicalName: `index-${i}`, + }) + ) + } + + const results = await Promise.all(promises) + + // All should succeed + expect(results).toHaveLength(20) + + // All should have unique slot numbers + const slotNumbers = results.map((r) => r.slotNo) + const uniqueSlots = new Set(slotNumbers) + expect(uniqueSlots.size).toBe(20) + }) + + it('should handle concurrent reservations on nearly-full shard', async () => { + // Create a shard with only 5 slots + await db('shard').delete() + await catalog.createShard({ + kind: 'vector', + shardKey: 'small-shard', + capacity: 5, + }) + + // Fill 3 slots + for (let i = 0; i < 3; i++) { + await catalog.reserve({ + kind: 'vector', + tenantId: 'tenant-1', + bucketName: 'bucket-1', + logicalName: `index-${i}`, + }) + } + + // Try to reserve 5 slots concurrently (only 2 should succeed) + const promises = [] + for (let i = 3; i < 8; i++) { + promises.push( + catalog + .reserve({ + kind: 'vector', + tenantId: 'tenant-1', + bucketName: 'bucket-1', + logicalName: `index-${i}`, + }) + .catch((e) => e) + ) + } + + const results = await Promise.all(promises) + + const successes = results.filter((r) => r.reservationId) + const failures = results.filter((r) => r instanceof Error) + + expect(successes.length).toBe(2) + expect(failures.length).toBe(3) + }) + + it('should handle concurrent confirm operations', async () => { + const reservations: { + reservationId: string + shardId: string + shardKey: string + slotNo: number + leaseExpiresAt: string + }[] = [] + for (let i = 0; i < 10; i++) { + const res = await catalog.reserve({ + kind: 'vector', + tenantId: 'tenant-1', + bucketName: 'bucket-1', + logicalName: `index-${i}`, + }) + reservations.push(res) + } + + // Confirm all concurrently + const confirmPromises = reservations.map((res) => + catalog.confirm(res.reservationId, { + kind: 'vector', + tenantId: 'tenant-1', + bucketName: 'bucket-1', + logicalName: `index-${reservations.indexOf(res)}`, + }) + ) + + await Promise.all(confirmPromises) + + // All should be confirmed + const slots = await db('shard_slots').where({ resource_id: null }).count('* as count') + expect(parseInt(slots[0].count as string)).toBe(0) + }) + + it('should handle race condition when selecting same nearly-full shard', async () => { + // Create two shards + await db('shard').delete() + await catalog.createShards([ + { kind: 'vector', shardKey: 'shard-1', capacity: 3 }, + { kind: 'vector', shardKey: 'shard-2', capacity: 100 }, + ]) + + // Fill shard-1 with 2 slots + for (let i = 0; i < 2; i++) { + await catalog.reserve({ + kind: 'vector', + tenantId: 'tenant-1', + bucketName: 'bucket-1', + logicalName: `index-${i}`, + }) + } + + // Try to reserve 5 slots concurrently + // Both processes might initially select shard-1 (has 1 slot free) + // One should get shard-1, others should fall back to shard-2 + const promises = [] + for (let i = 2; i < 7; i++) { + promises.push( + catalog.reserve({ + kind: 'vector', + tenantId: 'tenant-1', + bucketName: 'bucket-1', + logicalName: `index-${i}`, + }) + ) + } + + const results = await Promise.all(promises) + + // All should succeed (no false negatives) + expect(results).toHaveLength(5) + + // Check shard distribution + const shard1Count = results.filter((r) => r.shardKey === 'shard-1').length + const shard2Count = results.filter((r) => r.shardKey === 'shard-2').length + + expect(shard1Count).toBe(1) // Only 1 slot available in shard-1 + expect(shard2Count).toBe(4) // Rest go to shard-2 + }) + }) + + describe('Edge Cases', () => { + it('should throw error when no shards exist', async () => { + await expect( + catalog.reserve({ + kind: 'vector', + tenantId: 'tenant-1', + bucketName: 'bucket-1', + logicalName: 'index-1', + }) + ).rejects.toThrow(NoActiveShardError) + }) + + it('should throw error when all shards are disabled', async () => { + await catalog.createShard({ + kind: 'vector', + shardKey: 'test-shard-1', + capacity: 100, + status: 'disabled', + }) + + await expect( + catalog.reserve({ + kind: 'vector', + tenantId: 'tenant-1', + bucketName: 'bucket-1', + logicalName: 'index-1', + }) + ).rejects.toThrow(NoActiveShardError) + }) + + it('should handle zero-capacity shard', async () => { + await catalog.createShard({ + kind: 'vector', + shardKey: 'zero-capacity', + capacity: 0, + }) + + await expect( + catalog.reserve({ + kind: 'vector', + tenantId: 'tenant-1', + bucketName: 'bucket-1', + logicalName: 'index-1', + }) + ).rejects.toThrow(NoActiveShardError) + }) + + it('should isolate different resource kinds', async () => { + await catalog.createShards([ + { kind: 'vector', shardKey: 'vector-shard', capacity: 10 }, + { kind: 'iceberg', shardKey: 'iceberg-shard', capacity: 10 }, + ]) + + const vectorRes = await catalog.reserve({ + kind: 'vector', + tenantId: 'tenant-1', + bucketName: 'bucket-1', + logicalName: 'index-1', + }) + + const icebergRes = await catalog.reserve({ + kind: 'iceberg', + tenantId: 'tenant-1', + bucketName: 'bucket-1', + logicalName: 'table-1', + }) + + expect(vectorRes.shardKey).toBe('vector-shard') + expect(icebergRes.shardKey).toBe('iceberg-shard') + }) + + it('should find shard by resource id', async () => { + await catalog.createShard({ + kind: 'vector', + shardKey: 'test-shard-1', + capacity: 100, + }) + + const res = await catalog.reserve({ + kind: 'vector', + tenantId: 'tenant-1', + bucketName: 'bucket-1', + logicalName: 'index-1', + }) + + await catalog.confirm(res.reservationId, { + kind: 'vector', + tenantId: 'tenant-1', + bucketName: 'bucket-1', + logicalName: 'index-1', + }) + + const foundShard = await catalog.findShardByResourceId({ + kind: 'vector', + tenantId: 'tenant-1', + bucketName: 'bucket-1', + logicalName: 'index-1', + }) + + expect(foundShard?.shard_key).toBe('test-shard-1') + }) + }) +}) diff --git a/src/test/vectors.test.ts b/src/test/vectors.test.ts index 470213b58..b9bae2afa 100644 --- a/src/test/vectors.test.ts +++ b/src/test/vectors.test.ts @@ -15,8 +15,11 @@ import { import { KnexVectorMetadataDB, VectorStore, VectorStoreManager } from '@storage/protocols/vector' import { useStorage } from './utils/storage' import { signJWT } from '@internal/auth' +import { SingleShard } from '@internal/sharding' -const { serviceKeyAsync, vectorBucketS3, tenantId, jwtSecret } = getConfig() +const { serviceKeyAsync, vectorS3Buckets, tenantId, jwtSecret } = getConfig() + +const vectorBucketS3 = vectorS3Buckets[0] let appInstance: FastifyInstance let serviceToken: string @@ -60,10 +63,13 @@ describe('Vectors API', () => { serviceToken = await serviceKeyAsync // Create real S3Vector instance with mocked client and mock DB + const shard = new SingleShard({ + shardKey: 'test-bucket', + capacity: 1000, + }) const mockVectorDB = new KnexVectorMetadataDB(storageTest.database.connection.pool.acquire()) - s3Vector = new VectorStoreManager(mockVectorStore, mockVectorDB, { + s3Vector = new VectorStoreManager(mockVectorStore, mockVectorDB, shard, { tenantId: 'test-tenant', - vectorBucketName: 'test-bucket', maxBucketCount: Infinity, maxIndexCount: Infinity, }) @@ -88,7 +94,7 @@ describe('Vectors API', () => { await s3Vector.createBucket(vectorBucketName) }) - describe('POST /vectors/CreateIndex', () => { + describe('POST /vector/CreateIndex', () => { let validCreateIndexRequest: { dataType: 'float32' dimension: number @@ -115,7 +121,7 @@ describe('Vectors API', () => { it('should create vector index successfully with valid request', async () => { const response = await appInstance.inject({ method: 'POST', - url: '/vectors/CreateIndex', + url: '/vector/CreateIndex', headers: { authorization: `Bearer ${serviceToken}`, }, @@ -153,7 +159,7 @@ describe('Vectors API', () => { it('should require authentication with service role', async () => { const response = await appInstance.inject({ method: 'POST', - url: '/vectors/CreateIndex', + url: '/vector/CreateIndex', payload: validCreateIndexRequest, }) @@ -166,7 +172,7 @@ describe('Vectors API', () => { const response = await appInstance.inject({ method: 'POST', - url: '/vectors/CreateIndex', + url: '/vector/CreateIndex', headers: { authorization: `Bearer ${invalidToken}`, }, @@ -186,7 +192,7 @@ describe('Vectors API', () => { const response = await appInstance.inject({ method: 'POST', - url: '/vectors/CreateIndex', + url: '/vector/CreateIndex', headers: { authorization: `Bearer ${serviceToken}`, }, @@ -205,7 +211,7 @@ describe('Vectors API', () => { const response = await appInstance.inject({ method: 'POST', - url: '/vectors/CreateIndex', + url: '/vector/CreateIndex', headers: { authorization: `Bearer ${serviceToken}`, }, @@ -224,7 +230,7 @@ describe('Vectors API', () => { const response = await appInstance.inject({ method: 'POST', - url: '/vectors/CreateIndex', + url: '/vector/CreateIndex', headers: { authorization: `Bearer ${serviceToken}`, }, @@ -243,7 +249,7 @@ describe('Vectors API', () => { const response = await appInstance.inject({ method: 'POST', - url: '/vectors/CreateIndex', + url: '/vector/CreateIndex', headers: { authorization: `Bearer ${serviceToken}`, }, @@ -265,7 +271,7 @@ describe('Vectors API', () => { const response = await appInstance.inject({ method: 'POST', - url: '/vectors/CreateIndex', + url: '/vector/CreateIndex', headers: { authorization: `Bearer ${serviceToken}`, }, @@ -283,7 +289,7 @@ describe('Vectors API', () => { const response = await appWithoutVector.inject({ method: 'POST', - url: '/vectors/CreateIndex', + url: '/vector/CreateIndex', headers: { authorization: `Bearer ${serviceToken}`, }, @@ -304,7 +310,7 @@ describe('Vectors API', () => { const response = await appInstance.inject({ method: 'POST', - url: '/vectors/CreateIndex', + url: '/vector/CreateIndex', headers: { authorization: `Bearer ${serviceToken}`, }, @@ -326,7 +332,7 @@ describe('Vectors API', () => { const response = await appInstance.inject({ method: 'POST', - url: '/vectors/CreateIndex', + url: '/vector/CreateIndex', headers: { authorization: `Bearer ${serviceToken}`, }, @@ -343,14 +349,14 @@ describe('Vectors API', () => { }) }) - describe('POST /vectors/CreateVectorBucket', () => { + describe('POST /vector/CreateVectorBucket', () => { beforeEach(async () => {}) it('should create vector bucket successfully with valid request', async () => { const newBucketName = `test-bucket-${Date.now()}-new` const response = await appInstance.inject({ method: 'POST', - url: '/vectors/CreateVectorBucket', + url: '/vector/CreateVectorBucket', headers: { authorization: `Bearer ${serviceToken}`, }, @@ -376,7 +382,7 @@ describe('Vectors API', () => { it('should require authentication with service role', async () => { const response = await appInstance.inject({ method: 'POST', - url: '/vectors/CreateVectorBucket', + url: '/vector/CreateVectorBucket', payload: { vectorBucketName: 'test-bucket', }, @@ -390,7 +396,7 @@ describe('Vectors API', () => { const response = await appInstance.inject({ method: 'POST', - url: '/vectors/CreateVectorBucket', + url: '/vector/CreateVectorBucket', headers: { authorization: `Bearer ${token}`, }, @@ -405,7 +411,7 @@ describe('Vectors API', () => { it('should validate required fields', async () => { const response = await appInstance.inject({ method: 'POST', - url: '/vectors/CreateVectorBucket', + url: '/vector/CreateVectorBucket', headers: { authorization: `Bearer ${serviceToken}`, }, @@ -420,7 +426,7 @@ describe('Vectors API', () => { const newVectorBucketName = `test-bucket-${Date.now()}-dup` const response1 = await appInstance.inject({ method: 'POST', - url: '/vectors/CreateVectorBucket', + url: '/vector/CreateVectorBucket', headers: { authorization: `Bearer ${serviceToken}`, }, @@ -434,7 +440,7 @@ describe('Vectors API', () => { // Second creation should return conflict const response2 = await appInstance.inject({ method: 'POST', - url: '/vectors/CreateVectorBucket', + url: '/vector/CreateVectorBucket', headers: { authorization: `Bearer ${serviceToken}`, }, @@ -447,12 +453,12 @@ describe('Vectors API', () => { }) }) - describe('POST /vectors/DeleteVectorBucket', () => { + describe('POST /vector/DeleteVectorBucket', () => { beforeEach(async () => {}) it('should delete empty vector bucket successfully', async () => { const response = await appInstance.inject({ method: 'POST', - url: '/vectors/DeleteVectorBucket', + url: '/vector/DeleteVectorBucket', headers: { authorization: `Bearer ${serviceToken}`, }, @@ -477,7 +483,7 @@ describe('Vectors API', () => { // First create an index in the bucket await appInstance.inject({ method: 'POST', - url: '/vectors/CreateIndex', + url: '/vector/CreateIndex', headers: { authorization: `Bearer ${serviceToken}`, }, @@ -493,7 +499,7 @@ describe('Vectors API', () => { // Try to delete the bucket const response = await appInstance.inject({ method: 'POST', - url: '/vectors/DeleteVectorBucket', + url: '/vector/DeleteVectorBucket', headers: { authorization: `Bearer ${serviceToken}`, }, @@ -510,7 +516,7 @@ describe('Vectors API', () => { it('should require authentication with service role', async () => { const response = await appInstance.inject({ method: 'POST', - url: '/vectors/DeleteVectorBucket', + url: '/vector/DeleteVectorBucket', payload: { vectorBucketName: vectorBucketName, }, @@ -522,7 +528,7 @@ describe('Vectors API', () => { it('should validate required fields', async () => { const response = await appInstance.inject({ method: 'POST', - url: '/vectors/DeleteVectorBucket', + url: '/vector/DeleteVectorBucket', headers: { authorization: `Bearer ${serviceToken}`, }, @@ -535,7 +541,7 @@ describe('Vectors API', () => { it('should handle non-existent bucket', async () => { const response = await appInstance.inject({ method: 'POST', - url: '/vectors/DeleteVectorBucket', + url: '/vector/DeleteVectorBucket', headers: { authorization: `Bearer ${serviceToken}`, }, @@ -548,7 +554,7 @@ describe('Vectors API', () => { }) }) - describe('POST /vectors/ListVectorBuckets', () => { + describe('POST /vector/ListVectorBuckets', () => { beforeEach(async () => { // Create multiple buckets for listing await s3Vector.createBucket(`test-bucket-a-${Date.now()}`) @@ -559,7 +565,7 @@ describe('Vectors API', () => { it('should list all vector buckets', async () => { const response = await appInstance.inject({ method: 'POST', - url: '/vectors/ListVectorBuckets', + url: '/vector/ListVectorBuckets', headers: { authorization: `Bearer ${serviceToken}`, }, @@ -583,7 +589,7 @@ describe('Vectors API', () => { it('should support maxResults parameter', async () => { const response = await appInstance.inject({ method: 'POST', - url: '/vectors/ListVectorBuckets', + url: '/vector/ListVectorBuckets', headers: { authorization: `Bearer ${serviceToken}`, }, @@ -603,7 +609,7 @@ describe('Vectors API', () => { it('should support pagination with nextToken', async () => { const response1 = await appInstance.inject({ method: 'POST', - url: '/vectors/ListVectorBuckets', + url: '/vector/ListVectorBuckets', headers: { authorization: `Bearer ${serviceToken}`, }, @@ -617,7 +623,7 @@ describe('Vectors API', () => { if (body1.nextToken) { const response2 = await appInstance.inject({ method: 'POST', - url: '/vectors/ListVectorBuckets', + url: '/vector/ListVectorBuckets', headers: { authorization: `Bearer ${serviceToken}`, }, @@ -644,7 +650,7 @@ describe('Vectors API', () => { const prefix = 'test-bucket-a' const response = await appInstance.inject({ method: 'POST', - url: '/vectors/ListVectorBuckets', + url: '/vector/ListVectorBuckets', headers: { authorization: `Bearer ${serviceToken}`, }, @@ -663,7 +669,7 @@ describe('Vectors API', () => { it('should require authentication with service role', async () => { const response = await appInstance.inject({ method: 'POST', - url: '/vectors/ListVectorBuckets', + url: '/vector/ListVectorBuckets', payload: {}, }) @@ -671,11 +677,11 @@ describe('Vectors API', () => { }) }) - describe('POST /vectors/GetVectorBucket', () => { + describe('POST /vector/GetVectorBucket', () => { it('should get vector bucket details successfully', async () => { const response = await appInstance.inject({ method: 'POST', - url: '/vectors/GetVectorBucket', + url: '/vector/GetVectorBucket', headers: { authorization: `Bearer ${serviceToken}`, }, @@ -695,7 +701,7 @@ describe('Vectors API', () => { it('should require authentication with service role', async () => { const response = await appInstance.inject({ method: 'POST', - url: '/vectors/GetVectorBucket', + url: '/vector/GetVectorBucket', payload: { vectorBucketName: vectorBucketName, }, @@ -707,7 +713,7 @@ describe('Vectors API', () => { it('should validate required fields', async () => { const response = await appInstance.inject({ method: 'POST', - url: '/vectors/GetVectorBucket', + url: '/vector/GetVectorBucket', headers: { authorization: `Bearer ${serviceToken}`, }, @@ -720,7 +726,7 @@ describe('Vectors API', () => { it('should handle non-existent bucket', async () => { const response = await appInstance.inject({ method: 'POST', - url: '/vectors/GetVectorBucket', + url: '/vector/GetVectorBucket', headers: { authorization: `Bearer ${serviceToken}`, }, @@ -733,7 +739,7 @@ describe('Vectors API', () => { }) }) - describe('POST /vectors/DeleteIndex', () => { + describe('POST /vector/DeleteIndex', () => { let indexName: string beforeEach(async () => { @@ -755,7 +761,7 @@ describe('Vectors API', () => { it('should delete vector index successfully', async () => { const response = await appInstance.inject({ method: 'POST', - url: '/vectors/DeleteIndex', + url: '/vector/DeleteIndex', headers: { authorization: `Bearer ${serviceToken}`, }, @@ -789,7 +795,7 @@ describe('Vectors API', () => { it('should require authentication with service role', async () => { const response = await appInstance.inject({ method: 'POST', - url: '/vectors/DeleteIndex', + url: '/vector/DeleteIndex', payload: { indexName: indexName, vectorBucketName: vectorBucketName, @@ -802,7 +808,7 @@ describe('Vectors API', () => { it('should validate required fields', async () => { const response = await appInstance.inject({ method: 'POST', - url: '/vectors/DeleteIndex', + url: '/vector/DeleteIndex', headers: { authorization: `Bearer ${serviceToken}`, }, @@ -817,7 +823,7 @@ describe('Vectors API', () => { it('should validate indexName pattern', async () => { const response = await appInstance.inject({ method: 'POST', - url: '/vectors/DeleteIndex', + url: '/vector/DeleteIndex', headers: { authorization: `Bearer ${serviceToken}`, }, @@ -833,7 +839,7 @@ describe('Vectors API', () => { it('should handle non-existent index', async () => { const response = await appInstance.inject({ method: 'POST', - url: '/vectors/DeleteIndex', + url: '/vector/DeleteIndex', headers: { authorization: `Bearer ${serviceToken}`, }, @@ -847,12 +853,12 @@ describe('Vectors API', () => { }) }) - describe('POST /vectors/ListIndexes', () => { + describe('POST /vector/ListIndexes', () => { beforeEach(async () => { // Create multiple indexes for listing await appInstance.inject({ method: 'POST', - url: '/vectors/CreateIndex', + url: '/vector/CreateIndex', headers: { authorization: `Bearer ${serviceToken}`, }, @@ -867,7 +873,7 @@ describe('Vectors API', () => { await appInstance.inject({ method: 'POST', - url: '/vectors/CreateIndex', + url: '/vector/CreateIndex', headers: { authorization: `Bearer ${serviceToken}`, }, @@ -884,7 +890,7 @@ describe('Vectors API', () => { it('should list all indexes in a bucket', async () => { const response = await appInstance.inject({ method: 'POST', - url: '/vectors/ListIndexes', + url: '/vector/ListIndexes', headers: { authorization: `Bearer ${serviceToken}`, }, @@ -911,7 +917,7 @@ describe('Vectors API', () => { it('should support maxResults parameter', async () => { const response = await appInstance.inject({ method: 'POST', - url: '/vectors/ListIndexes', + url: '/vector/ListIndexes', headers: { authorization: `Bearer ${serviceToken}`, }, @@ -930,7 +936,7 @@ describe('Vectors API', () => { const prefix = 'index-a' const response = await appInstance.inject({ method: 'POST', - url: '/vectors/ListIndexes', + url: '/vector/ListIndexes', headers: { authorization: `Bearer ${serviceToken}`, }, @@ -950,7 +956,7 @@ describe('Vectors API', () => { it('should require authentication with service role', async () => { const response = await appInstance.inject({ method: 'POST', - url: '/vectors/ListIndexes', + url: '/vector/ListIndexes', payload: { vectorBucketName: vectorBucketName, }, @@ -962,7 +968,7 @@ describe('Vectors API', () => { it('should validate required fields', async () => { const response = await appInstance.inject({ method: 'POST', - url: '/vectors/ListIndexes', + url: '/vector/ListIndexes', headers: { authorization: `Bearer ${serviceToken}`, }, @@ -973,14 +979,14 @@ describe('Vectors API', () => { }) }) - describe('POST /vectors/GetIndex', () => { + describe('POST /vector/GetIndex', () => { let indexName: string beforeEach(async () => { indexName = `test-index-${Date.now()}` await appInstance.inject({ method: 'POST', - url: '/vectors/CreateIndex', + url: '/vector/CreateIndex', headers: { authorization: `Bearer ${serviceToken}`, }, @@ -1000,7 +1006,7 @@ describe('Vectors API', () => { it('should get index details successfully', async () => { const response = await appInstance.inject({ method: 'POST', - url: '/vectors/GetIndex', + url: '/vector/GetIndex', headers: { authorization: `Bearer ${serviceToken}`, }, @@ -1028,7 +1034,7 @@ describe('Vectors API', () => { it('should require authentication with service role', async () => { const response = await appInstance.inject({ method: 'POST', - url: '/vectors/GetIndex', + url: '/vector/GetIndex', payload: { indexName: indexName, vectorBucketName: vectorBucketName, @@ -1041,7 +1047,7 @@ describe('Vectors API', () => { it('should validate required fields', async () => { const response = await appInstance.inject({ method: 'POST', - url: '/vectors/GetIndex', + url: '/vector/GetIndex', headers: { authorization: `Bearer ${serviceToken}`, }, @@ -1056,7 +1062,7 @@ describe('Vectors API', () => { it('should validate indexName pattern', async () => { const response = await appInstance.inject({ method: 'POST', - url: '/vectors/GetIndex', + url: '/vector/GetIndex', headers: { authorization: `Bearer ${serviceToken}`, }, @@ -1072,7 +1078,7 @@ describe('Vectors API', () => { it('should handle non-existent index', async () => { const response = await appInstance.inject({ method: 'POST', - url: '/vectors/GetIndex', + url: '/vector/GetIndex', headers: { authorization: `Bearer ${serviceToken}`, }, @@ -1086,14 +1092,14 @@ describe('Vectors API', () => { }) }) - describe('POST /vectors/PutVectors', () => { + describe('POST /vector/PutVectors', () => { let indexName: string beforeEach(async () => { indexName = `test-index-${Date.now()}` await appInstance.inject({ method: 'POST', - url: '/vectors/CreateIndex', + url: '/vector/CreateIndex', headers: { authorization: `Bearer ${serviceToken}`, }, @@ -1111,10 +1117,10 @@ describe('Vectors API', () => { } as PutVectorsOutput) }) - it('should put vectors successfully', async () => { + it('should put vector successfully', async () => { const response = await appInstance.inject({ method: 'POST', - url: '/vectors/PutVectors', + url: '/vector/PutVectors', headers: { authorization: `Bearer ${serviceToken}`, }, @@ -1144,7 +1150,6 @@ describe('Vectors API', () => { // Verify putVectors was called with correct parameters expect(mockVectorStore.putVectors).toHaveBeenCalledWith({ - vectorBucketName: vectorBucketS3, indexName: `${tenantId}-${indexName}`, vectors: [ { @@ -1163,17 +1168,18 @@ describe('Vectors API', () => { key: undefined, }, ], + vectorBucketName: vectorBucketS3, }) }) it('should require authentication with service role', async () => { const response = await appInstance.inject({ method: 'POST', - url: '/vectors/PutVectors', + url: '/vector/PutVectors', payload: { vectorBucketName: vectorBucketName, indexName: indexName, - vectors: [ + vector: [ { data: { float32: [1.0, 2.0, 3.0], @@ -1189,7 +1195,7 @@ describe('Vectors API', () => { it('should validate required fields', async () => { const response = await appInstance.inject({ method: 'POST', - url: '/vectors/PutVectors', + url: '/vector/PutVectors', headers: { authorization: `Bearer ${serviceToken}`, }, @@ -1205,14 +1211,14 @@ describe('Vectors API', () => { it('should validate vector data structure', async () => { const response = await appInstance.inject({ method: 'POST', - url: '/vectors/PutVectors', + url: '/vector/PutVectors', headers: { authorization: `Bearer ${serviceToken}`, }, payload: { vectorBucketName: vectorBucketName, indexName: indexName, - vectors: [ + vector: [ { data: { // missing float32 @@ -1234,14 +1240,14 @@ describe('Vectors API', () => { const response = await appInstance.inject({ method: 'POST', - url: '/vectors/PutVectors', + url: '/vector/PutVectors', headers: { authorization: `Bearer ${serviceToken}`, }, payload: { vectorBucketName: vectorBucketName, indexName: indexName, - vectors: tooManyVectors, + vector: tooManyVectors, }, }) @@ -1251,7 +1257,7 @@ describe('Vectors API', () => { it('should handle non-existent index', async () => { const response = await appInstance.inject({ method: 'POST', - url: '/vectors/PutVectors', + url: '/vector/PutVectors', headers: { authorization: `Bearer ${serviceToken}`, }, @@ -1272,14 +1278,14 @@ describe('Vectors API', () => { }) }) - describe('POST /vectors/QueryVectors', () => { + describe('POST /vector/QueryVectors', () => { let indexName: string beforeEach(async () => { indexName = `test-index-${Date.now()}` await appInstance.inject({ method: 'POST', - url: '/vectors/CreateIndex', + url: '/vector/CreateIndex', headers: { authorization: `Bearer ${serviceToken}`, }, @@ -1302,10 +1308,10 @@ describe('Vectors API', () => { } as QueryVectorsOutput) }) - it('should query vectors successfully', async () => { + it('should query vector successfully', async () => { const response = await appInstance.inject({ method: 'POST', - url: '/vectors/QueryVectors', + url: '/vector/QueryVectors', headers: { authorization: `Bearer ${serviceToken}`, }, @@ -1343,7 +1349,7 @@ describe('Vectors API', () => { it('should support metadata filtering', async () => { const response = await appInstance.inject({ method: 'POST', - url: '/vectors/QueryVectors', + url: '/vector/QueryVectors', headers: { authorization: `Bearer ${serviceToken}`, }, @@ -1373,7 +1379,7 @@ describe('Vectors API', () => { it('should support complex logical filters', async () => { const response = await appInstance.inject({ method: 'POST', - url: '/vectors/QueryVectors', + url: '/vector/QueryVectors', headers: { authorization: `Bearer ${serviceToken}`, }, @@ -1403,7 +1409,7 @@ describe('Vectors API', () => { it('should require authentication with service role', async () => { const response = await appInstance.inject({ method: 'POST', - url: '/vectors/QueryVectors', + url: '/vector/QueryVectors', payload: { vectorBucketName: vectorBucketName, indexName: indexName, @@ -1420,7 +1426,7 @@ describe('Vectors API', () => { it('should validate required fields', async () => { const response = await appInstance.inject({ method: 'POST', - url: '/vectors/QueryVectors', + url: '/vector/QueryVectors', headers: { authorization: `Bearer ${serviceToken}`, }, @@ -1439,7 +1445,7 @@ describe('Vectors API', () => { it('should validate queryVector structure', async () => { const response = await appInstance.inject({ method: 'POST', - url: '/vectors/QueryVectors', + url: '/vector/QueryVectors', headers: { authorization: `Bearer ${serviceToken}`, }, @@ -1459,7 +1465,7 @@ describe('Vectors API', () => { it('should handle non-existent index', async () => { const response = await appInstance.inject({ method: 'POST', - url: '/vectors/QueryVectors', + url: '/vector/QueryVectors', headers: { authorization: `Bearer ${serviceToken}`, }, @@ -1477,14 +1483,14 @@ describe('Vectors API', () => { }) }) - describe('POST /vectors/DeleteVectors', () => { + describe('POST /vector/DeleteVectors', () => { let indexName: string beforeEach(async () => { indexName = `test-index-${Date.now()}` await appInstance.inject({ method: 'POST', - url: '/vectors/CreateIndex', + url: '/vector/CreateIndex', headers: { authorization: `Bearer ${serviceToken}`, }, @@ -1500,10 +1506,10 @@ describe('Vectors API', () => { mockVectorStore.deleteVectors.mockResolvedValue({} as DeleteVectorsOutput) }) - it('should delete vectors successfully', async () => { + it('should delete vector successfully', async () => { const response = await appInstance.inject({ method: 'POST', - url: '/vectors/DeleteVectors', + url: '/vector/DeleteVectors', headers: { authorization: `Bearer ${serviceToken}`, }, @@ -1527,7 +1533,7 @@ describe('Vectors API', () => { it('should require authentication with service role', async () => { const response = await appInstance.inject({ method: 'POST', - url: '/vectors/DeleteVectors', + url: '/vector/DeleteVectors', payload: { vectorBucketName: vectorBucketName, indexName: indexName, @@ -1541,7 +1547,7 @@ describe('Vectors API', () => { it('should validate required fields', async () => { const response = await appInstance.inject({ method: 'POST', - url: '/vectors/DeleteVectors', + url: '/vector/DeleteVectors', headers: { authorization: `Bearer ${serviceToken}`, }, @@ -1557,7 +1563,7 @@ describe('Vectors API', () => { it('should handle non-existent index', async () => { const response = await appInstance.inject({ method: 'POST', - url: '/vectors/DeleteVectors', + url: '/vector/DeleteVectors', headers: { authorization: `Bearer ${serviceToken}`, }, @@ -1572,14 +1578,14 @@ describe('Vectors API', () => { }) }) - describe('POST /vectors/ListVectors', () => { + describe('POST /vector/ListVectors', () => { let indexName: string beforeEach(async () => { indexName = `test-index-${Date.now()}` await appInstance.inject({ method: 'POST', - url: '/vectors/CreateIndex', + url: '/vector/CreateIndex', headers: { authorization: `Bearer ${serviceToken}`, }, @@ -1598,10 +1604,10 @@ describe('Vectors API', () => { } as ListVectorsOutput) }) - it('should list vectors successfully', async () => { + it('should list vector successfully', async () => { const response = await appInstance.inject({ method: 'POST', - url: '/vectors/ListVectors', + url: '/vector/ListVectors', headers: { authorization: `Bearer ${serviceToken}`, }, @@ -1628,7 +1634,7 @@ describe('Vectors API', () => { it('should support maxResults parameter', async () => { const response = await appInstance.inject({ method: 'POST', - url: '/vectors/ListVectors', + url: '/vector/ListVectors', headers: { authorization: `Bearer ${serviceToken}`, }, @@ -1650,7 +1656,7 @@ describe('Vectors API', () => { it('should support returnData and returnMetadata flags', async () => { const response = await appInstance.inject({ method: 'POST', - url: '/vectors/ListVectors', + url: '/vector/ListVectors', headers: { authorization: `Bearer ${serviceToken}`, }, @@ -1674,7 +1680,7 @@ describe('Vectors API', () => { it('should support pagination with nextToken', async () => { const response = await appInstance.inject({ method: 'POST', - url: '/vectors/ListVectors', + url: '/vector/ListVectors', headers: { authorization: `Bearer ${serviceToken}`, }, @@ -1696,7 +1702,7 @@ describe('Vectors API', () => { it('should support segmentation parameters', async () => { const response = await appInstance.inject({ method: 'POST', - url: '/vectors/ListVectors', + url: '/vector/ListVectors', headers: { authorization: `Bearer ${serviceToken}`, }, @@ -1720,7 +1726,7 @@ describe('Vectors API', () => { it('should require authentication with service role', async () => { const response = await appInstance.inject({ method: 'POST', - url: '/vectors/ListVectors', + url: '/vector/ListVectors', payload: { vectorBucketName: vectorBucketName, indexName: indexName, @@ -1733,7 +1739,7 @@ describe('Vectors API', () => { it('should validate required fields', async () => { const response = await appInstance.inject({ method: 'POST', - url: '/vectors/ListVectors', + url: '/vector/ListVectors', headers: { authorization: `Bearer ${serviceToken}`, }, @@ -1748,7 +1754,7 @@ describe('Vectors API', () => { it('should validate maxResults range', async () => { const response = await appInstance.inject({ method: 'POST', - url: '/vectors/ListVectors', + url: '/vector/ListVectors', headers: { authorization: `Bearer ${serviceToken}`, }, @@ -1765,7 +1771,7 @@ describe('Vectors API', () => { it('should validate segmentIndex range', async () => { const response = await appInstance.inject({ method: 'POST', - url: '/vectors/ListVectors', + url: '/vector/ListVectors', headers: { authorization: `Bearer ${serviceToken}`, }, @@ -1783,7 +1789,7 @@ describe('Vectors API', () => { it('should handle non-existent index', async () => { const response = await appInstance.inject({ method: 'POST', - url: '/vectors/ListVectors', + url: '/vector/ListVectors', headers: { authorization: `Bearer ${serviceToken}`, }, @@ -1797,14 +1803,14 @@ describe('Vectors API', () => { }) }) - describe('POST /vectors/GetVectors', () => { + describe('POST /vector/GetVectors', () => { let indexName: string beforeEach(async () => { indexName = `test-index-${Date.now()}` await appInstance.inject({ method: 'POST', - url: '/vectors/CreateIndex', + url: '/vector/CreateIndex', headers: { authorization: `Bearer ${serviceToken}`, }, @@ -1836,10 +1842,10 @@ describe('Vectors API', () => { } as GetVectorsCommandOutput) }) - it('should get vectors successfully', async () => { + it('should get vector successfully', async () => { const response = await appInstance.inject({ method: 'POST', - url: '/vectors/GetVectors', + url: '/vector/GetVectors', headers: { authorization: `Bearer ${serviceToken}`, }, @@ -1870,7 +1876,7 @@ describe('Vectors API', () => { it('should work with default returnData and returnMetadata', async () => { const response = await appInstance.inject({ method: 'POST', - url: '/vectors/GetVectors', + url: '/vector/GetVectors', headers: { authorization: `Bearer ${serviceToken}`, }, @@ -1893,7 +1899,7 @@ describe('Vectors API', () => { it('should require authentication with service role', async () => { const response = await appInstance.inject({ method: 'POST', - url: '/vectors/GetVectors', + url: '/vector/GetVectors', payload: { vectorBucketName: vectorBucketName, indexName: indexName, @@ -1907,7 +1913,7 @@ describe('Vectors API', () => { it('should validate required fields', async () => { const response = await appInstance.inject({ method: 'POST', - url: '/vectors/GetVectors', + url: '/vector/GetVectors', headers: { authorization: `Bearer ${serviceToken}`, }, @@ -1923,7 +1929,7 @@ describe('Vectors API', () => { it('should handle non-existent index', async () => { const response = await appInstance.inject({ method: 'POST', - url: '/vectors/GetVectors', + url: '/vector/GetVectors', headers: { authorization: `Bearer ${serviceToken}`, },