Skip to content

Add support svs vamana index creation #3025

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ jobs:
fail-fast: false
matrix:
node-version: ["18", "20", "22"]
redis-version: ["rs-7.4.0-v1", "8.0.2", "8.2-rc1"]
redis-version: ["rs-7.4.0-v1", "8.0.2", "8.2-rc2-pre"]
steps:
- uses: actions/checkout@v4
with:
Expand Down
2 changes: 1 addition & 1 deletion packages/bloom/lib/test-utils.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ import RedisBloomModules from '.';
export default TestUtils.createFromConfig({
dockerImageName: 'redislabs/client-libs-test',
dockerImageVersionArgument: 'redis-version',
defaultDockerVersion: '8.2-rc1'
defaultDockerVersion: '8.2-rc2-pre'
});

export const GLOBAL = {
Expand Down
2 changes: 1 addition & 1 deletion packages/client/lib/sentinel/test-util.ts
Original file line number Diff line number Diff line change
Expand Up @@ -174,7 +174,7 @@ export class SentinelFramework extends DockerBase {
this.#testUtils = TestUtils.createFromConfig({
dockerImageName: 'redislabs/client-libs-test',
dockerImageVersionArgument: 'redis-version',
defaultDockerVersion: '8.2-rc1'
defaultDockerVersion: '8.2-rc2-pre'
});
this.#nodeMap = new Map<string, ArrayElement<Awaited<ReturnType<SentinelFramework['spawnRedisSentinelNodes']>>>>();
this.#sentinelMap = new Map<string, ArrayElement<Awaited<ReturnType<SentinelFramework['spawnRedisSentinelSentinels']>>>>();
Expand Down
2 changes: 1 addition & 1 deletion packages/client/lib/test-utils.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ import RedisBloomModules from '@redis/bloom';
const utils = TestUtils.createFromConfig({
dockerImageName: 'redislabs/client-libs-test',
dockerImageVersionArgument: 'redis-version',
defaultDockerVersion: '8.2-rc1'
defaultDockerVersion: '8.2-rc2-pre'
});

export default utils;
Expand Down
2 changes: 1 addition & 1 deletion packages/entraid/lib/test-utils.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ import { EntraidCredentialsProvider } from './entraid-credentials-provider';
export const testUtils = TestUtils.createFromConfig({
dockerImageName: 'redislabs/client-libs-test',
dockerImageVersionArgument: 'redis-version',
defaultDockerVersion: '8.2-rc1'
defaultDockerVersion: '8.2-rc2-pre'
});

const DEBUG_MODE_ARGS = testUtils.isVersionGreaterThan([7]) ?
Expand Down
2 changes: 1 addition & 1 deletion packages/json/lib/test-utils.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ import RedisJSON from '.';
export default TestUtils.createFromConfig({
dockerImageName: 'redislabs/client-libs-test',
dockerImageVersionArgument: 'redis-version',
defaultDockerVersion: '8.2-rc1'
defaultDockerVersion: '8.2-rc2-pre'
});

export const GLOBAL = {
Expand Down
112 changes: 111 additions & 1 deletion packages/search/lib/commands/CREATE.spec.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import { strict as assert } from 'node:assert';
import testUtils, { GLOBAL } from '../test-utils';
import CREATE, { SCHEMA_FIELD_TYPE, SCHEMA_TEXT_FIELD_PHONETIC, SCHEMA_VECTOR_FIELD_ALGORITHM, REDISEARCH_LANGUAGE } from './CREATE';
import CREATE, { SCHEMA_FIELD_TYPE, SCHEMA_TEXT_FIELD_PHONETIC, SCHEMA_VECTOR_FIELD_ALGORITHM, REDISEARCH_LANGUAGE, VAMANA_COMPRESSION_ALGORITHM } from './CREATE';
import { parseArgs } from '@redis/client/lib/commands/generic-transformers';

describe('FT.CREATE', () => {
Expand Down Expand Up @@ -206,6 +206,33 @@ describe('FT.CREATE', () => {
]
);
});

it('VAMANA algorithm', () => {
assert.deepEqual(
parseArgs(CREATE, 'index', {
field: {
type: SCHEMA_FIELD_TYPE.VECTOR,
ALGORITHM: SCHEMA_VECTOR_FIELD_ALGORITHM.VAMANA,
TYPE: "FLOAT32",
COMPRESSION: VAMANA_COMPRESSION_ALGORITHM.LVQ8,
DIM: 1024,
DISTANCE_METRIC: 'COSINE',
CONSTRUCTION_WINDOW_SIZE: 300,
GRAPH_MAX_DEGREE: 128,
SEARCH_WINDOW_SIZE: 20,
EPSILON: 0.02,
TRAINING_THRESHOLD: 20480,
REDUCE: 512,
}
}),
[
'FT.CREATE', 'index', 'SCHEMA', 'field', 'VECTOR', 'SVS-VAMANA', '20', 'TYPE',
'FLOAT32', 'DIM', '1024', 'DISTANCE_METRIC', 'COSINE', 'COMPRESSION', 'LVQ8',
'CONSTRUCTION_WINDOW_SIZE', '300', 'GRAPH_MAX_DEGREE', '128', 'SEARCH_WINDOW_SIZE', '20',
'EPSILON', '0.02', 'TRAINING_THRESHOLD', '20480', 'REDUCE', '512'
]
);
});
});

describe('GEOSHAPE', () => {
Expand Down Expand Up @@ -556,4 +583,87 @@ describe('FT.CREATE', () => {
"OK"
);
}, GLOBAL.SERVERS.OPEN);

testUtils.testWithClientIfVersionWithinRange([[8, 2], 'LATEST'], 'client.ft.create vector svs-vamana', async client => {
assert.equal(
await client.ft.create("index_svs_vamana_min_config", {
field: {
type: SCHEMA_FIELD_TYPE.VECTOR,
ALGORITHM: SCHEMA_VECTOR_FIELD_ALGORITHM.VAMANA,
TYPE: "FLOAT32",
DIM: 768,
DISTANCE_METRIC: 'L2',
},
}),
"OK"
);

assert.equal(
await client.ft.create("index_svs_vamana_no_compression", {
field: {
type: SCHEMA_FIELD_TYPE.VECTOR,
ALGORITHM: SCHEMA_VECTOR_FIELD_ALGORITHM.VAMANA,
TYPE: "FLOAT32",
DIM: 512,
DISTANCE_METRIC: 'L2',
CONSTRUCTION_WINDOW_SIZE: 200,
GRAPH_MAX_DEGREE: 64,
SEARCH_WINDOW_SIZE: 50,
EPSILON: 0.01
},
}),
"OK"
);

assert.equal(
await client.ft.create("index_svs_vamana_compression", {
field: {
type: SCHEMA_FIELD_TYPE.VECTOR,
ALGORITHM: SCHEMA_VECTOR_FIELD_ALGORITHM.VAMANA,
TYPE: "FLOAT32",
COMPRESSION: VAMANA_COMPRESSION_ALGORITHM.LeanVec4x8,
DIM: 1024,
DISTANCE_METRIC: 'COSINE',
CONSTRUCTION_WINDOW_SIZE: 300,
GRAPH_MAX_DEGREE: 128,
SEARCH_WINDOW_SIZE: 20,
EPSILON: 0.02,
TRAINING_THRESHOLD: 20480,
REDUCE: 512,
},
}),
"OK"
);

assert.equal(
await client.ft.create("index_svs_vamana_float16", {
field: {
type: SCHEMA_FIELD_TYPE.VECTOR,
ALGORITHM: SCHEMA_VECTOR_FIELD_ALGORITHM.VAMANA,
TYPE: "FLOAT16",
DIM: 128,
DISTANCE_METRIC: 'IP',
},
}),
"OK"
);

await assert.rejects(
client.ft.create("index_svs_vamana_invalid_config", {
field: {
type: SCHEMA_FIELD_TYPE.VECTOR,
ALGORITHM: SCHEMA_VECTOR_FIELD_ALGORITHM.VAMANA,
TYPE: "FLOAT32",
DIM: 2,
DISTANCE_METRIC: 'L2',
CONSTRUCTION_WINDOW_SIZE: 200,
GRAPH_MAX_DEGREE: 64,
SEARCH_WINDOW_SIZE: 50,
EPSILON: 0.01,
// TRAINING_THRESHOLD should error without COMPRESSION
TRAINING_THRESHOLD: 2048
},
}),
)
}, GLOBAL.SERVERS.OPEN);
});
81 changes: 74 additions & 7 deletions packages/search/lib/commands/CREATE.ts
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,11 @@ interface SchemaTagField extends SchemaCommonField<typeof SCHEMA_FIELD_TYPE['TAG

export const SCHEMA_VECTOR_FIELD_ALGORITHM = {
FLAT: 'FLAT',
HNSW: 'HNSW'
HNSW: 'HNSW',
/**
* available since 8.2
*/
VAMANA: 'SVS-VAMANA'
} as const;

export type SchemaVectorFieldAlgorithm = typeof SCHEMA_VECTOR_FIELD_ALGORITHM[keyof typeof SCHEMA_VECTOR_FIELD_ALGORITHM];
Expand All @@ -79,6 +83,37 @@ interface SchemaHNSWVectorField extends SchemaVectorField {
EF_RUNTIME?: number;
}

export const VAMANA_COMPRESSION_ALGORITHM = {
LVQ4: 'LVQ4',
LVQ8: 'LVQ8',
LVQ4x4: 'LVQ4x4',
LVQ4x8: 'LVQ4x8',
LeanVec4x8: 'LeanVec4x8',
LeanVec8x8: 'LeanVec8x8'
} as const;

export type VamanaCompressionAlgorithm =
typeof VAMANA_COMPRESSION_ALGORITHM[keyof typeof VAMANA_COMPRESSION_ALGORITHM];

interface SchemaVAMANAVectorField extends SchemaVectorField {
ALGORITHM: typeof SCHEMA_VECTOR_FIELD_ALGORITHM['VAMANA'];
TYPE: 'FLOAT16' | 'FLOAT32';
// VAMANA-specific parameters
COMPRESSION?: VamanaCompressionAlgorithm;
CONSTRUCTION_WINDOW_SIZE?: number;
GRAPH_MAX_DEGREE?: number;
SEARCH_WINDOW_SIZE?: number;
EPSILON?: number;
/**
* applicable only with COMPRESSION
*/
TRAINING_THRESHOLD?: number;
/**
* applicable only with LeanVec COMPRESSION
*/
REDUCE?: number;
}

export const SCHEMA_GEO_SHAPE_COORD_SYSTEM = {
SPHERICAL: 'SPHERICAL',
FLAT: 'FLAT'
Expand All @@ -98,6 +133,7 @@ export interface RediSearchSchema {
SchemaTagField |
SchemaFlatVectorField |
SchemaHNSWVectorField |
SchemaVAMANAVectorField |
SchemaGeoShapeField |
SchemaFieldType
);
Expand Down Expand Up @@ -142,7 +178,7 @@ export function parseSchema(parser: CommandParser, schema: RediSearchSchema) {
parser.push('NOSTEM');
}

if (fieldOptions.WEIGHT) {
if (fieldOptions.WEIGHT !== undefined) {
parser.push('WEIGHT', fieldOptions.WEIGHT.toString());
}

Expand Down Expand Up @@ -197,31 +233,62 @@ export function parseSchema(parser: CommandParser, schema: RediSearchSchema) {
'DISTANCE_METRIC', fieldOptions.DISTANCE_METRIC
);

if (fieldOptions.INITIAL_CAP) {
if (fieldOptions.INITIAL_CAP !== undefined) {
args.push('INITIAL_CAP', fieldOptions.INITIAL_CAP.toString());
}

switch (fieldOptions.ALGORITHM) {
case SCHEMA_VECTOR_FIELD_ALGORITHM.FLAT:
if (fieldOptions.BLOCK_SIZE) {
if (fieldOptions.BLOCK_SIZE !== undefined) {
args.push('BLOCK_SIZE', fieldOptions.BLOCK_SIZE.toString());
}

break;

case SCHEMA_VECTOR_FIELD_ALGORITHM.HNSW:
if (fieldOptions.M) {
if (fieldOptions.M !== undefined) {
args.push('M', fieldOptions.M.toString());
}

if (fieldOptions.EF_CONSTRUCTION) {
if (fieldOptions.EF_CONSTRUCTION !== undefined) {
args.push('EF_CONSTRUCTION', fieldOptions.EF_CONSTRUCTION.toString());
}

if (fieldOptions.EF_RUNTIME) {
if (fieldOptions.EF_RUNTIME !== undefined) {
args.push('EF_RUNTIME', fieldOptions.EF_RUNTIME.toString());
}

break;

case SCHEMA_VECTOR_FIELD_ALGORITHM['VAMANA']:
if (fieldOptions.COMPRESSION) {
args.push('COMPRESSION', fieldOptions.COMPRESSION);
}

if (fieldOptions.CONSTRUCTION_WINDOW_SIZE !== undefined) {
args.push('CONSTRUCTION_WINDOW_SIZE', fieldOptions.CONSTRUCTION_WINDOW_SIZE.toString());
}

if (fieldOptions.GRAPH_MAX_DEGREE !== undefined) {
args.push('GRAPH_MAX_DEGREE', fieldOptions.GRAPH_MAX_DEGREE.toString());
}

if (fieldOptions.SEARCH_WINDOW_SIZE !== undefined) {
args.push('SEARCH_WINDOW_SIZE', fieldOptions.SEARCH_WINDOW_SIZE.toString());
}

if (fieldOptions.EPSILON !== undefined) {
args.push('EPSILON', fieldOptions.EPSILON.toString());
}

if (fieldOptions.TRAINING_THRESHOLD !== undefined) {
args.push('TRAINING_THRESHOLD', fieldOptions.TRAINING_THRESHOLD.toString());
}

if (fieldOptions.REDUCE !== undefined) {
args.push('REDUCE', fieldOptions.REDUCE.toString());
}

break;
}
parser.pushVariadicWithLength(args);
Expand Down
2 changes: 1 addition & 1 deletion packages/search/lib/test-utils.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ import { RespVersions } from '@redis/client';
export default TestUtils.createFromConfig({
dockerImageName: 'redislabs/client-libs-test',
dockerImageVersionArgument: 'redis-version',
defaultDockerVersion: '8.2-rc1'
defaultDockerVersion: '8.2-rc2-pre'
});

export const GLOBAL = {
Expand Down
2 changes: 1 addition & 1 deletion packages/time-series/lib/test-utils.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ import TimeSeries from '.';
export default TestUtils.createFromConfig({
dockerImageName: 'redislabs/client-libs-test',
dockerImageVersionArgument: 'redis-version',
defaultDockerVersion: '8.2-rc1'
defaultDockerVersion: '8.2-rc2-pre'
});

export const GLOBAL = {
Expand Down
Loading