Skip to content

refactoring: Standardize format of search params in engine configs #122

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 5 commits into from
Apr 15, 2024
Merged

refactoring: Standardize format of search params in engine configs #122

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
d371150
refactoring: Standardize format of search params in engine configs
tellet-q Apr 12, 2024
ac28d90
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] Apr 12, 2024
8a85f3b
address review and rename params to config
tellet-q Apr 12, 2024
f9c011c
address review
tellet-q Apr 12, 2024
f3e6d9b
fix: upgrade milvusdb image version
tellet-q Apr 12, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion engine/clients/elasticsearch/search.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ def search_one(cls, vector, meta_conditions, top) -> List[Tuple[int, float]]:
"field": "vector",
"query_vector": vector,
"k": top,
**{"num_candidates": 100, **cls.search_params},
**cls.search_params["config"],
}

meta_conditions = cls.parser.parse(meta_conditions)
Expand Down
1 change: 1 addition & 0 deletions engine/clients/milvus/config.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,4 +23,5 @@
DataType.INT64: 0,
DataType.VARCHAR: "---MILVUS DOES NOT ACCEPT EMPTY STRINGS---",
DataType.FLOAT: 0.0,
DataType.DOUBLE: 0.0,
}
2 changes: 1 addition & 1 deletion engine/clients/milvus/search.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ def get_mp_start_method(cls):

@classmethod
def search_one(cls, vector, meta_conditions, top) -> List[Tuple[int, float]]:
param = {"metric_type": cls.distance, "params": cls.search_params["params"]}
param = {"metric_type": cls.distance, "params": cls.search_params["config"]}
try:
res = cls.collection.search(
data=[vector],
Expand Down
2 changes: 1 addition & 1 deletion engine/clients/opensearch/search.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -84,5 +84,5 @@ def search_one(cls, vector, meta_conditions, top) -> List[Tuple[int, float]]:
def setup_search(cls):
if cls.search_params:
cls.client.indices.put_settings(
body=cls.search_params, index=OPENSEARCH_INDEX
body=cls.search_params["config"], index=OPENSEARCH_INDEX
)
4 changes: 1 addition & 3 deletions engine/clients/pgvector/search.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,9 +23,7 @@ def init_client(cls, host, distance, connection_params: dict, search_params: dic
cls.conn = psycopg.connect(**get_db_config(host, connection_params))
register_vector(cls.conn)
cls.cur = cls.conn.cursor()
cls.cur.execute(
f"SET hnsw.ef_search = {search_params['search_params']['hnsw_ef']}"
)
cls.cur.execute(f"SET hnsw.ef_search = {search_params['config']['hnsw_ef']}")
if distance == Distance.COSINE:
cls.query = f"SELECT id, embedding <=> %s AS _score FROM items ORDER BY _score LIMIT %s"
elif distance == Distance.L2:
Expand Down
4 changes: 1 addition & 3 deletions engine/clients/qdrant/search.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,8 +40,6 @@ def search_one(cls, vector, meta_conditions, top) -> List[Tuple[int, float]]:
query_vector=vector,
query_filter=cls.parser.parse(meta_conditions),
limit=top,
search_params=rest.SearchParams(
**cls.search_params.get("search_params", {})
),
search_params=rest.SearchParams(**cls.search_params.get("config", {})),
)
return [(hit.id, hit.score) for hit in res]
2 changes: 1 addition & 1 deletion engine/clients/redis/search.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ def search_one(cls, vector, meta_conditions, top) -> List[Tuple[int, float]]:
params_dict = {
"vec_param": np.array(vector).astype(np.float32).tobytes(),
"K": top,
"EF": cls.search_params["search_params"]["ef"],
**cls.search_params["config"],
**params,
}
results = cls._ft.search(q, query_params=params_dict)
Expand Down
2 changes: 1 addition & 1 deletion engine/clients/weaviate/search.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ def search_one(self, vector, meta_conditions, top) -> List[Tuple[int, float]]:
def setup_search(self):
self.collection.config.update(
vector_index_config=Reconfigure.VectorIndex.hnsw(
ef=self.search_params["vectorIndexConfig"]["ef"]
ef=self.search_params["config"]["ef"]
)
)

Expand Down
2 changes: 1 addition & 1 deletion engine/servers/milvus-limit-ram/docker-compose.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ services:

standalone:
container_name: milvus-standalone
image: milvusdb/milvus:v2.3.0-beta
image: milvusdb/milvus:v2.3.1
command: ["milvus", "run", "standalone"]
environment:
ETCD_ENDPOINTS: etcd:2379
Expand Down
28 changes: 14 additions & 14 deletions experiments/configurations/elasticsearch-single-node.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,8 @@
},
"collection_params": { "index_options": { "m": 16, "ef_construction": 100 } },
"search_params": [
{ "parallel": 1, "num_candidates": 128 }, { "parallel": 1, "num_candidates": 256 }, { "parallel": 1, "num_candidates": 512 },
{ "parallel": 100, "num_candidates": 128 }, { "parallel": 100, "num_candidates": 256 }, { "parallel": 100, "num_candidates": 512 }
{ "parallel": 1, "config": { "num_candidates": 128 } }, { "parallel": 1, "config": { "num_candidates": 256 } }, { "parallel": 1, "config": { "num_candidates": 512 } },
{ "parallel": 100, "config": { "num_candidates": 128 } }, { "parallel": 100, "config": { "num_candidates": 256 } }, { "parallel": 100, "config": { "num_candidates": 512 } }
],
"upload_params": { "parallel": 16 }
},
Expand All @@ -20,8 +20,8 @@
},
"collection_params": { "index_options": { "m": 16, "ef_construction": 128 } },
"search_params": [
{ "parallel": 1, "num_candidates": 128 }, { "parallel": 1, "num_candidates": 256 }, { "parallel": 1, "num_candidates": 512 },
{ "parallel": 100, "num_candidates": 128 }, { "parallel": 100, "num_candidates": 256 }, { "parallel": 100, "num_candidates": 512 }
{ "parallel": 1, "config": { "num_candidates": 128 } }, { "parallel": 1, "config": { "num_candidates": 256 } }, { "parallel": 1, "config": { "num_candidates": 512 } },
{ "parallel": 100, "config": { "num_candidates": 128 } }, { "parallel": 100, "config": { "num_candidates": 256 } }, { "parallel": 100, "config": { "num_candidates": 512 } }
],
"upload_params": { "parallel": 16 }
},
Expand All @@ -33,8 +33,8 @@
},
"collection_params": { "index_options": { "m": 32, "ef_construction": 128 } },
"search_params": [
{ "parallel": 1, "num_candidates": 128 }, { "parallel": 1, "num_candidates": 256 }, { "parallel": 1, "num_candidates": 512 },
{ "parallel": 100, "num_candidates": 128 }, { "parallel": 100, "num_candidates": 256 }, { "parallel": 100, "num_candidates": 512 }
{ "parallel": 1, "config": { "num_candidates": 128 } }, { "parallel": 1, "config": { "num_candidates": 256 } }, { "parallel": 1, "config": { "num_candidates": 512 } },
{ "parallel": 100, "config": { "num_candidates": 128 } }, { "parallel": 100, "config": { "num_candidates": 256 } }, { "parallel": 100, "config": { "num_candidates": 512 } }
],
"upload_params": { "parallel": 16 }
},
Expand All @@ -46,8 +46,8 @@
},
"collection_params": { "index_options": { "m": 32, "ef_construction": 256 } },
"search_params": [
{ "parallel": 1, "num_candidates": 128 }, { "parallel": 1, "num_candidates": 256 }, { "parallel": 1, "num_candidates": 512 },
{ "parallel": 100, "num_candidates": 128 }, { "parallel": 100, "num_candidates": 256 }, { "parallel": 100, "num_candidates": 512 }
{ "parallel": 1, "config": { "num_candidates": 128 } }, { "parallel": 1, "config": { "num_candidates": 256 } }, { "parallel": 1, "config": { "num_candidates": 512 } },
{ "parallel": 100, "config": { "num_candidates": 128 } }, { "parallel": 100, "config": { "num_candidates": 256 } }, { "parallel": 100, "config": { "num_candidates": 512 } }
],
"upload_params": { "parallel": 16 }
},
Expand All @@ -59,8 +59,8 @@
},
"collection_params": { "index_options": { "m": 32, "ef_construction": 512 } },
"search_params": [
{ "parallel": 1, "num_candidates": 128 }, { "parallel": 1, "num_candidates": 256 }, { "parallel": 1, "num_candidates": 512 },
{ "parallel": 100, "num_candidates": 128 }, { "parallel": 100, "num_candidates": 256 }, { "parallel": 100, "num_candidates": 512 }
{ "parallel": 1, "config": { "num_candidates": 128 } }, { "parallel": 1, "config": { "num_candidates": 256 } }, { "parallel": 1, "config": { "num_candidates": 512 } },
{ "parallel": 100, "config": { "num_candidates": 128 } }, { "parallel": 100, "config": { "num_candidates": 256 } }, { "parallel": 100, "config": { "num_candidates": 512 } }
],
"upload_params": { "parallel": 16 }
},
Expand All @@ -72,8 +72,8 @@
},
"collection_params": { "index_options": { "m": 64, "ef_construction": 256 } },
"search_params": [
{ "parallel": 1, "num_candidates": 128 }, { "parallel": 1, "num_candidates": 256 }, { "parallel": 1, "num_candidates": 512 },
{ "parallel": 100, "num_candidates": 128 }, { "parallel": 100, "num_candidates": 256 }, { "parallel": 100, "num_candidates": 512 }
{ "parallel": 1, "config": { "num_candidates": 128 } }, { "parallel": 1, "config": { "num_candidates": 256 } }, { "parallel": 1, "config": { "num_candidates": 512 } },
{ "parallel": 100, "config": { "num_candidates": 128 } }, { "parallel": 100, "config": { "num_candidates": 256 } }, { "parallel": 100, "config": { "num_candidates": 512 } }
],
"upload_params": { "parallel": 16 }
},
Expand All @@ -85,8 +85,8 @@
},
"collection_params": { "index_options": { "m": 64, "ef_construction": 512 } },
"search_params": [
{ "parallel": 1, "num_candidates": 128 }, { "parallel": 1, "num_candidates": 256 }, { "parallel": 1, "num_candidates": 512 },
{ "parallel": 100, "num_candidates": 128 }, { "parallel": 100, "num_candidates": 256 }, { "parallel": 100, "num_candidates": 512 }
{ "parallel": 1, "config": { "num_candidates": 128 } }, { "parallel": 1, "config": { "num_candidates": 256 } }, { "parallel": 1, "config": { "num_candidates": 512 } },
{ "parallel": 100, "config": { "num_candidates": 128 } }, { "parallel": 100, "config": { "num_candidates": 256 } }, { "parallel": 100, "config": { "num_candidates": 512 } }
],
"upload_params": { "parallel": 16 }
}
Expand Down
2 changes: 1 addition & 1 deletion experiments/configurations/milvus-on-disk.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
"connection_params": {},
"collection_params": {},
"search_params": [
{ "parallel": 8, "params": { } }
{ "parallel": 8, "config": { } }
],
"upload_params": { "parallel": 4, "index_type": "DISKANN", "index_params": { } }
}
Expand Down
28 changes: 14 additions & 14 deletions experiments/configurations/milvus-single-node.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,8 @@
"connection_params": {},
"collection_params": {},
"search_params": [
{ "parallel": 1, "params": { "ef": 128 } }, { "parallel": 1, "params": { "ef": 256 } }, { "parallel": 1, "params": { "ef": 512 } },
{ "parallel": 100, "params": { "ef": 128 } }, { "parallel": 100, "params": { "ef": 256 } }, { "parallel": 100, "params": { "ef": 512 } }
{ "parallel": 1, "config": { "ef": 128 } }, { "parallel": 1, "config": { "ef": 256 } }, { "parallel": 1, "config": { "ef": 512 } },
{ "parallel": 100, "config": { "ef": 128 } }, { "parallel": 100, "config": { "ef": 256 } }, { "parallel": 100, "config": { "ef": 512 } }
],
"upload_params": { "parallel": 16, "index_params": { "efConstruction": 100, "M": 16 } }
},
Expand All @@ -16,8 +16,8 @@
"connection_params": {},
"collection_params": {},
"search_params": [
{ "parallel": 1, "params": { "ef": 128 } }, { "parallel": 1, "params": { "ef": 256 } }, { "parallel": 1, "params": { "ef": 512 } },
{ "parallel": 100, "params": { "ef": 128 } }, { "parallel": 100, "params": { "ef": 256 } }, { "parallel": 100, "params": { "ef": 512 } }
{ "parallel": 1, "config": { "ef": 128 } }, { "parallel": 1, "config": { "ef": 256 } }, { "parallel": 1, "config": { "ef": 512 } },
{ "parallel": 100, "config": { "ef": 128 } }, { "parallel": 100, "config": { "ef": 256 } }, { "parallel": 100, "config": { "ef": 512 } }
],
"upload_params": { "parallel": 16, "index_params": { "efConstruction": 128, "M": 16 } }
},
Expand All @@ -27,8 +27,8 @@
"connection_params": {},
"collection_params": {},
"search_params": [
{ "parallel": 1, "params": { "ef": 128 } }, { "parallel": 1, "params": { "ef": 256 } }, { "parallel": 1, "params": { "ef": 512 } },
{ "parallel": 100, "params": { "ef": 128 } }, { "parallel": 100, "params": { "ef": 256 } }, { "parallel": 100, "params": { "ef": 512 } }
{ "parallel": 1, "config": { "ef": 128 } }, { "parallel": 1, "config": { "ef": 256 } }, { "parallel": 1, "config": { "ef": 512 } },
{ "parallel": 100, "config": { "ef": 128 } }, { "parallel": 100, "config": { "ef": 256 } }, { "parallel": 100, "config": { "ef": 512 } }
],
"upload_params": { "parallel": 16, "index_params": { "efConstruction": 128, "M": 32 } }
},
Expand All @@ -38,8 +38,8 @@
"connection_params": {},
"collection_params": {},
"search_params": [
{ "parallel": 1, "params": { "ef": 128 } }, { "parallel": 1, "params": { "ef": 256 } }, { "parallel": 1, "params": { "ef": 512 } },
{ "parallel": 100, "params": { "ef": 128 } }, { "parallel": 100, "params": { "ef": 256 } }, { "parallel": 100, "params": { "ef": 512 } }
{ "parallel": 1, "config": { "ef": 128 } }, { "parallel": 1, "config": { "ef": 256 } }, { "parallel": 1, "config": { "ef": 512 } },
{ "parallel": 100, "config": { "ef": 128 } }, { "parallel": 100, "config": { "ef": 256 } }, { "parallel": 100, "config": { "ef": 512 } }
],
"upload_params": { "parallel": 16, "index_params": { "efConstruction": 256, "M": 32 } }
},
Expand All @@ -49,8 +49,8 @@
"connection_params": {},
"collection_params": {},
"search_params": [
{ "parallel": 1, "params": { "ef": 128 } }, { "parallel": 1, "params": { "ef": 256 } }, { "parallel": 1, "params": { "ef": 512 } },
{ "parallel": 100, "params": { "ef": 128 } }, { "parallel": 100, "params": { "ef": 256 } }, { "parallel": 100, "params": { "ef": 512 } }
{ "parallel": 1, "config": { "ef": 128 } }, { "parallel": 1, "config": { "ef": 256 } }, { "parallel": 1, "config": { "ef": 512 } },
{ "parallel": 100, "config": { "ef": 128 } }, { "parallel": 100, "config": { "ef": 256 } }, { "parallel": 100, "config": { "ef": 512 } }
],
"upload_params": { "parallel": 16, "index_params": { "efConstruction": 512, "M": 32 } }
},
Expand All @@ -60,8 +60,8 @@
"connection_params": {},
"collection_params": {},
"search_params": [
{ "parallel": 1, "params": { "ef": 128 } }, { "parallel": 1, "params": { "ef": 256 } }, { "parallel": 1, "params": { "ef": 512 } },
{ "parallel": 100, "params": { "ef": 128 } }, { "parallel": 100, "params": { "ef": 256 } }, { "parallel": 100, "params": { "ef": 512 } }
{ "parallel": 1, "config": { "ef": 128 } }, { "parallel": 1, "config": { "ef": 256 } }, { "parallel": 1, "config": { "ef": 512 } },
{ "parallel": 100, "config": { "ef": 128 } }, { "parallel": 100, "config": { "ef": 256 } }, { "parallel": 100, "config": { "ef": 512 } }
],
"upload_params": { "parallel": 16, "index_params": { "efConstruction": 256, "M": 64 } }
},
Expand All @@ -71,8 +71,8 @@
"connection_params": {},
"collection_params": {},
"search_params": [
{ "parallel": 1, "params": { "ef": 128 } }, { "parallel": 1, "params": { "ef": 256 } }, { "parallel": 1, "params": { "ef": 512 } },
{ "parallel": 100, "params": { "ef": 128 } }, { "parallel": 100, "params": { "ef": 256 } }, { "parallel": 100, "params": { "ef": 512 } }
{ "parallel": 1, "config": { "ef": 128 } }, { "parallel": 1, "config": { "ef": 256 } }, { "parallel": 1, "config": { "ef": 512 } },
{ "parallel": 100, "config": { "ef": 128 } }, { "parallel": 100, "config": { "ef": 256 } }, { "parallel": 100, "config": { "ef": 512 } }
],
"upload_params": { "parallel": 16, "index_params": { "efConstruction": 512, "M": 64 } }
}
Expand Down
Loading