From 9d4e00373ec1ec47b73e59e1b6b7a6f9c2cbc055 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mihai=20Capot=C4=83?= Date: Wed, 21 May 2025 09:15:00 -0700 Subject: [PATCH] Record start time before query processing starts Also record end time immediately after obtaining results, before joining the processes. Also remove progress bar because all processes do not report intermediate progress and all processes finish in a similar time. --- engine/base_client/search.py | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/engine/base_client/search.py b/engine/base_client/search.py index 6a512722..c50b3391 100644 --- a/engine/base_client/search.py +++ b/engine/base_client/search.py @@ -163,37 +163,32 @@ def worker_function(chunk, result_queue): # Create a queue to collect results result_queue = Queue() - # Create and start worker processes + # Create worker processes processes = [] for chunk in query_chunks: process = Process(target=worker_function, args=(chunk, result_queue)) processes.append(process) - process.start() # Start measuring time for the critical work start = time.perf_counter() - # Create a progress bar for the total number of queries - pbar = tqdm.tqdm(total=total_query_count, desc="Processing queries", unit="queries") + # Start worker processes + for process in processes: + process.start() # Collect results from all worker processes results = [] for _ in processes: chunk_results = result_queue.get() results.extend(chunk_results) - # Update the progress bar with the number of processed queries in this chunk - pbar.update(len(chunk_results)) - # Close the progress bar - pbar.close() + # Stop measuring time for the critical work + total_time = time.perf_counter() - start # Wait for all worker processes to finish for process in processes: process.join() - # Stop measuring time for the critical work - total_time = time.perf_counter() - start - # Extract precisions and latencies (outside the timed section) precisions, latencies = zip(*results)