11name : Manual Benchmark to compare versions
2+ description : |
3+ This workflow is used to compare two versions of qdrant using the same dataset and engine config.
4+ It is triggered manually and requires the user to provide the versions of qdrant to compare, dataset and engine config.
5+ The workflow will prepare the images for the provided versions (if needed), run the benchmark for each version and compare the results.
26
37on :
48 workflow_dispatch :
2024 prepareImage1 :
2125 name : Prepare image ${{ inputs.qdrant_version_1 }}
2226 runs-on : ubuntu-latest
27+ timeout-minutes : 180
2328 steps :
2429 - uses : actions/checkout@v3
2530 -
uses :
webfactory/[email protected] @@ -29,12 +34,13 @@ jobs:
2934 run : |
3035 # The branch, tag or SHA to checkout.
3136 export QDRANT_VERSION=${{ inputs.qdrant_version_1 }}
32- export BEARER_TOKEN="${{ secrets.GITHUB_TOKEN }}"
37+ export BEARER_TOKEN="${{ secrets.TRIGGER_GH_TOKEN }}"
3338 bash -x tools/compare_versions/prepare_image.sh
3439
3540 prepareImage2 :
36- name : Prepare image ${{ inputs.qdrant_version_1 }}
41+ name : Prepare image ${{ inputs.qdrant_version_2 }}
3742 runs-on : ubuntu-latest
43+ timeout-minutes : 180
3844 steps :
3945 - uses : actions/checkout@v3
4046 -
uses :
webfactory/[email protected] @@ -43,50 +49,126 @@ jobs:
4349 - name : Image for ${{ inputs.qdrant_version_2 }}
4450 run : |
4551 export QDRANT_VERSION=${{ inputs.qdrant_version_2 }}
46- export BEARER_TOKEN="${{ secrets.GITHUB_TOKEN }}"
52+ export BEARER_TOKEN="${{ secrets.TRIGGER_GH_TOKEN }}"
4753 bash -x tools/compare_versions/prepare_image.sh
4854
49- # runBenchmarkForVersion1:
50- # name: compare - ${{ inputs.qdrant_version_1 }} vs ${{ inputs.qdrant_version_2 }} - ${{ inputs.dataset }}
51- # needs:
52- # - prepareImage1
53- # - prepareImage2
54- # runs-on: ubuntu-latest
55- # steps:
56- # - uses: actions/checkout@v3
57- # - uses: webfactory/[email protected] 58- # with:
59- # ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }}
60- # - name: Bench ${{ inputs.qdrant_version_1 }}
61- # run: |
62- # export HCLOUD_TOKEN=${{ secrets.HCLOUD_TOKEN }}
63- # export POSTGRES_PASSWORD=${{ secrets.POSTGRES_PASSWORD }}
64- # export POSTGRES_HOST=${{ secrets.POSTGRES_HOST }}
65- # export QDRANT_VERSION=${{ inputs.qdrant_version_1 }}
66- # export DATASETS=${{ inputs.dataset }}
67- # export ENGINE_NAME=${{ inputs.engine_config }}
68- # export POSTGRES_TABLE=benchmark_manual
69- # bash -x tools/setup_ci.sh
70- # bash -x tools/run_ci.sh
71- #
72- # runBenchmarkForVersion2:
73- # name: compare - ${{ inputs.qdrant_version_1 }} vs ${{ inputs.qdrant_version_2 }} - ${{ inputs.dataset }}
74- # needs:
75- # - runBenchmarkForVersion1
76- # runs-on: ubuntu-latest
77- # steps:
78- # - uses: actions/checkout@v3
79- # - uses: webfactory/[email protected] 80- # with:
81- # ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }}
82- # - name: Bench ${{ inputs.qdrant_version_2 }}
83- # run: |
84- # export HCLOUD_TOKEN=${{ secrets.HCLOUD_TOKEN }}
85- # export POSTGRES_PASSWORD=${{ secrets.POSTGRES_PASSWORD }}
86- # export POSTGRES_HOST=${{ secrets.POSTGRES_HOST }}
87- # export QDRANT_VERSION=${{ inputs.qdrant_version_2 }}
88- # export DATASETS=${{ inputs.dataset }}
89- # export ENGINE_NAME=${{ inputs.engine_config }}
90- # export POSTGRES_TABLE=benchmark_manual
91- # bash -x tools/setup_ci.sh
92- # bash -x tools/run_ci.sh
55+ runBenchmarkForVersion1 :
56+ name : execute - ${{ inputs.qdrant_version_1 }} - ${{ inputs.dataset }}
57+ needs :
58+ - prepareImage1
59+ runs-on : ubuntu-latest
60+ concurrency :
61+ group : continuous-benchmark
62+ outputs :
63+ collection_load_time : ${{ steps.bench.outputs.collection_load_time }}
64+ rps : ${{ steps.bench.outputs.rps }}
65+ mean_precisions : ${{ steps.bench.outputs.mean_precisions }}
66+ p95_time : ${{ steps.bench.outputs.p95_time }}
67+ p99_time : ${{ steps.bench.outputs.p99_time }}
68+ vm_rss_memory_usage : ${{ steps.bench.outputs.vm_rss_memory_usage }}
69+ rss_anon_memory_usage : ${{ steps.bench.outputs.rss_anon_memory_usage }}
70+ upload_time : ${{ steps.bench.outputs.upload_time }}
71+ indexing_time : ${{ steps.bench.outputs.indexing_time }}
72+ steps :
73+ - uses : actions/checkout@v3
74+ -
uses :
webfactory/[email protected] 75+ with :
76+ ssh-private-key : ${{ secrets.SSH_PRIVATE_KEY }}
77+ - name : Bench ${{ inputs.qdrant_version_1 }}
78+ id : bench
79+ run : |
80+ export HCLOUD_TOKEN=${{ secrets.HCLOUD_TOKEN }}
81+ export POSTGRES_PASSWORD=${{ secrets.POSTGRES_PASSWORD }}
82+ export POSTGRES_HOST=${{ secrets.POSTGRES_HOST }}
83+ export QDRANT_VERSION=${{ inputs.qdrant_version_1 }}
84+ export DATASETS=${{ inputs.dataset }}
85+ export ENGINE_NAME=${{ inputs.engine_config }}
86+ export POSTGRES_TABLE=benchmark_manual
87+ bash -x tools/setup_ci.sh
88+ bash -x tools/run_ci.sh
89+
90+ runBenchmarkForVersion2 :
91+ name : execute - ${{ inputs.qdrant_version_2 }} - ${{ inputs.dataset }}
92+ needs :
93+ - prepareImage2
94+ - runBenchmarkForVersion1
95+ runs-on : ubuntu-latest
96+ concurrency :
97+ group : continuous-benchmark
98+ outputs :
99+ collection_load_time : ${{ steps.bench.outputs.collection_load_time }}
100+ rps : ${{ steps.bench.outputs.rps }}
101+ mean_precisions : ${{ steps.bench.outputs.mean_precisions }}
102+ p95_time : ${{ steps.bench.outputs.p95_time }}
103+ p99_time : ${{ steps.bench.outputs.p99_time }}
104+ vm_rss_memory_usage : ${{ steps.bench.outputs.vm_rss_memory_usage }}
105+ rss_anon_memory_usage : ${{ steps.bench.outputs.rss_anon_memory_usage }}
106+ upload_time : ${{ steps.bench.outputs.upload_time }}
107+ indexing_time : ${{ steps.bench.outputs.indexing_time }}
108+ steps :
109+ - uses : actions/checkout@v3
110+ -
uses :
webfactory/[email protected] 111+ with :
112+ ssh-private-key : ${{ secrets.SSH_PRIVATE_KEY }}
113+ - name : Bench ${{ inputs.qdrant_version_2 }}
114+ id : bench
115+ run : |
116+ export HCLOUD_TOKEN=${{ secrets.HCLOUD_TOKEN }}
117+ export POSTGRES_PASSWORD=${{ secrets.POSTGRES_PASSWORD }}
118+ export POSTGRES_HOST=${{ secrets.POSTGRES_HOST }}
119+ export QDRANT_VERSION=${{ inputs.qdrant_version_2 }}
120+ export DATASETS=${{ inputs.dataset }}
121+ export ENGINE_NAME=${{ inputs.engine_config }}
122+ export POSTGRES_TABLE=benchmark_manual
123+ bash -x tools/setup_ci.sh
124+ bash -x tools/run_ci.sh
125+
126+ compareVersions :
127+ name : compare - ${{ inputs.qdrant_version_1 }} vs ${{ inputs.qdrant_version_2 }}
128+ needs :
129+ - runBenchmarkForVersion1
130+ - runBenchmarkForVersion2
131+ runs-on : ubuntu-latest
132+ steps :
133+ - name : compare
134+ run : |
135+ compare() {
136+ local var_name=$1
137+ local value_v1=$2
138+ local value_v2=$3
139+
140+ if (( $(echo "$value_v1 > $value_v2" | bc -l) )); then
141+ local diff=$(echo "$value_v1 - $value_v2" | bc -l)
142+ local percentage=$(echo "($diff / $value_v1) * 100" | bc -l)
143+ echo -e "${{ inputs.qdrant_version_1 }} > ${{ inputs.qdrant_version_2 }} by $diff ($percentage% greater)"
144+ elif (( $(echo "$value_v1 < $value_v2" | bc -l) )); then
145+ local diff=$(echo "$value_v2 - $value_v1" | bc -l)
146+ local percentage=$(echo "($diff / $value_v2) * 100" | bc -l)
147+ echo -e "${{ inputs.qdrant_version_1 }} < ${{ inputs.qdrant_version_2 }} by $diff ($percentage% less)"
148+ else
149+ echo -e "equal"
150+ fi
151+ }
152+
153+ res_collection_load_time=$(compare "collection_load_time" "${{ needs.runBenchmarkForVersion1.outputs.collection_load_time }}" "${{ needs.runBenchmarkForVersion2.outputs.collection_load_time }}")
154+ res_rps=$(compare "rps" "${{ needs.runBenchmarkForVersion1.outputs.rps }}" "${{ needs.runBenchmarkForVersion2.outputs.rps }}")
155+ res_mean_precisions=$(compare "mean_precisions" "${{ needs.runBenchmarkForVersion1.outputs.mean_precisions }}" "${{ needs.runBenchmarkForVersion2.outputs.mean_precisions }}")
156+ res_p95_time=$(compare "p95_time" "${{ needs.runBenchmarkForVersion1.outputs.p95_time }}" "${{ needs.runBenchmarkForVersion2.outputs.p95_time }}")
157+ res_p99_time=$(compare "p99_time" "${{ needs.runBenchmarkForVersion1.outputs.p99_time }}" "${{ needs.runBenchmarkForVersion2.outputs.p99_time }}")
158+ res_vm_rss_memory_usage=$(compare "vm_rss_memory_usage" "${{ needs.runBenchmarkForVersion1.outputs.vm_rss_memory_usage }}" "${{ needs.runBenchmarkForVersion2.outputs.vm_rss_memory_usage }}")
159+ res_rss_anon_memory_usage=$(compare "rss_anon_memory_usage" "${{ needs.runBenchmarkForVersion1.outputs.rss_anon_memory_usage }}" "${{ needs.runBenchmarkForVersion2.outputs.rss_anon_memory_usage }}")
160+ res_upload_time=$(compare "upload_time" "${{ needs.runBenchmarkForVersion1.outputs.upload_time }}" "${{ needs.runBenchmarkForVersion2.outputs.upload_time }}")
161+ res_indexing_time=$(compare "indexing_time" "${{ needs.runBenchmarkForVersion1.outputs.indexing_time }}" "${{ needs.runBenchmarkForVersion2.outputs.indexing_time }}")
162+
163+ echo "# Comparison results" >> $GITHUB_STEP_SUMMARY
164+ echo "| Name | ${{ inputs.qdrant_version_1 }} | ${{ inputs.qdrant_version_2 }} | Result |" >> $GITHUB_STEP_SUMMARY
165+ echo "| --------------------- | ------------ | ------------ | ------------ |" >> $GITHUB_STEP_SUMMARY
166+ echo "| collection_load_time | ${{ needs.runBenchmarkForVersion1.outputs.collection_load_time }} | ${{ needs.runBenchmarkForVersion2.outputs.collection_load_time }} | ${res_collection_load_time} |" >> $GITHUB_STEP_SUMMARY
167+ echo "| rps | ${{ needs.runBenchmarkForVersion1.outputs.rps }} | ${{ needs.runBenchmarkForVersion2.outputs.rps }} | ${res_rps} |" >> $GITHUB_STEP_SUMMARY
168+ echo "| mean_precisions | ${{ needs.runBenchmarkForVersion1.outputs.mean_precisions }} | ${{ needs.runBenchmarkForVersion2.outputs.mean_precisions }} | ${res_mean_precisions} |" >> $GITHUB_STEP_SUMMARY
169+ echo "| p95_time | ${{ needs.runBenchmarkForVersion1.outputs.p95_time }} | ${{ needs.runBenchmarkForVersion2.outputs.p95_time }} | ${res_p95_time} |" >> $GITHUB_STEP_SUMMARY
170+ echo "| p99_time | ${{ needs.runBenchmarkForVersion1.outputs.p99_time }} | ${{ needs.runBenchmarkForVersion2.outputs.p99_time }} | ${res_p99_time} |" >> $GITHUB_STEP_SUMMARY
171+ echo "| vm_rss_memory_usage | ${{ needs.runBenchmarkForVersion1.outputs.vm_rss_memory_usage }} | ${{ needs.runBenchmarkForVersion2.outputs.vm_rss_memory_usage }} | ${res_vm_rss_memory_usage} |" >> $GITHUB_STEP_SUMMARY
172+ echo "| rss_anon_memory_usage | ${{ needs.runBenchmarkForVersion1.outputs.rss_anon_memory_usage }} | ${{ needs.runBenchmarkForVersion2.outputs.rss_anon_memory_usage }} | ${res_rss_anon_memory_usage} |" >> $GITHUB_STEP_SUMMARY
173+ echo "| upload_time | ${{ needs.runBenchmarkForVersion1.outputs.upload_time }} | ${{ needs.runBenchmarkForVersion2.outputs.upload_time }} | ${res_upload_time} |" >> $GITHUB_STEP_SUMMARY
174+ echo "| indexing_time | ${{ needs.runBenchmarkForVersion1.outputs.indexing_time }} | ${{ needs.runBenchmarkForVersion2.outputs.indexing_time }} | ${res_indexing_time} |" >> $GITHUB_STEP_SUMMARY
0 commit comments