feat(benchmark): collect benchmarks for last 5 versions in line protocol format

benchmarks/collect_bench.sh

@@ -0,0 +1,70 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# This script is meant for developers of DataFusion -- it is runnable
+# from the standard DataFusion development environment and uses cargo,
+# etc and orchestrates gathering data and run the benchmark binary to
+# collect benchmarks from the current main and last 5 major releases.
+
+trap 'git checkout main' EXIT #checkout to main on exit
+ARG1=$1
+
+get_json_file_name() {
+  if [[ "$1" == "tpch" ]]; then
+    echo "tpch_sf1"
+  else
+    echo "$1"
+  fi
+}
+
+main(){
+timestamp=$(date +%s)
+lp_file="results/$ARG1-$timestamp.lp"
+mkdir -p results
+touch $lp_file
+
+json_file_name=$(get_json_file_name $ARG1)
+
+cp lineprotocol.py results/
+
+git fetch upstream main
+git checkout main
+
+# get current major version 
+output=$(cargo metadata --format-version=1 --no-deps | jq '.packages[] | select(.name == "datafusion") | .version')
+major_version=$(echo "$output" | grep -oE '[0-9]+' | head -n1)
+
+
+# run for current main
+echo "current major version: $major_version"  
+export RESULTS_DIR="results/$major_version.0.0"
+./bench.sh run $ARG1
+python3 results/lineprotocol.py $RESULTS_DIR/$json_file_name.json >> $lp_file
+
+# run for last 5 major releases
+for i in {1..5}; do
+    echo "running benchmark on $((major_version-i)).0.0"
+    git fetch upstream $((major_version-i)).0.0
+    git checkout $((major_version-i)).0.0
+    export RESULTS_DIR="results/$((major_version-i)).0.0"
+    ./bench.sh run $ARG1
+    python3 results/lineprotocol.py $RESULTS_DIR/$json_file_name.json >> $lp_file
+done
+}
+
+main

benchmarks/lineprotocol.py

@@ -1,23 +1,22 @@
 #!/usr/bin/env python
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-
-""" 
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+
 Converts a given json to LineProtocol format that can be 
 visualised by grafana/other systems that support LineProtocol. 
 
@@ -27,10 +26,8 @@
 benchmark,name=sort,version=28.0.0,datafusion_version=28.0.0,num_cpus=8 query="sort utf8",iteration=1,row_count=10838832,elapsed_ms=68694468 1691105678000000000
 benchmark,name=sort,version=28.0.0,datafusion_version=28.0.0,num_cpus=8 query="sort utf8",iteration=2,row_count=10838832,elapsed_ms=63392883 1691105678000000000
 benchmark,name=sort,version=28.0.0,datafusion_version=28.0.0,num_cpus=8 query="sort utf8",iteration=3,row_count=10838832,elapsed_ms=66388367 1691105678000000000
-"""
 
-# sort.json
-"""
+sort.json
 {
   "queries": [
     {
@@ -169,7 +166,7 @@ def lineformat(
         query_str = f"query=\"{query.query}\""
         timestamp = f"{query.start_time*10**9}"
         for iter_num, result in enumerate(query.iterations):
-            print(f"{benchamrk_str} {query_str},iteration={iter_num},row_count={result.row_count},elapsed_ms={result.elapsed*1000:.0f} {timestamp}\n")
+            print(f"{benchamrk_str} {query_str},iteration={iter_num},row_count={result.row_count},elapsed_ms={result.elapsed:.3f} {timestamp}\n")
 
 def main() -> None:
     parser = ArgumentParser()
@@ -180,7 +177,7 @@ def main() -> None:
     )
     options = parser.parse_args()
 
-    lineformat(options.baseline_path)
+    lineformat(options.path)