Added support for MULTI EXEC

redis_benchmarks_specification/__cli__/args.py

@@ -19,6 +19,8 @@
 
 START_TIME_NOW_UTC, _, _ = get_start_time_vars()
 START_TIME_LAST_YEAR_UTC = START_TIME_NOW_UTC - datetime.timedelta(days=7)
+CLI_TOOL_STATS = "stats"
+CLI_TOOL_TRIGGER = "trigger"
 
 
 def spec_cli_args(parser):
@@ -31,6 +33,12 @@ def spec_cli_args(parser):
     parser.add_argument("--redis_host", type=str, default=GH_REDIS_SERVER_HOST)
     parser.add_argument("--branch", type=str, default="unstable")
     parser.add_argument("--commandstats-csv", type=str, default="")
+    parser.add_argument(
+        "--commandstats-csv-include-modules",
+        default=False,
+        action="store_true",
+        help="Include modules statistics on commandstats.",
+    )
     parser.add_argument("--summary-csv", type=str, default="")
     parser.add_argument("--commands-json-file", type=str, default="./commands.json")
     parser.add_argument("--groups-json-file", type=str, default="./groups.json")
@@ -52,7 +60,14 @@ def spec_cli_args(parser):
         action="store_true",
         help="Push test stats to redis.",
     )
-    parser.add_argument("--tool", type=str, default="trigger")
+    parser.add_argument(
+        "--tool",
+        type=str,
+        default=CLI_TOOL_TRIGGER,
+        help="subtool to use. One of '{}' ".format(
+            ",".join([CLI_TOOL_STATS, CLI_TOOL_TRIGGER])
+        ),
+    )
     parser.add_argument("--gh_token", type=str, default=GH_TOKEN)
     parser.add_argument("--redis_port", type=int, default=GH_REDIS_SERVER_PORT)
     parser.add_argument("--redis_pass", type=str, default=GH_REDIS_SERVER_AUTH)

redis_benchmarks_specification/__cli__/stats.py

@@ -22,6 +22,7 @@ def generate_stats_cli_command_logic(args, project_name, project_version):
         )
     )
     commands_json_file = os.path.abspath(args.commands_json_file)
+    include_modules = args.commandstats_csv_include_modules
     logging.info("Reading commands.json file from {}".format(commands_json_file))
     commands_json = {}
     tracked_commands_json = {}
@@ -202,7 +203,7 @@ def generate_stats_cli_command_logic(args, project_name, project_version):
 
                 if cmdstat in tracked_commands_json:
                     tracked = True
-                if module is False:
+                if module is False or include_modules:
                     row = [cmdstat, group, count, tracked, deprecated]
                     rows.append(row)
 

redis_benchmarks_specification/__runner__/runner.py

@@ -196,7 +196,8 @@ def prepare_memtier_benchmark_parameters(
     if resp_version:
         tool = clientconfig["tool"]
         if tool == "memtier_benchmark":
-            benchmark_command.extend(["--resp", resp_version])
+            if resp_version == "3":
+                benchmark_command.extend(["--protocol", "resp{}".format(resp_version)])
         elif tool == "redis-benchmark":
             if resp_version == "3":
                 benchmark_command.append("-3")

redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-hash-transactions-multi-exec-pipeline-20.yml

@@ -0,0 +1,39 @@
+version: 0.4
+name: memtier_benchmark-1Mkeys-hash-transactions-multi-exec-pipeline-20
+description: Runs memtier_benchmark, for a keyspace length of 1M keys with a data size of 100 Bytes for each key, focusing on the MULTI EXEC scenario.
+tested-groups:
+- transactions
+- hash
+tested-commands:
+- multi
+- exec
+- hsetnx
+- hexists
+- hget
+- hdel
+redis-topologies:
+- oss-standalone
+build-variants:
+- gcc:8.5.0-amd64-debian-buster-default
+dbconfig:
+  configuration-parameters:
+    save: '""'
+clientconfig:
+  run_image: redislabs/memtier_benchmark:edge
+  tool: memtier_benchmark
+  arguments: '"--data-size" "100" --command=MULTI --command-key-pattern=S --command="HSETNX __key__ __key__ __data__" --command-key-pattern=S --command="HEXISTS __key__ __key__" --command-key-pattern=S  --command="HGET __key__ __key__" --command-key-pattern=S --command="HDEL __key__ __key__"  --command-key-pattern=S --command=EXEC --command-key-pattern=S --pipeline=20 -c 50 -t 2 --hide-histogram --test-time 300'
+  resources:
+    requests:
+      cpus: '3'
+      memory: 2g
+exporter:
+  redistimeseries:
+    break_by:
+    - version
+    - commit
+    timemetric: $."ALL STATS".Runtime."Start time"
+    metrics:
+    - $."ALL STATS".Totals."Ops/sec"
+    - $."ALL STATS".Totals."Latency"
+    - $."ALL STATS".Totals."Misses/sec"
+    - $."ALL STATS".Totals."Percentile Latencies"."p50.00"