Merge branch 'main' into package

Author: adarshyoga

.github/workflows/build_and_run.yml

@@ -15,6 +15,7 @@ jobs:
     runs-on: ubuntu-latest
 
     strategy:
+      fail-fast: false
       matrix:
         python: ["3.9", "3.10"]
 
@@ -56,26 +57,25 @@ jobs:
         run: |
           CC=icx CXX=icpx python setup.py develop -- -Dpybind11_DIR=$(python -m pybind11 --cmakedir) -DDPCTL_MODULE_PATH=$(python -m dpctl --cmakedir)
 
-      - name: Validate dpbench
+      - name: Run benchmarks
         shell: bash -l {0}
         run: |
           export OCL_ICD_FILENAMES=libintelocl.so
-
           export NUMBA_MLIR_GPU_RUNTIME=sycl
-
           # Turn off numba-dpex autofall back
           export NUMBA_DPEX_FALLBACK_ON_CPU=0
           # Make sure numba-dpex is using native atomics in github CI
           export NUMBA_DPEX_ACTIVATE_ATOMICS_FP_NATIVE=1
           ls $(dirname $(dirname `which icx`))/bin-llvm || exit 1
           export NUMBA_DPEX_LLVM_SPIRV_ROOT=$(dirname $(dirname `which icx`))/bin-llvm
 
-          python -c "import dpbench; dpbench.run_benchmarks(repeat=2)" || exit 1
-          # LD_PRELOAD="${CONDA_PREFIX}/lib/libmkl_intel_ilp64.so ${CONDA_PREFIX}/lib/libmkl_tbb_thread.so ${CONDA_PREFIX}/lib/libmkl_core.so" python -c "import dpbench; dpbench.all_benchmarks_passed_validation(\"dpbench.db\");"
-          # res=`                                                                    \
-          #   printf "%s"                                                            \
-          #   "$(python -c "import dpbench;                                          \
-          #       res=int(dpbench.all_benchmarks_passed_validation(\"dpbench.db\")); \
-          #       print(\"Validation:\",res)"                                        \
-          #     )" | grep Validation | cut -f 2 -d ":"` || exit 1
-          # if [ ${res} == 0 ]; then exit 1; fi
+          dpbench \
+          -i python,numpy,dpnp,sycl,numba_n,numba_np,numba_npr,numba_dpex_k,numba_dpex_n,numba_dpex_p \
+          run -r2 --no-print-results
+
+      - name: Generate report
+        shell: bash -l {0}
+        run: |
+          dpbench \
+          -i python,numpy,dpnp,sycl,numba_n,numba_np,numba_npr,numba_dpex_k,numba_dpex_n,numba_dpex_p \
+          report || exit 1

.pre-commit-config.yaml

@@ -50,7 +50,7 @@ repos:
     hooks:
     -   id: pydocstyle
         # TODO: add packages one by one to enforce pydocstyle eventually
-        files: (^dpbench/config/|^scripts/)
+        files: (^dpbench/config/|^scripts/|^dpbench/console/)
         args: ["--convention=google"]
         # D417 does not work properly:
         # https://github.com/PyCQA/pydocstyle/issues/459

README.md

@@ -35,11 +35,11 @@ SPDX-License-Identifier: Apache-2.0
         ```
     - To run, taking black_scholes for example:
         ```bash
-        $  python -c "import dpbench; dpbench.run_benchmark(\"black_scholes\")" 2> /dev/null
+        $  dpbench -b black_scholes run
         ```
     - Similarly, to run all the cases in DPBench:
         ```bash
-        $  python -c "import dpbench; dpbench.run_benchmarks()" 2> /dev/null
+        $  dpbench -a run
         ```
 
 3. Device Customization
@@ -68,17 +68,19 @@ SPDX-License-Identifier: Apache-2.0
 
     > **_NOTE:_**  The `arch` option is deprecated and not used by dpbench.
 
-   To run with custimized framework JSON file, pass it as an argument to the `run_benchmark` or
+   To run with customized framework JSON file, pass it as an argument to the `run_benchmark` or
    `run_benchmarks` functions.
 
+   TODO: current way not working anymore.
+
     ```bash
     $ python -c "import dpbench; dpbench.run_benchmark(\"black_scholes\", "<absolute path to json file>")"
     ```
 
 ## Running numba-mlir benchmarks
 1. Setting up conda environment and installing dependencies:
 
-    Use same instructions as for usual dpbench setup, but do not install numba-dpex.
+    Use same instructions as for usual dpbench setup.
 
     Install latest `numba-mlir` dev package:
 
@@ -88,12 +90,19 @@ SPDX-License-Identifier: Apache-2.0
 
     Use same commands to setup and run dpbench:
 
-        $ python -c "import dpbench; dpbench.run_benchmark(\"black_scholes\")" 2> /dev/null
+        ```bash
+        $  dpbench -b black_scholes run
+        ```
 
     or, to run specific version:
 
-        $ python -c "import dpbench; dpbench.run_benchmark(\"black_scholes\",implementation_postfix=\"numba_mlir_k\")" 2> /dev/null
+
+        ```bash
+        $  dpbench -b black_scholes -i numba_mlir_k s run
+        ```
 
     to run all `numba-mlir` benchmarks:
 
-        $ python -c "import dpbench; dpbench.run_benchmarks(implementations=[\"numba_mlir_n\",\"numba_mlir_p\",\"numba_mlir_k\"])" 2> /dev/null
+        ```bash
+        $  dpbench -b black_scholes -i numba_mlir_k,numba_mlir_n,numba_mlir_p s run
+        ```

alembic.ini

@@ -6,7 +6,7 @@
 
 [alembic]
 # path to migration scripts
-script_location = alembic
+script_location = dpbench:migrations
 
 # template used to generate migration file names; The default value is %%(rev)s_%%(slug)s
 # Uncomment the line below if you want the files to be prepended with date and time

dpbench/__init__.py

@@ -1,10 +1,3 @@
 # SPDX-FileCopyrightText: 2022 - 2023 Intel Corporation
 #
 # SPDX-License-Identifier: Apache-2.0
-
-from .runner import run_benchmark, run_benchmarks
-
-__all__ = [
-    "run_benchmark",
-    "run_benchmarks",
-]

dpbench/config/__init__.py

@@ -17,7 +17,7 @@
 from .reader import read_configs
 
 """Use this variable for reading configurations"""
-GLOBAL: Config = read_configs()
+GLOBAL: Config = Config()
 
 __all__ = [
     "GLOBAL",

dpbench/config/reader.py

@@ -21,8 +21,12 @@
 
 
 def read_configs(  # noqa: C901: TODO: move modules into config
-    benchmarks: list[str] = None,
-    implementations: list[str] = None,
+    benchmarks: set[str] = None,
+    implementations: set[str] = None,
+    no_dpbench: bool = False,
+    with_npbench: bool = False,
+    with_polybench: bool = False,
+    load_implementations: bool = True,
 ) -> Config:
     """Read all configuration files and populate those settings into Config.
 
@@ -53,11 +57,9 @@ def read_configs(  # noqa: C901: TODO: move modules into config
         ),
     ]
 
-    no_dpbench = os.getenv("NO_DPBENCH")
     if no_dpbench:
         modules[0].benchmark_configs_path = ""
 
-    with_npbench = os.getenv("WITH_NPBENCH")
     if with_npbench:
         modules.append(
             Module(
@@ -69,7 +71,6 @@ def read_configs(  # noqa: C901: TODO: move modules into config
             )
         )
 
-    with_polybench = os.getenv("WITH_POLYBENCH")
     if with_polybench:
         modules.append(
             Module(
@@ -92,7 +93,7 @@ def read_configs(  # noqa: C901: TODO: move modules into config
                 benchmarks=benchmarks,
             )
         if mod.framework_configs_path != "":
-            read_frameworks(config, mod.framework_configs_path)
+            read_frameworks(config, mod.framework_configs_path, implementations)
         if mod.precision_dtypes_path != "":
             read_precision_dtypes(config, mod.precision_dtypes_path)
         if mod.path != "":
@@ -102,13 +103,20 @@ def read_configs(  # noqa: C901: TODO: move modules into config
         config.implementations += framework.postfixes
 
     if implementations is None:
-        implementations = [impl.postfix for impl in config.implementations]
+        implementations = {impl.postfix for impl in config.implementations}
 
-    for benchmark in config.benchmarks:
-        read_benchmark_implementations(
-            benchmark,
-            implementations,
-        )
+    if load_implementations:
+        for benchmark in config.benchmarks:
+            read_benchmark_implementations(
+                benchmark,
+                implementations,
+            )
+
+        config.benchmarks = [
+            benchmark
+            for benchmark in config.benchmarks
+            if len(benchmark.implementations) > 0
+        ]
 
     return config
 
@@ -118,7 +126,7 @@ def read_benchmarks(
     bench_info_dir: str,
     recursive: bool = False,
     parent_package: str = "dpbench.benchmarks",
-    benchmarks: list[str] = None,
+    benchmarks: set[str] = None,
 ):
     """Read and populate benchmark configuration files.
 
@@ -163,12 +171,20 @@ def read_benchmarks(
         config.benchmarks.append(benchmark)
 
 
-def read_frameworks(config: Config, framework_info_dir: str) -> None:
+def read_frameworks(
+    config: Config,
+    framework_info_dir: str,
+    implementations: set[str] = None,
+) -> None:
     """Read and populate framework configuration files.
 
     Args:
         config: Configuration object where settings should be populated.
         framework_info_dir: Path to the directory with configuration files.
+        implementations: Set of the implementations to load. If framework
+            does not have any implementation from this list - it won't be
+            loaded. If set None or empty - all frameworks/implementations get
+            loaded.
     """
     for framework_info_file in os.listdir(framework_info_dir):
         if not framework_info_file.endswith(".toml"):
@@ -183,9 +199,20 @@ def read_frameworks(config: Config, framework_info_dir: str) -> None:
 
         framework_info = tomli.loads(file_contents)
         framework_dict = framework_info.get("framework")
-        if framework_dict:
-            framework = Framework.from_dict(framework_dict)
-            config.frameworks.append(framework)
+        if not framework_dict:
+            continue
+        framework = Framework.from_dict(framework_dict)
+        if implementations:
+            framework.postfixes = [
+                postfix
+                for postfix in framework.postfixes
+                if postfix.postfix in implementations
+            ]
+
+        if len(framework.postfixes) == 0:
+            continue
+
+        config.frameworks.append(framework)
 
 
 def read_implementation_postfixes(
@@ -249,8 +276,8 @@ def setup_init(config: Benchmark, modules: list[str]) -> None:
         impl_mod = importlib.import_module(config.init.package_path)
 
         if not hasattr(impl_mod, config.init.func_name):
-            print(
-                f"WARNING: could not find init function for {config.module_name}"
+            logging.warn(
+                f"could not find init function for {config.module_name}"
             )
 
 
@@ -284,7 +311,7 @@ def discover_module_name_and_postfix(module: str, config: Config):
 
 def read_benchmark_implementations(
     config: Benchmark,
-    implementations: list[str] = None,
+    implementations: set[str] = None,
 ) -> None:
     """Read and discover implementation modules and functions.
 
@@ -318,8 +345,10 @@ def read_benchmark_implementations(
     for module in modules:
         module_name, postfix = discover_module_name_and_postfix(module, config)
 
-        if (postfix not in implementations) or (
-            config.init and config.init.module_name.endswith(module_name)
+        if (
+            implementations
+            and (postfix not in implementations)
+            or (config.init and config.init.module_name.endswith(module_name))
         ):
             continue
 

dpbench/console/_namespace.py

@@ -0,0 +1,30 @@
+# SPDX-FileCopyrightText: 2022 - 2023 Intel Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+
+"""Namespace class for parsed arguments."""
+
+import argparse
+from typing import Union
+
+
+class Namespace(argparse.Namespace):
+    """Namespace class for parsed arguments."""
+
+    benchmarks: set[str]
+    implementations: set[str]
+    all_implementations: bool
+    preset: str
+    dpbench: bool
+    npbench: bool
+    polybench: bool
+    print_results: bool
+    validate: bool
+    run_id: Union[int, None]
+    last_run: bool
+    results_db: str
+    save: bool
+    repeat: int
+    timeout: float
+    precision: Union[str, None]
+    program: str