Merge pull request #281 from IntelPython/feature/add_devices_to_legend

Add devices to legend

Author: ZzEeKkAa

dpbench/console/report.py

@@ -62,6 +62,5 @@ def execute_report(args: Namespace, conn: sqlalchemy.Engine):
     print_report(
         conn=conn,
         run_id=args.run_id,
-        implementations=args.implementations,
         comparison_pairs=comparison_pairs,
     )

dpbench/console/run.py

@@ -12,6 +12,8 @@
 import dpbench.config as cfg
 import dpbench.infrastructure as dpbi
 from dpbench.infrastructure.benchmark_runner import BenchmarkRunner, RunConfig
+from dpbench.infrastructure.datamodel import Postfix, store_postfix
+from dpbench.infrastructure.frameworks.fabric import build_framework
 
 from ._namespace import Namespace
 
@@ -149,6 +151,30 @@ def execute_run(args: Namespace, conn: sqlalchemy.Engine):
 
     runner = BenchmarkRunner()
 
+    implementation_descriptions = {
+        impl.postfix: impl.description for impl in cfg.GLOBAL.implementations
+    }
+    for implementation in args.implementations:
+        framework_config = _find_framework_config(implementation)
+
+        if not framework_config:
+            logging.error(
+                f"Could not find framework for {implementation} implementation"
+            )
+            continue
+
+        framework = build_framework(framework_config)
+
+        store_postfix(
+            conn=conn,
+            postfix=Postfix(
+                run_id=args.run_id,
+                postfix=implementation,
+                description=implementation_descriptions[implementation],
+                device=framework.device_info,
+            ),
+        )
+
     for benchmark in cfg.GLOBAL.benchmarks:
         print("")
         print(
@@ -160,9 +186,6 @@ def execute_run(args: Namespace, conn: sqlalchemy.Engine):
             framework = _find_framework_config(implementation)
 
             if not framework:
-                logging.error(
-                    f"Could not find framework for {implementation} implementation"
-                )
                 continue
 
             logging.info(

dpbench/infrastructure/datamodel.py

@@ -112,6 +112,17 @@ class Result(Base):
     )
 
 
+class Postfix(Base):
+    __tablename__ = "postfixes"
+
+    run_id: Mapped[int] = mapped_column(ForeignKey("runs.id"))
+    postfix: Mapped[str]
+    description: Mapped[str]
+    device: Mapped[str]
+
+    __table_args__ = (UniqueConstraint("run_id", "postfix"),)
+
+
 def create_connection(db_file) -> Engine:
     """create a database connection to the SQLite database
         specified by db_file
@@ -164,9 +175,20 @@ def create_results_table(db_file: str):
 def store_results(conn: Engine, result: Result):
     """creates result record in database.
     :param conn: sqlalchemy engine
-    :param result: result resord to be inserted into db
+    :param result: result record to be inserted into db
     :return:
     """
     with Session(conn) as session:
         session.add(result)
         session.commit()
+
+
+def store_postfix(conn: Engine, postfix: Postfix):
+    """creates postfix record in database.
+    :param conn: sqlalchemy engine
+    :param postfix: postfix record to be inserted into db
+    :return:
+    """
+    with Session(conn) as session:
+        session.add(postfix)
+        session.commit()

dpbench/infrastructure/frameworks/dpcpp_framework.py

@@ -24,7 +24,7 @@ def __init__(self, fname: str = None, config: cfg.Framework = None):
 
         try:
             self.sycl_device = self.info.sycl_device
-            dpctl.SyclDevice(self.sycl_device)
+            self.device_info = dpctl.SyclDevice(self.sycl_device).name
         except KeyError:
             pass
         except dpctl.SyclDeviceCreationError as sdce:

dpbench/infrastructure/frameworks/dpnp_framework.py

@@ -24,7 +24,7 @@ def __init__(self, fname: str = None, config: cfg.Framework = None):
 
         try:
             self.sycl_device = self.info.sycl_device
-            dpctl.SyclDevice(self.sycl_device)
+            self.device_info = dpctl.SyclDevice(self.sycl_device).name
         except KeyError:
             pass
         except dpctl.SyclDeviceCreationError as sdce:

dpbench/infrastructure/frameworks/framework.py

@@ -41,6 +41,10 @@ def __init__(
         self.info = config
         self.fname = self.info.simple_name
 
+        import cpuinfo
+
+        self.device_info = cpuinfo.get_cpu_info().get("brand_raw")
+
     def device_filter_string(self) -> str:
         """Returns the sycl device's filter string if the framework has an
         associated sycl device."""

dpbench/infrastructure/frameworks/numba_dpex_framework.py

@@ -24,7 +24,7 @@ def __init__(self, fname: str = None, config: cfg.Framework = None):
 
         try:
             self.sycl_device = self.info.sycl_device
-            dpctl.SyclDevice(self.sycl_device)
+            self.device_info = dpctl.SyclDevice(self.sycl_device).name
         except dpctl.SyclDeviceCreationError as sdce:
             logging.exception(
                 "Could not create a Sycl device using filter {} string".format(

dpbench/infrastructure/frameworks/numba_mlir_framework.py

@@ -23,6 +23,10 @@ def __init__(self, fname: str = None, config: cfg.Framework = None):
         super().__init__(fname, config)
 
         self.sycl_device = self.info.sycl_device
+        if self.sycl_device:
+            import dpctl
+
+            self.device_info = dpctl.SyclDevice(self.sycl_device).name
 
     def copy_to_func(self) -> Callable:
         """Returns the copy-method that should be used

dpbench/infrastructure/reporter.py

@@ -1,7 +1,6 @@
 # Copyright 2022 Intel Corp.
 # SPDX-FileCopyrightText: 2022 Intel Corporation
 #
-# SPDX-License-Identifier: Apache 2.0
 # SPDX-License-Identifier: Apache-2.0
 
 """The module generates reports for implementation summary and timing summary
@@ -10,7 +9,6 @@
 
 import dataclasses
 import logging
-import pathlib
 from typing import Final, Union
 
 import pandas as pd
@@ -82,8 +80,23 @@ def generate_header(conn: sqlalchemy.Engine, run_id: int):
         print("==================================")
 
 
-def generate_legend(legends: pd.DataFrame):
-    """prints legend section"""
+def generate_legend(conn: sqlalchemy.Engine, run_id: int) -> list[str]:
+    """prints legend section and returns implementation list"""
+    sql = (
+        sqlalchemy.select(
+            dm.Postfix.postfix,
+            dm.Postfix.description,
+            dm.Postfix.device,
+        )
+        .order_by(dm.Postfix.postfix)
+        .where(dm.Postfix.run_id == run_id)
+    )
+
+    legends = pd.read_sql_query(
+        sql=sql,
+        con=conn.connect(),
+    )
+
     formatters = {}
     for col in legends.select_dtypes("object"):
         len_max = legends[col].str.len().max()
@@ -94,6 +107,8 @@ def generate_legend(legends: pd.DataFrame):
     print(legends.to_string(formatters=formatters))
     print("")
 
+    return legends["postfix"].values.tolist()
+
 
 def generate_summary(data: pd.DataFrame):
     """prints summary section"""
@@ -108,11 +123,6 @@ def generate_impl_summary_report(
     implementations: list[str],
 ):
     """generate implementation summary report with status of each benchmark"""
-    legends = read_legends()
-
-    generate_header(conn, run_id)
-    generate_legend(legends)
-
     columns = [
         func.max(dm.Result.input_size_human).label("input_size"),
         dm.Result.benchmark,
@@ -157,15 +167,8 @@ def generate_performance_report(
     conn: sqlalchemy.Engine,
     run_id: int,
     implementations: list[str],
-    headless=False,
 ):
     """generate performance report with median times for each benchmark"""
-    legends = read_legends()
-
-    if not headless:
-        generate_header(conn, run_id)
-        generate_legend(legends)
-
     columns = [
         func.max(dm.Result.input_size_human).label("input_size"),
         dm.Result.benchmark,
@@ -222,18 +225,11 @@ def generate_comparison_report(
     run_id: int,
     implementations: list[str],
     comparison_pairs: list[tuple[str, str]],
-    headless=False,
 ):
     """generate comparison report with median times for each benchmark"""
     if len(comparison_pairs) == 0:
         return
 
-    legends = read_legends()
-
-    if not headless:
-        generate_header(conn, run_id)
-        generate_legend(legends)
-
     columns = [
         func.max(dm.Result.input_size_human).label("input_size"),
         dm.Result.benchmark,
@@ -331,14 +327,10 @@ def get_unexpected_failures(
 def print_report(
     conn: sqlalchemy.Engine,
     run_id: int,
-    implementations: set[str],
     comparison_pairs: list[tuple[str, str]] = [],
 ):
-    if not implementations:
-        implementations = {impl.postfix for impl in cfg.GLOBAL.implementations}
-
-    implementations = list(implementations)
-    implementations.sort()
+    generate_header(conn, run_id)
+    implementations = generate_legend(conn, run_id)
 
     generate_impl_summary_report(
         conn, run_id=run_id, implementations=implementations
@@ -348,15 +340,13 @@ def print_report(
         conn,
         run_id=run_id,
         implementations=implementations,
-        headless=True,
     )
 
     generate_comparison_report(
         conn,
         run_id=run_id,
         implementations=implementations,
         comparison_pairs=comparison_pairs,
-        headless=True,
     )
 
     unexpected_failures = get_unexpected_failures(conn, run_id=run_id)

dpbench/migrations/versions/c1afe59771e9_add_postfixes.py

@@ -0,0 +1,50 @@
+# SPDX-FileCopyrightText: 2022 - 2023 Intel Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+
+"""Add postfixes
+
+Revision ID: c1afe59771e9
+Revises: 6477ee64f532
+Create Date: 2023-06-20 19:55:37.386101
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "c1afe59771e9"
+down_revision = "6477ee64f532"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table(
+        "postfixes",
+        sa.Column("run_id", sa.Integer(), nullable=False),
+        sa.Column("postfix", sa.String(), nullable=False),
+        sa.Column("description", sa.String(), nullable=False),
+        sa.Column("device", sa.String(), nullable=False),
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column(
+            "created_at",
+            sa.Integer(),
+            server_default=sa.text("(strftime('%s','now'))"),
+            nullable=False,
+        ),
+        sa.ForeignKeyConstraint(
+            ["run_id"],
+            ["runs.id"],
+        ),
+        sa.PrimaryKeyConstraint("id"),
+        sa.UniqueConstraint("run_id", "postfix"),
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table("postfixes")
+    # ### end Alembic commands ###