From 70019711d390c33538877644290d184a427d2a9f Mon Sep 17 00:00:00 2001 From: syntron Date: Tue, 24 Jun 2025 16:31:42 +0200 Subject: [PATCH 01/29] [ModelicaSystemDoE] add class --- OMPython/ModelicaSystem.py | 236 ++++++++++++++++++++++++++++++++++++- 1 file changed, 235 insertions(+), 1 deletion(-) diff --git a/OMPython/ModelicaSystem.py b/OMPython/ModelicaSystem.py index b1f9a32e..9ae28199 100644 --- a/OMPython/ModelicaSystem.py +++ b/OMPython/ModelicaSystem.py @@ -34,12 +34,16 @@ import ast from dataclasses import dataclass +import itertools import logging import numbers import numpy as np import os +import pandas as pd +import queue import textwrap -from typing import Optional, Any +import threading +from typing import Any, Optional import warnings import xml.etree.ElementTree as ET @@ -1791,3 +1795,233 @@ def getLinearOutputs(self) -> list[str]: def getLinearStates(self) -> list[str]: """Get names of state variables of the linearized model.""" return self._linearized_states + + +class ModelicaSystemDoE: + def __init__( + self, + fileName: Optional[str | os.PathLike | pathlib.Path] = None, + modelName: Optional[str] = None, + lmodel: Optional[list[str | tuple[str, str]]] = None, + commandLineOptions: Optional[str] = None, + variableFilter: Optional[str] = None, + customBuildDirectory: Optional[str | os.PathLike | pathlib.Path] = None, + omhome: Optional[str] = None, + + simargs: Optional[dict[str, Optional[str | dict[str, str]]]] = None, + timeout: Optional[int] = None, + + resultpath: Optional[pathlib.Path] = None, + parameters: Optional[dict[str, list[str] | list[int] | list[float]]] = None, + ) -> None: + self._lmodel = lmodel + self._modelName = modelName + self._fileName = fileName + + self._CommandLineOptions = commandLineOptions + self._variableFilter = variableFilter + self._customBuildDirectory = customBuildDirectory + self._omhome = omhome + + # reference for the model; not used for any simulations but to evaluate parameters, etc. + self._mod = ModelicaSystem( + fileName=self._fileName, + modelName=self._modelName, + lmodel=self._lmodel, + commandLineOptions=self._CommandLineOptions, + variableFilter=self._variableFilter, + customBuildDirectory=self._customBuildDirectory, + omhome=self._omhome, + ) + + self._simargs = simargs + self._timeout = timeout + + if isinstance(resultpath, pathlib.Path): + self._resultpath = resultpath + else: + self._resultpath = pathlib.Path('.') + + if isinstance(parameters, dict): + self._parameters = parameters + else: + self._parameters = {} + + self._sim_df: Optional[pd.DataFrame] = None + self._sim_task_query: queue.Queue = queue.Queue() + + def prepare(self) -> int: + + param_structure = {} + param_simple = {} + for param_name in self._parameters.keys(): + changeable = self._mod.isParameterChangeable(name=param_name) + logger.info(f"Parameter {repr(param_name)} is changeable? {changeable}") + + if changeable: + param_simple[param_name] = self._parameters[param_name] + else: + param_structure[param_name] = self._parameters[param_name] + + param_structure_combinations = list(itertools.product(*param_structure.values())) + param_simple_combinations = list(itertools.product(*param_simple.values())) + + df_entries: list[pd.DataFrame] = [] + for idx_pc_structure, pc_structure in enumerate(param_structure_combinations): + mod_structure = ModelicaSystem( + fileName=self._fileName, + modelName=self._modelName, + lmodel=self._lmodel, + commandLineOptions=self._CommandLineOptions, + variableFilter=self._variableFilter, + customBuildDirectory=self._customBuildDirectory, + omhome=self._omhome, + ) + + sim_args_structure = {} + for idx_structure, pk_structure in enumerate(param_structure.keys()): + sim_args_structure[pk_structure] = pc_structure[idx_structure] + + pk_value = pc_structure[idx_structure] + if isinstance(pk_value, str): + expression = f"setParameterValue({self._modelName}, {pk_structure}, $Code(=\"{pk_value}\"))" + elif isinstance(pk_value, bool): + pk_value_bool_str = "true" if pk_value else "false" + expression = f"setParameterValue({self._modelName}, {pk_structure}, $Code(={pk_value_bool_str}));" + else: + expression = f"setParameterValue({self._modelName}, {pk_structure}, {pk_value})" + mod_structure.sendExpression(expression) + + for idx_pc_simple, pc_simple in enumerate(param_simple_combinations): + sim_args_simple = {} + for idx_simple, pk_simple in enumerate(param_simple.keys()): + sim_args_simple[pk_simple] = str(pc_simple[idx_simple]) + + resfilename = f"DOE_{idx_pc_structure:09d}_{idx_pc_simple:09d}.mat" + logger.info(f"use result file {repr(resfilename)} " + f"for structural parameters: {sim_args_structure} " + f"and simple parameters: {sim_args_simple}") + resultfile = self._resultpath / resfilename + + df_data = ( + { + 'ID structure': idx_pc_structure, + 'ID simple': idx_pc_simple, + 'resulfilename': resfilename, + 'structural parameters ID': idx_pc_structure, + } + | sim_args_structure + | { + 'non-structural parameters ID': idx_pc_simple, + } + | sim_args_simple + | { + 'results available': False, + } + ) + + df_entries.append(pd.DataFrame.from_dict(df_data)) + + cmd = mod_structure.simulate_cmd( + resultfile=resultfile.absolute().resolve(), + simargs={"override": sim_args_simple}, + ) + + self._sim_task_query.put(cmd) + + self._sim_df = pd.concat(df_entries, ignore_index=True) + + logger.info(f"Prepared {self._sim_df.shape[0]} simulation definitions for the defined DoE.") + + return self._sim_df.shape[0] + + def get_doe(self) -> Optional[pd.DataFrame]: + return self._sim_df + + def simulate(self, num_workers: int = 3) -> None: + + sim_count_total = self._sim_task_query.qsize() + + def worker(worker_id, task_queue): + while True: + sim_data = {} + try: + # Get the next task from the queue + cmd: ModelicaSystemCmd = task_queue.get(block=False) + except queue.Empty: + logger.info(f"[Worker {worker_id}] No more simulations to run.") + break + + resultfile = cmd.arg_get(key='r') + resultpath = pathlib.Path(resultfile) + + logger.info(f"[Worker {worker_id}] Performing task: {resultpath.name}") + + try: + sim_data['sim'].run() + except ModelicaSystemError as ex: + logger.warning(f"Simulation error for {resultpath.name}: {ex}") + + # Mark the task as done + task_queue.task_done() + + sim_count_done = sim_count_total - self._sim_task_query.qsize() + logger.info(f"[Worker {worker_id}] Task completed: {resultpath.name} " + f"({sim_count_done}/{sim_count_total} = " + f"{sim_count_done / sim_count_total * 100:.2f}% of tasks left)") + + logger.info(f"Start simulations for DoE with {sim_count_total} simulations " + f"using {num_workers} workers ...") + + # Create and start worker threads + threads = [] + for i in range(num_workers): + thread = threading.Thread(target=worker, args=(i, self._sim_task_query)) + thread.start() + threads.append(thread) + + # Wait for all threads to complete + for thread in threads: + thread.join() + + for idx, row in self._sim_df.to_dict('index').items(): + resultfilename = row['resultfilename'] + resultfile = self._resultpath / resultfilename + + if resultfile.exists(): + self._sim_df.loc[idx, 'results available'] = True + + sim_total = self._sim_df.shape[0] + sim_done = self._sim_df['results available'].sum() + logger.info(f"All workers finished ({sim_done} of {sim_total} simulations with a result file).") + + def get_solutions( + self, + var_list: Optional[list] = None, + ) -> Optional[tuple[str] | dict[str, pd.DataFrame | str]]: + if self._sim_df is None: + return None + + if self._sim_df.shape[0] == 0 or self._sim_df['results available'].sum() == 0: + raise ModelicaSystemError("No result files available - all simulations did fail?") + + if var_list is None: + resultfilename = self._sim_df['resultfilename'].values[0] + resultfile = self._resultpath / resultfilename + return self._mod.getSolutions(resultfile=resultfile) + + sol_dict: dict[str, pd.DataFrame | str] = {} + for row in self._sim_df.to_dict('records'): + resultfilename = row['resultfilename'] + resultfile = self._resultpath / resultfilename + + try: + sol = self._mod.getSolutions(varList=var_list, resultfile=resultfile) + sol_data = {var: sol[idx] for idx, var in var_list} + sol_df = pd.DataFrame(sol_data) + sol_dict[resultfilename] = sol_df + except ModelicaSystemError as ex: + logger.warning(f"No solution for {resultfilename}: {ex}") + sol_dict[resultfilename] = str(ex) + + return sol_dict From 0242d150fd1afd1c722c2e68cb34762fd2fd13bd Mon Sep 17 00:00:00 2001 From: syntron Date: Tue, 24 Jun 2025 16:32:04 +0200 Subject: [PATCH 02/29] [__init__] add class ModelicaSystemDoE --- OMPython/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/OMPython/__init__.py b/OMPython/__init__.py index 6144f1c2..649b3e60 100644 --- a/OMPython/__init__.py +++ b/OMPython/__init__.py @@ -36,7 +36,8 @@ CONDITIONS OF OSMC-PL. """ -from OMPython.ModelicaSystem import LinearizationResult, ModelicaSystem, ModelicaSystemCmd, ModelicaSystemError +from OMPython.ModelicaSystem import (LinearizationResult, ModelicaSystem, ModelicaSystemCmd, ModelicaSystemDoE, + ModelicaSystemError) from OMPython.OMCSession import (OMCSessionCmd, OMCSessionException, OMCSessionRunData, OMCSessionZMQ, OMCProcessPort, OMCProcessLocal, OMCProcessDocker, OMCProcessDockerContainer, OMCProcessWSL) @@ -46,6 +47,7 @@ 'LinearizationResult', 'ModelicaSystem', 'ModelicaSystemCmd', + 'ModelicaSystemDoE', 'ModelicaSystemError', 'OMCSessionCmd', From d2df43529e9d42d6eb6bd32dc7b4ea0bd03a515c Mon Sep 17 00:00:00 2001 From: syntron Date: Tue, 24 Jun 2025 16:32:57 +0200 Subject: [PATCH 03/29] [test_ModelicaSystemDoE] add test --- tests/test_ModelicaSystemDoE.py | 58 +++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 tests/test_ModelicaSystemDoE.py diff --git a/tests/test_ModelicaSystemDoE.py b/tests/test_ModelicaSystemDoE.py new file mode 100644 index 00000000..a49ef111 --- /dev/null +++ b/tests/test_ModelicaSystemDoE.py @@ -0,0 +1,58 @@ +import OMPython +import pandas as pd +import pathlib +import pytest + +@pytest.fixture +def model_doe(tmp_path: pathlib.Path) -> pathlib.Path: + # see: https://trac.openmodelica.org/OpenModelica/ticket/4052 + mod = tmp_path / "M.mo" + mod.write_text(f""" +model M + parameter Integer p=1; + parameter Integer q=1; + parameter Real a = -1; + parameter Real b = -1; + Real x[p]; + Real y[q]; +equation + der(x) = a * fill(1.0, p); + der(y) = b * fill(1.0, q); +end M; +""") + return mod + + +@pytest.fixture +def param_doe() -> dict[str, list]: + param = { + # structural + 'p': [1, 2], + 'q': [3, 4], + # simple + 'a': [5, 6], + 'b': [7, 8], + } + return param + + +def test_ModelicaSystemDoE(tmp_path, model_doe, param_doe): + + tmpdir = tmp_path / 'DoE' + tmpdir.mkdir(exist_ok=True) + + mod_doe = OMPython.ModelicaSystemDoE( + fileName=model_doe.as_posix(), + modelName="M", + parameters=param_doe, + resultpath=tmpdir, + ) + mod_doe.prepare() + df_doe = mod_doe.get_doe() + assert isinstance(df_doe, pd.DataFrame) + assert df_doe.shape[0] == 16 + assert df_doe['results available'].sum() == 16 + + mod_doe.simulate() + sol = mod_doe.get_solutions(var_list=['x[1]', 'y[1]']) + assert len(sol) == 16 From 891a826416514c2a1726de7110d095da8a51ebf2 Mon Sep 17 00:00:00 2001 From: syntron Date: Wed, 25 Jun 2025 21:01:25 +0200 Subject: [PATCH 04/29] [ModelicaSystemDoE] add docstrings --- OMPython/ModelicaSystem.py | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/OMPython/ModelicaSystem.py b/OMPython/ModelicaSystem.py index 9ae28199..38bcf0a4 100644 --- a/OMPython/ModelicaSystem.py +++ b/OMPython/ModelicaSystem.py @@ -1798,6 +1798,10 @@ def getLinearStates(self) -> list[str]: class ModelicaSystemDoE: + """ + Class to run DoEs based on a (Open)Modelica model using ModelicaSystem + """ + def __init__( self, fileName: Optional[str | os.PathLike | pathlib.Path] = None, @@ -1814,6 +1818,11 @@ def __init__( resultpath: Optional[pathlib.Path] = None, parameters: Optional[dict[str, list[str] | list[int] | list[float]]] = None, ) -> None: + """ + Initialisation of ModelicaSystemDoE. The parameters are based on: ModelicaSystem.__init__() and + ModelicaSystem.simulate(). Additionally, the path to store the result files is needed (= resultpath) as well as + a list of parameters to vary for the Doe (= parameters). All possible combinations are considered. + """ self._lmodel = lmodel self._modelName = modelName self._fileName = fileName @@ -1851,6 +1860,12 @@ def __init__( self._sim_task_query: queue.Queue = queue.Queue() def prepare(self) -> int: + """ + Prepare the DoE by evaluating the parameters. Each structural parameter requires a new instance of + ModelicaSystem while the non-structural parameters can just be set on the executable. + + The return value is the number of simulation defined. + """ param_structure = {} param_simple = {} @@ -1936,9 +1951,17 @@ def prepare(self) -> int: return self._sim_df.shape[0] def get_doe(self) -> Optional[pd.DataFrame]: + """ + Get the defined Doe as a poandas dataframe. + """ return self._sim_df def simulate(self, num_workers: int = 3) -> None: + """ + Simulate the DoE using the defined number of workers. + + Returns True if all simulations were done successfully, else False. + """ sim_count_total = self._sim_task_query.qsize() @@ -1999,6 +2022,15 @@ def get_solutions( self, var_list: Optional[list] = None, ) -> Optional[tuple[str] | dict[str, pd.DataFrame | str]]: + """ + Get all solutions of the DoE run. The following return values are possible: + + * None, if there no simulation was run + + * A list of variables if val_list == None + + * The Solutions as dict[str, pd.DataFrame] if a value list (== val_list) is defined. + """ if self._sim_df is None: return None From f54fa00394fbb5df641a00949d7dfe21cadad995 Mon Sep 17 00:00:00 2001 From: syntron Date: Wed, 25 Jun 2025 21:03:59 +0200 Subject: [PATCH 05/29] [ModelicaSystemDoE] define dict keys as constants --- OMPython/ModelicaSystem.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/OMPython/ModelicaSystem.py b/OMPython/ModelicaSystem.py index 38bcf0a4..ad5a323d 100644 --- a/OMPython/ModelicaSystem.py +++ b/OMPython/ModelicaSystem.py @@ -1802,6 +1802,9 @@ class ModelicaSystemDoE: Class to run DoEs based on a (Open)Modelica model using ModelicaSystem """ + DF_COLUMNS_RESULTFILENAME: str = 'resultfilename' + DF_COLUMNS_RESULTS_AVAILABLE: str = 'results available' + def __init__( self, fileName: Optional[str | os.PathLike | pathlib.Path] = None, @@ -1922,7 +1925,7 @@ def prepare(self) -> int: { 'ID structure': idx_pc_structure, 'ID simple': idx_pc_simple, - 'resulfilename': resfilename, + self.DF_COLUMNS_RESULTFILENAME: resfilename, 'structural parameters ID': idx_pc_structure, } | sim_args_structure @@ -1931,7 +1934,7 @@ def prepare(self) -> int: } | sim_args_simple | { - 'results available': False, + self.DF_COLUMNS_RESULTS_AVAILABLE: False, } ) @@ -2008,14 +2011,15 @@ def worker(worker_id, task_queue): thread.join() for idx, row in self._sim_df.to_dict('index').items(): - resultfilename = row['resultfilename'] + resultfilename = row[self.DF_COLUMNS_RESULTFILENAME] resultfile = self._resultpath / resultfilename if resultfile.exists(): - self._sim_df.loc[idx, 'results available'] = True + mask = self._sim_df[self.DF_COLUMNS_RESULTFILENAME] == resultfilename + self._sim_df.loc[mask, self.DF_COLUMNS_RESULTS_AVAILABLE] = True + sim_done = self._sim_df[self.DF_COLUMNS_RESULTS_AVAILABLE].sum() sim_total = self._sim_df.shape[0] - sim_done = self._sim_df['results available'].sum() logger.info(f"All workers finished ({sim_done} of {sim_total} simulations with a result file).") def get_solutions( @@ -2034,17 +2038,17 @@ def get_solutions( if self._sim_df is None: return None - if self._sim_df.shape[0] == 0 or self._sim_df['results available'].sum() == 0: + if self._sim_df.shape[0] == 0 or self._sim_df[self.DF_COLUMNS_RESULTS_AVAILABLE].sum() == 0: raise ModelicaSystemError("No result files available - all simulations did fail?") if var_list is None: - resultfilename = self._sim_df['resultfilename'].values[0] + resultfilename = self._sim_df[self.DF_COLUMNS_RESULTFILENAME].values[0] resultfile = self._resultpath / resultfilename return self._mod.getSolutions(resultfile=resultfile) sol_dict: dict[str, pd.DataFrame | str] = {} for row in self._sim_df.to_dict('records'): - resultfilename = row['resultfilename'] + resultfilename = row[self.DF_COLUMNS_RESULTFILENAME] resultfile = self._resultpath / resultfilename try: From e8f1bbaba9c89d3fb177c29f36be5c8a2ae03871 Mon Sep 17 00:00:00 2001 From: syntron Date: Wed, 25 Jun 2025 21:04:34 +0200 Subject: [PATCH 06/29] [ModelicaSystemDoE] build model after all structural parameters are defined --- OMPython/ModelicaSystem.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/OMPython/ModelicaSystem.py b/OMPython/ModelicaSystem.py index ad5a323d..1bcb5f24 100644 --- a/OMPython/ModelicaSystem.py +++ b/OMPython/ModelicaSystem.py @@ -1894,6 +1894,7 @@ def prepare(self) -> int: variableFilter=self._variableFilter, customBuildDirectory=self._customBuildDirectory, omhome=self._omhome, + build=False, ) sim_args_structure = {} @@ -1910,6 +1911,8 @@ def prepare(self) -> int: expression = f"setParameterValue({self._modelName}, {pk_structure}, {pk_value})" mod_structure.sendExpression(expression) + mod_structure.buildModel(variableFilter=self._variableFilter) + for idx_pc_simple, pc_simple in enumerate(param_simple_combinations): sim_args_simple = {} for idx_simple, pk_simple in enumerate(param_simple.keys()): From 979a026b2a680d21006649b9e4421278a2fcab67 Mon Sep 17 00:00:00 2001 From: syntron Date: Wed, 25 Jun 2025 21:05:30 +0200 Subject: [PATCH 07/29] [ModelicaSystemDoE] cleanup prepare() / rename variables --- OMPython/ModelicaSystem.py | 32 +++++++++++++++++++------------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/OMPython/ModelicaSystem.py b/OMPython/ModelicaSystem.py index 1bcb5f24..a28876d6 100644 --- a/OMPython/ModelicaSystem.py +++ b/OMPython/ModelicaSystem.py @@ -1897,9 +1897,9 @@ def prepare(self) -> int: build=False, ) - sim_args_structure = {} + sim_param_structure = {} for idx_structure, pk_structure in enumerate(param_structure.keys()): - sim_args_structure[pk_structure] = pc_structure[idx_structure] + sim_param_structure[pk_structure] = pc_structure[idx_structure] pk_value = pc_structure[idx_structure] if isinstance(pk_value, str): @@ -1909,19 +1909,22 @@ def prepare(self) -> int: expression = f"setParameterValue({self._modelName}, {pk_structure}, $Code(={pk_value_bool_str}));" else: expression = f"setParameterValue({self._modelName}, {pk_structure}, {pk_value})" - mod_structure.sendExpression(expression) + res = mod_structure.sendExpression(expression) + if not res: + raise ModelicaSystemError(f"Cannot set structural parameter {self._modelName}.{pk_structure} " + f"to {pk_value} using {repr(expression)}") mod_structure.buildModel(variableFilter=self._variableFilter) for idx_pc_simple, pc_simple in enumerate(param_simple_combinations): - sim_args_simple = {} + sim_param_simple = {} for idx_simple, pk_simple in enumerate(param_simple.keys()): - sim_args_simple[pk_simple] = str(pc_simple[idx_simple]) + sim_param_simple[pk_simple] = str(pc_simple[idx_simple]) resfilename = f"DOE_{idx_pc_structure:09d}_{idx_pc_simple:09d}.mat" logger.info(f"use result file {repr(resfilename)} " - f"for structural parameters: {sim_args_structure} " - f"and simple parameters: {sim_args_simple}") + f"for structural parameters: {sim_param_structure} " + f"and simple parameters: {sim_param_simple}") resultfile = self._resultpath / resfilename df_data = ( @@ -1931,24 +1934,27 @@ def prepare(self) -> int: self.DF_COLUMNS_RESULTFILENAME: resfilename, 'structural parameters ID': idx_pc_structure, } - | sim_args_structure + | sim_param_structure | { 'non-structural parameters ID': idx_pc_simple, } - | sim_args_simple + | sim_param_simple | { self.DF_COLUMNS_RESULTS_AVAILABLE: False, } ) - df_entries.append(pd.DataFrame.from_dict(df_data)) + df_entries.append(pd.DataFrame(data=df_data, index=[0])) - cmd = mod_structure.simulate_cmd( + mscmd = mod_structure.simulate_cmd( resultfile=resultfile.absolute().resolve(), - simargs={"override": sim_args_simple}, + timeout=self._timeout, ) + if self._simargs is not None: + mscmd.args_set(args=self._simargs) + mscmd.args_set(args={"override": sim_param_simple}) - self._sim_task_query.put(cmd) + self._sim_task_query.put(mscmd) self._sim_df = pd.concat(df_entries, ignore_index=True) From 25f5d6ab6451b2200b913bf06a039d45f199e8c0 Mon Sep 17 00:00:00 2001 From: syntron Date: Wed, 25 Jun 2025 21:06:26 +0200 Subject: [PATCH 08/29] [ModelicaSystemDoE] cleanup simulate() / rename variables --- OMPython/ModelicaSystem.py | 29 +++++++++++++++++++---------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/OMPython/ModelicaSystem.py b/OMPython/ModelicaSystem.py index a28876d6..22a33865 100644 --- a/OMPython/ModelicaSystem.py +++ b/OMPython/ModelicaSystem.py @@ -1968,44 +1968,53 @@ def get_doe(self) -> Optional[pd.DataFrame]: """ return self._sim_df - def simulate(self, num_workers: int = 3) -> None: + def simulate( + self, + num_workers: int = 3, + ) -> bool: """ Simulate the DoE using the defined number of workers. Returns True if all simulations were done successfully, else False. """ - sim_count_total = self._sim_task_query.qsize() + sim_query_total = self._sim_task_query.qsize() + if not isinstance(self._sim_df, pd.DataFrame): + raise ModelicaSystemError("Missing Doe Summary!") + sim_df_total = self._sim_df.shape[0] def worker(worker_id, task_queue): while True: - sim_data = {} + mscmd: Optional[ModelicaSystemCmd] = None try: # Get the next task from the queue - cmd: ModelicaSystemCmd = task_queue.get(block=False) + mscmd = task_queue.get(block=False) except queue.Empty: logger.info(f"[Worker {worker_id}] No more simulations to run.") break - resultfile = cmd.arg_get(key='r') + if mscmd is None: + raise ModelicaSystemError("Missing simulation definition!") + + resultfile = mscmd.arg_get(key='r') resultpath = pathlib.Path(resultfile) logger.info(f"[Worker {worker_id}] Performing task: {resultpath.name}") try: - sim_data['sim'].run() + mscmd.run() except ModelicaSystemError as ex: logger.warning(f"Simulation error for {resultpath.name}: {ex}") # Mark the task as done task_queue.task_done() - sim_count_done = sim_count_total - self._sim_task_query.qsize() + sim_query_done = sim_query_total - self._sim_task_query.qsize() logger.info(f"[Worker {worker_id}] Task completed: {resultpath.name} " - f"({sim_count_done}/{sim_count_total} = " - f"{sim_count_done / sim_count_total * 100:.2f}% of tasks left)") + f"({sim_query_done}/{sim_query_total} = " + f"{sim_query_done / sim_query_total * 100:.2f}% of tasks left)") - logger.info(f"Start simulations for DoE with {sim_count_total} simulations " + logger.info(f"Start simulations for DoE with {sim_query_total} simulations " f"using {num_workers} workers ...") # Create and start worker threads From ca33bc7cedc466cf42a86ac0fe0af60137e84d2a Mon Sep 17 00:00:00 2001 From: syntron Date: Wed, 25 Jun 2025 21:06:58 +0200 Subject: [PATCH 09/29] [ModelicaSystemDoE] cleanup get_solutions() / rename variables --- OMPython/ModelicaSystem.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/OMPython/ModelicaSystem.py b/OMPython/ModelicaSystem.py index 22a33865..21fcc443 100644 --- a/OMPython/ModelicaSystem.py +++ b/OMPython/ModelicaSystem.py @@ -2028,7 +2028,7 @@ def worker(worker_id, task_queue): for thread in threads: thread.join() - for idx, row in self._sim_df.to_dict('index').items(): + for row in self._sim_df.to_dict('records'): resultfilename = row[self.DF_COLUMNS_RESULTFILENAME] resultfile = self._resultpath / resultfilename @@ -2036,9 +2036,10 @@ def worker(worker_id, task_queue): mask = self._sim_df[self.DF_COLUMNS_RESULTFILENAME] == resultfilename self._sim_df.loc[mask, self.DF_COLUMNS_RESULTS_AVAILABLE] = True - sim_done = self._sim_df[self.DF_COLUMNS_RESULTS_AVAILABLE].sum() - sim_total = self._sim_df.shape[0] - logger.info(f"All workers finished ({sim_done} of {sim_total} simulations with a result file).") + sim_df_done = self._sim_df[self.DF_COLUMNS_RESULTS_AVAILABLE].sum() + logger.info(f"All workers finished ({sim_df_done} of {sim_df_total} simulations with a result file).") + + return sim_df_total == sim_df_done def get_solutions( self, From 02823ca5550ca8bdb89849da7470c0c6d9f2fc22 Mon Sep 17 00:00:00 2001 From: syntron Date: Wed, 25 Jun 2025 21:07:14 +0200 Subject: [PATCH 10/29] [test_ModelicaSystemDoE] update test --- tests/test_ModelicaSystemDoE.py | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/tests/test_ModelicaSystemDoE.py b/tests/test_ModelicaSystemDoE.py index a49ef111..90b36c51 100644 --- a/tests/test_ModelicaSystemDoE.py +++ b/tests/test_ModelicaSystemDoE.py @@ -1,3 +1,4 @@ +import numpy as np import OMPython import pandas as pd import pathlib @@ -37,7 +38,6 @@ def param_doe() -> dict[str, list]: def test_ModelicaSystemDoE(tmp_path, model_doe, param_doe): - tmpdir = tmp_path / 'DoE' tmpdir.mkdir(exist_ok=True) @@ -46,13 +46,32 @@ def test_ModelicaSystemDoE(tmp_path, model_doe, param_doe): modelName="M", parameters=param_doe, resultpath=tmpdir, + simargs={"override": {'stopTime': 1.0}}, ) mod_doe.prepare() df_doe = mod_doe.get_doe() assert isinstance(df_doe, pd.DataFrame) assert df_doe.shape[0] == 16 - assert df_doe['results available'].sum() == 16 + assert df_doe['results available'].sum() == 0 mod_doe.simulate() - sol = mod_doe.get_solutions(var_list=['x[1]', 'y[1]']) - assert len(sol) == 16 + assert df_doe['results available'].sum() == 16 + + for row in df_doe.to_dict('records'): + resultfilename = row[mod_doe.DF_COLUMNS_RESULTFILENAME] + resultfile = mod_doe._resultpath / resultfilename + + var_dict = { + # simple / non-structural parameters + 'a': float(row['a']), + 'b': float(row['b']), + # structural parameters + 'p': float(row['p']), + 'q': float(row['q']), + # variables using the structural parameters + f"x[{row['p']}]": float(row['a']), + f"y[{row['p']}]": float(row['b']), + } + sol = mod_doe._mod.getSolutions(resultfile=resultfile.as_posix(), varList=list(var_dict.keys())) + + assert np.isclose(sol[:, -1], np.array(list(var_dict.values()))).all() From cb212c20f57b0dfd77514e5dfff3139d4c91845c Mon Sep 17 00:00:00 2001 From: syntron Date: Wed, 25 Jun 2025 21:46:28 +0200 Subject: [PATCH 11/29] [ModelicaSystemDoE] add example to show the usage --- OMPython/ModelicaSystem.py | 56 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/OMPython/ModelicaSystem.py b/OMPython/ModelicaSystem.py index 21fcc443..26edb064 100644 --- a/OMPython/ModelicaSystem.py +++ b/OMPython/ModelicaSystem.py @@ -1800,6 +1800,62 @@ def getLinearStates(self) -> list[str]: class ModelicaSystemDoE: """ Class to run DoEs based on a (Open)Modelica model using ModelicaSystem + + Example + ------- + ``` + import OMPython + import pathlib + + + def run_doe(): + mypath = pathlib.Path('.') + + model = mypath / "M.mo" + model.write_text( + " model M\n" + " parameter Integer p=1;\n" + " parameter Integer q=1;\n" + " parameter Real a = -1;\n" + " parameter Real b = -1;\n" + " Real x[p];\n" + " Real y[q];\n" + " equation\n" + " der(x) = a * fill(1.0, p);\n" + " der(y) = b * fill(1.0, q);\n" + " end M;\n" + ) + + param = { + # structural + 'p': [1, 2], + 'q': [3, 4], + # simple + 'a': [5, 6], + 'b': [7, 8], + } + + resdir = mypath / 'DoE' + resdir.mkdir(exist_ok=True) + + mod_doe = OMPython.ModelicaSystemDoE( + fileName=model.as_posix(), + modelName="M", + parameters=param, + resultpath=resdir, + simargs={"override": {'stopTime': 1.0}}, + ) + mod_doe.prepare() + df_doe = mod_doe.get_doe() + mod_doe.simulate() + var_list = mod_doe.get_solutions() + sol_dict = mod_doe.get_solutions(var_list=var_list) + + + if __name__ == "__main__": + run_doe() + ``` + """ DF_COLUMNS_RESULTFILENAME: str = 'resultfilename' From 886a99abe106e720a6c15c1bb335485dfffca095 Mon Sep 17 00:00:00 2001 From: syntron Date: Wed, 25 Jun 2025 21:49:36 +0200 Subject: [PATCH 12/29] add pandas as new dependency (use in ModelicaSystemDoE) --- .pre-commit-config.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 484570b6..9eb5021f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -37,5 +37,6 @@ repos: additional_dependencies: - pyparsing - types-psutil + - pandas-stubs - pyzmq - numpy From 3ae803e4df3c25a102ada85d981f5aa66090adb2 Mon Sep 17 00:00:00 2001 From: syntron Date: Wed, 25 Jun 2025 22:07:20 +0200 Subject: [PATCH 13/29] [test_ModelicaSystemDoE] fix mypy --- tests/test_ModelicaSystemDoE.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_ModelicaSystemDoE.py b/tests/test_ModelicaSystemDoE.py index 90b36c51..288db522 100644 --- a/tests/test_ModelicaSystemDoE.py +++ b/tests/test_ModelicaSystemDoE.py @@ -4,11 +4,12 @@ import pathlib import pytest + @pytest.fixture def model_doe(tmp_path: pathlib.Path) -> pathlib.Path: # see: https://trac.openmodelica.org/OpenModelica/ticket/4052 mod = tmp_path / "M.mo" - mod.write_text(f""" + mod.write_text(""" model M parameter Integer p=1; parameter Integer q=1; From 55355cb92dcdeeb41e9d22bbf11505165f2d94fb Mon Sep 17 00:00:00 2001 From: syntron Date: Wed, 25 Jun 2025 22:11:28 +0200 Subject: [PATCH 14/29] add pandas to requirements in pyproject.toml --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index e82745c9..d3cd379b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,6 +17,7 @@ license = { file = "LICENSE" } requires-python = ">=3.10" dependencies = [ "numpy", + "pandas", "psutil", "pyparsing", "pyzmq", From 46667644209e34bc0ef8bf3d5330ec30bce05f7b Mon Sep 17 00:00:00 2001 From: syntron Date: Wed, 25 Jun 2025 22:43:07 +0200 Subject: [PATCH 15/29] [ModelicaSystemDoE] rename class constants --- OMPython/ModelicaSystem.py | 22 +++++++++++----------- tests/test_ModelicaSystemDoE.py | 2 +- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/OMPython/ModelicaSystem.py b/OMPython/ModelicaSystem.py index 26edb064..47251a44 100644 --- a/OMPython/ModelicaSystem.py +++ b/OMPython/ModelicaSystem.py @@ -1858,8 +1858,8 @@ def run_doe(): """ - DF_COLUMNS_RESULTFILENAME: str = 'resultfilename' - DF_COLUMNS_RESULTS_AVAILABLE: str = 'results available' + DF_COLUMNS_RESULT_FILENAME: str = 'result filename' + DF_COLUMNS_RESULT_AVAILABLE: str = 'result available' def __init__( self, @@ -1987,7 +1987,7 @@ def prepare(self) -> int: { 'ID structure': idx_pc_structure, 'ID simple': idx_pc_simple, - self.DF_COLUMNS_RESULTFILENAME: resfilename, + self.DF_COLUMNS_RESULT_FILENAME: resfilename, 'structural parameters ID': idx_pc_structure, } | sim_param_structure @@ -1996,7 +1996,7 @@ def prepare(self) -> int: } | sim_param_simple | { - self.DF_COLUMNS_RESULTS_AVAILABLE: False, + self.DF_COLUMNS_RESULT_AVAILABLE: False, } ) @@ -2085,14 +2085,14 @@ def worker(worker_id, task_queue): thread.join() for row in self._sim_df.to_dict('records'): - resultfilename = row[self.DF_COLUMNS_RESULTFILENAME] + resultfilename = row[self.DF_COLUMNS_RESULT_FILENAME] resultfile = self._resultpath / resultfilename if resultfile.exists(): - mask = self._sim_df[self.DF_COLUMNS_RESULTFILENAME] == resultfilename - self._sim_df.loc[mask, self.DF_COLUMNS_RESULTS_AVAILABLE] = True + mask = self._sim_df[self.DF_COLUMNS_RESULT_FILENAME] == resultfilename + self._sim_df.loc[mask, self.DF_COLUMNS_RESULT_AVAILABLE] = True - sim_df_done = self._sim_df[self.DF_COLUMNS_RESULTS_AVAILABLE].sum() + sim_df_done = self._sim_df[self.DF_COLUMNS_RESULT_AVAILABLE].sum() logger.info(f"All workers finished ({sim_df_done} of {sim_df_total} simulations with a result file).") return sim_df_total == sim_df_done @@ -2113,17 +2113,17 @@ def get_solutions( if self._sim_df is None: return None - if self._sim_df.shape[0] == 0 or self._sim_df[self.DF_COLUMNS_RESULTS_AVAILABLE].sum() == 0: + if self._sim_df.shape[0] == 0 or self._sim_df[self.DF_COLUMNS_RESULT_AVAILABLE].sum() == 0: raise ModelicaSystemError("No result files available - all simulations did fail?") if var_list is None: - resultfilename = self._sim_df[self.DF_COLUMNS_RESULTFILENAME].values[0] + resultfilename = self._sim_df[self.DF_COLUMNS_RESULT_FILENAME].values[0] resultfile = self._resultpath / resultfilename return self._mod.getSolutions(resultfile=resultfile) sol_dict: dict[str, pd.DataFrame | str] = {} for row in self._sim_df.to_dict('records'): - resultfilename = row[self.DF_COLUMNS_RESULTFILENAME] + resultfilename = row[self.DF_COLUMNS_RESULT_FILENAME] resultfile = self._resultpath / resultfilename try: diff --git a/tests/test_ModelicaSystemDoE.py b/tests/test_ModelicaSystemDoE.py index 288db522..7561dbf1 100644 --- a/tests/test_ModelicaSystemDoE.py +++ b/tests/test_ModelicaSystemDoE.py @@ -59,7 +59,7 @@ def test_ModelicaSystemDoE(tmp_path, model_doe, param_doe): assert df_doe['results available'].sum() == 16 for row in df_doe.to_dict('records'): - resultfilename = row[mod_doe.DF_COLUMNS_RESULTFILENAME] + resultfilename = row[mod_doe.DF_COLUMNS_RESULT_FILENAME] resultfile = mod_doe._resultpath / resultfilename var_dict = { From d4b4ed892ac8e6e8811ca17dbd248b0e8831e579 Mon Sep 17 00:00:00 2001 From: syntron Date: Sat, 28 Jun 2025 21:02:01 +0200 Subject: [PATCH 16/29] [ModelicaSystemDoE] remove dependency on pandas * no need to add aditional requirements * hint how to use pandas in the docstrings * update test to match code changes --- .pre-commit-config.yaml | 1 - OMPython/ModelicaSystem.py | 127 ++++++++++++++++++++------------ pyproject.toml | 1 - tests/test_ModelicaSystemDoE.py | 33 +++++---- 4 files changed, 97 insertions(+), 65 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9eb5021f..484570b6 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -37,6 +37,5 @@ repos: additional_dependencies: - pyparsing - types-psutil - - pandas-stubs - pyzmq - numpy diff --git a/OMPython/ModelicaSystem.py b/OMPython/ModelicaSystem.py index 47251a44..3027b27d 100644 --- a/OMPython/ModelicaSystem.py +++ b/OMPython/ModelicaSystem.py @@ -39,7 +39,6 @@ import numbers import numpy as np import os -import pandas as pd import queue import textwrap import threading @@ -1838,18 +1837,19 @@ def run_doe(): resdir = mypath / 'DoE' resdir.mkdir(exist_ok=True) - mod_doe = OMPython.ModelicaSystemDoE( + doe_mod = OMPython.ModelicaSystemDoE( fileName=model.as_posix(), modelName="M", parameters=param, resultpath=resdir, simargs={"override": {'stopTime': 1.0}}, ) - mod_doe.prepare() - df_doe = mod_doe.get_doe() - mod_doe.simulate() - var_list = mod_doe.get_solutions() - sol_dict = mod_doe.get_solutions(var_list=var_list) + doe_mod.prepare() + doe_dict = doe_mod.get_doe() + doe_mod.simulate() + doe_sol = doe_mod.get_solutions() + + # ... work with doe_df and doe_sol ... if __name__ == "__main__": @@ -1915,7 +1915,7 @@ def __init__( else: self._parameters = {} - self._sim_df: Optional[pd.DataFrame] = None + self._sim_dict: Optional[dict[str, dict[str, Any]]] = None self._sim_task_query: queue.Queue = queue.Queue() def prepare(self) -> int: @@ -1940,7 +1940,7 @@ def prepare(self) -> int: param_structure_combinations = list(itertools.product(*param_structure.values())) param_simple_combinations = list(itertools.product(*param_simple.values())) - df_entries: list[pd.DataFrame] = [] + self._sim_dict = {} for idx_pc_structure, pc_structure in enumerate(param_structure_combinations): mod_structure = ModelicaSystem( fileName=self._fileName, @@ -1986,13 +1986,10 @@ def prepare(self) -> int: df_data = ( { 'ID structure': idx_pc_structure, - 'ID simple': idx_pc_simple, - self.DF_COLUMNS_RESULT_FILENAME: resfilename, - 'structural parameters ID': idx_pc_structure, } | sim_param_structure | { - 'non-structural parameters ID': idx_pc_simple, + 'ID non-structure': idx_pc_simple, } | sim_param_simple | { @@ -2000,7 +1997,7 @@ def prepare(self) -> int: } ) - df_entries.append(pd.DataFrame(data=df_data, index=[0])) + self._sim_dict[resfilename] = df_data mscmd = mod_structure.simulate_cmd( resultfile=resultfile.absolute().resolve(), @@ -2012,17 +2009,26 @@ def prepare(self) -> int: self._sim_task_query.put(mscmd) - self._sim_df = pd.concat(df_entries, ignore_index=True) - - logger.info(f"Prepared {self._sim_df.shape[0]} simulation definitions for the defined DoE.") + logger.info(f"Prepared {self._sim_task_query.qsize()} simulation definitions for the defined DoE.") - return self._sim_df.shape[0] + return self._sim_task_query.qsize() - def get_doe(self) -> Optional[pd.DataFrame]: + def get_doe(self) -> Optional[dict[str, dict[str, Any]]]: """ - Get the defined Doe as a poandas dataframe. + Get the defined DoE as a dict, where each key is the result filename and the value is a dict of simulation + settings including structural and non-structural parameters. + + The following code snippet can be used to convert the data to a pandas dataframe: + + ``` + import pandas as pd + + doe_dict = doe_mod.get_doe() + doe_df = pd.DataFrame.from_dict(data=doe_dict, orient='index') + ``` + """ - return self._sim_df + return self._sim_dict def simulate( self, @@ -2035,9 +2041,9 @@ def simulate( """ sim_query_total = self._sim_task_query.qsize() - if not isinstance(self._sim_df, pd.DataFrame): + if not isinstance(self._sim_dict, dict) or len(self._sim_dict) == 0: raise ModelicaSystemError("Missing Doe Summary!") - sim_df_total = self._sim_df.shape[0] + sim_dict_total = len(self._sim_dict) def worker(worker_id, task_queue): while True: @@ -2084,55 +2090,78 @@ def worker(worker_id, task_queue): for thread in threads: thread.join() - for row in self._sim_df.to_dict('records'): - resultfilename = row[self.DF_COLUMNS_RESULT_FILENAME] + sim_dict_done = 0 + for resultfilename in self._sim_dict: resultfile = self._resultpath / resultfilename - if resultfile.exists(): - mask = self._sim_df[self.DF_COLUMNS_RESULT_FILENAME] == resultfilename - self._sim_df.loc[mask, self.DF_COLUMNS_RESULT_AVAILABLE] = True + # include check for an empty (=> 0B) result file which indicates a crash of the model executable + # see: https://github.com/OpenModelica/OMPython/issues/261 + # https://github.com/OpenModelica/OpenModelica/issues/13829 + if resultfile.is_file() and resultfile.stat().st_size > 0: + self._sim_dict[resultfilename][self.DF_COLUMNS_RESULTS_AVAILABLE] = True + sim_dict_done += 1 - sim_df_done = self._sim_df[self.DF_COLUMNS_RESULT_AVAILABLE].sum() - logger.info(f"All workers finished ({sim_df_done} of {sim_df_total} simulations with a result file).") + logger.info(f"All workers finished ({sim_dict_done} of {sim_dict_total} simulations with a result file).") - return sim_df_total == sim_df_done + return sim_dict_total == sim_dict_done def get_solutions( self, var_list: Optional[list] = None, - ) -> Optional[tuple[str] | dict[str, pd.DataFrame | str]]: + ) -> Optional[tuple[str] | dict[str, dict[str, np.ndarray]]]: """ Get all solutions of the DoE run. The following return values are possible: - * None, if there no simulation was run - * A list of variables if val_list == None * The Solutions as dict[str, pd.DataFrame] if a value list (== val_list) is defined. + + The following code snippet can be used to convert the solution data for each run to a pandas dataframe: + + ``` + import pandas as pd + + doe_sol = doe_mod.get_solutions() + for key in doe_sol: + data = doe_sol[key]['data'] + if data: + doe_sol[key]['df'] = pd.DataFrame.from_dict(data=data) + else: + doe_sol[key]['df'] = None + ``` + """ - if self._sim_df is None: + if not isinstance(self._sim_dict, dict): return None - if self._sim_df.shape[0] == 0 or self._sim_df[self.DF_COLUMNS_RESULT_AVAILABLE].sum() == 0: + if len(self._sim_dict) == 0: raise ModelicaSystemError("No result files available - all simulations did fail?") - if var_list is None: - resultfilename = self._sim_df[self.DF_COLUMNS_RESULT_FILENAME].values[0] + sol_dict: dict[str, dict[str, Any]] = {} + for resultfilename in self._sim_dict: resultfile = self._resultpath / resultfilename - return self._mod.getSolutions(resultfile=resultfile) - sol_dict: dict[str, pd.DataFrame | str] = {} - for row in self._sim_df.to_dict('records'): - resultfilename = row[self.DF_COLUMNS_RESULT_FILENAME] - resultfile = self._resultpath / resultfilename + sol_dict[resultfilename] = {} + + if self._sim_dict[resultfilename][self.DF_COLUMNS_RESULTS_AVAILABLE] != True: + sol_dict[resultfilename]['msg'] = 'No result file available!' + sol_dict[resultfilename]['data'] = {} + continue + + if var_list is None: + var_list_row = list(self._mod.getSolutions(resultfile=resultfile)) + else: + var_list_row = var_list try: - sol = self._mod.getSolutions(varList=var_list, resultfile=resultfile) - sol_data = {var: sol[idx] for idx, var in var_list} - sol_df = pd.DataFrame(sol_data) - sol_dict[resultfilename] = sol_df + sol = self._mod.getSolutions(varList=var_list_row, resultfile=resultfile) + sol_data = {var: sol[idx] for idx, var in enumerate(var_list_row)} + sol_dict[resultfilename]['msg'] = 'Simulation available' + sol_dict[resultfilename]['data'] = sol_data except ModelicaSystemError as ex: - logger.warning(f"No solution for {resultfilename}: {ex}") - sol_dict[resultfilename] = str(ex) + msg = f"Error reading solution for {resultfilename}: {ex}" + logger.warning(msg) + sol_dict[resultfilename]['msg'] = msg + sol_dict[resultfilename]['data'] = {} return sol_dict diff --git a/pyproject.toml b/pyproject.toml index d3cd379b..e82745c9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,7 +17,6 @@ license = { file = "LICENSE" } requires-python = ">=3.10" dependencies = [ "numpy", - "pandas", "psutil", "pyparsing", "pyzmq", diff --git a/tests/test_ModelicaSystemDoE.py b/tests/test_ModelicaSystemDoE.py index 7561dbf1..40fed90d 100644 --- a/tests/test_ModelicaSystemDoE.py +++ b/tests/test_ModelicaSystemDoE.py @@ -1,6 +1,5 @@ import numpy as np import OMPython -import pandas as pd import pathlib import pytest @@ -42,25 +41,30 @@ def test_ModelicaSystemDoE(tmp_path, model_doe, param_doe): tmpdir = tmp_path / 'DoE' tmpdir.mkdir(exist_ok=True) - mod_doe = OMPython.ModelicaSystemDoE( + doe_mod = OMPython.ModelicaSystemDoE( fileName=model_doe.as_posix(), modelName="M", parameters=param_doe, resultpath=tmpdir, simargs={"override": {'stopTime': 1.0}}, ) - mod_doe.prepare() - df_doe = mod_doe.get_doe() - assert isinstance(df_doe, pd.DataFrame) - assert df_doe.shape[0] == 16 - assert df_doe['results available'].sum() == 0 + doe_count = doe_mod.prepare() + assert doe_count == 16 - mod_doe.simulate() - assert df_doe['results available'].sum() == 16 + doe_dict = doe_mod.get_doe() + assert isinstance(doe_dict, dict) + assert len(doe_dict.keys()) == 16 - for row in df_doe.to_dict('records'): - resultfilename = row[mod_doe.DF_COLUMNS_RESULT_FILENAME] - resultfile = mod_doe._resultpath / resultfilename + doe_status = doe_mod.simulate() + assert doe_status is True + + doe_sol = doe_mod.get_solutions() + + for resultfilename in doe_dict: + row = doe_dict[resultfilename] + + assert resultfilename in doe_sol + sol = doe_sol[resultfilename] var_dict = { # simple / non-structural parameters @@ -73,6 +77,7 @@ def test_ModelicaSystemDoE(tmp_path, model_doe, param_doe): f"x[{row['p']}]": float(row['a']), f"y[{row['p']}]": float(row['b']), } - sol = mod_doe._mod.getSolutions(resultfile=resultfile.as_posix(), varList=list(var_dict.keys())) - assert np.isclose(sol[:, -1], np.array(list(var_dict.values()))).all() + for var in var_dict: + assert var in sol['data'] + assert np.isclose(sol['data'][var][-1], var_dict[var]) From a8fc61b4e39772cb6163e65ee4f7edfafe5da580 Mon Sep 17 00:00:00 2001 From: syntron Date: Sat, 28 Jun 2025 21:05:04 +0200 Subject: [PATCH 17/29] [ModelicaSystemDoE.simulate] fix percent of tasks left --- OMPython/ModelicaSystem.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/OMPython/ModelicaSystem.py b/OMPython/ModelicaSystem.py index 3027b27d..315e6373 100644 --- a/OMPython/ModelicaSystem.py +++ b/OMPython/ModelicaSystem.py @@ -2073,8 +2073,8 @@ def worker(worker_id, task_queue): sim_query_done = sim_query_total - self._sim_task_query.qsize() logger.info(f"[Worker {worker_id}] Task completed: {resultpath.name} " - f"({sim_query_done}/{sim_query_total} = " - f"{sim_query_done / sim_query_total * 100:.2f}% of tasks left)") + f"({sim_query_total - sim_query_done}/{sim_query_total} = " + f"{(sim_query_total - sim_query_done) / sim_query_total * 100:.2f}% of tasks left)") logger.info(f"Start simulations for DoE with {sim_query_total} simulations " f"using {num_workers} workers ...") From 20cc059f721e12c86830bd8debb3a02dd7627acf Mon Sep 17 00:00:00 2001 From: syntron Date: Sat, 28 Jun 2025 21:05:34 +0200 Subject: [PATCH 18/29] [ModelicaSystemDoE.prepare] do not convert all non-structural parameters to string --- OMPython/ModelicaSystem.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/OMPython/ModelicaSystem.py b/OMPython/ModelicaSystem.py index 315e6373..3e57a06e 100644 --- a/OMPython/ModelicaSystem.py +++ b/OMPython/ModelicaSystem.py @@ -1975,7 +1975,7 @@ def prepare(self) -> int: for idx_pc_simple, pc_simple in enumerate(param_simple_combinations): sim_param_simple = {} for idx_simple, pk_simple in enumerate(param_simple.keys()): - sim_param_simple[pk_simple] = str(pc_simple[idx_simple]) + sim_param_simple[pk_simple] = pc_simple[idx_simple] resfilename = f"DOE_{idx_pc_structure:09d}_{idx_pc_simple:09d}.mat" logger.info(f"use result file {repr(resfilename)} " From 70601aa74e52a937dfc8507980f88a8d591f985b Mon Sep 17 00:00:00 2001 From: syntron Date: Sat, 28 Jun 2025 21:05:59 +0200 Subject: [PATCH 19/29] [ModelicaSystemDoE] update set parameter expressions for str and bool --- OMPython/ModelicaSystem.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/OMPython/ModelicaSystem.py b/OMPython/ModelicaSystem.py index 3e57a06e..f9941e42 100644 --- a/OMPython/ModelicaSystem.py +++ b/OMPython/ModelicaSystem.py @@ -1959,10 +1959,10 @@ def prepare(self) -> int: pk_value = pc_structure[idx_structure] if isinstance(pk_value, str): - expression = f"setParameterValue({self._modelName}, {pk_structure}, $Code(=\"{pk_value}\"))" + expression = f"setParameterValue({self._modelName}, {pk_structure}, \"{pk_value}\")" elif isinstance(pk_value, bool): pk_value_bool_str = "true" if pk_value else "false" - expression = f"setParameterValue({self._modelName}, {pk_structure}, $Code(={pk_value_bool_str}));" + expression = f"setParameterValue({self._modelName}, {pk_structure}, {pk_value_bool_str});" else: expression = f"setParameterValue({self._modelName}, {pk_structure}, {pk_value})" res = mod_structure.sendExpression(expression) From d0a660692e36c474effc2e0ace0a5864e254ef9e Mon Sep 17 00:00:00 2001 From: syntron Date: Sat, 28 Jun 2025 21:10:17 +0200 Subject: [PATCH 20/29] [ModelicaSystemDoE] rename class constants --- OMPython/ModelicaSystem.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/OMPython/ModelicaSystem.py b/OMPython/ModelicaSystem.py index f9941e42..3624b28b 100644 --- a/OMPython/ModelicaSystem.py +++ b/OMPython/ModelicaSystem.py @@ -1858,8 +1858,8 @@ def run_doe(): """ - DF_COLUMNS_RESULT_FILENAME: str = 'result filename' - DF_COLUMNS_RESULT_AVAILABLE: str = 'result available' + DICT_RESULT_FILENAME: str = 'result filename' + DICT_RESULT_AVAILABLE: str = 'result available' def __init__( self, @@ -1993,7 +1993,7 @@ def prepare(self) -> int: } | sim_param_simple | { - self.DF_COLUMNS_RESULT_AVAILABLE: False, + self.DICT_RESULT_AVAILABLE: False, } ) @@ -2098,7 +2098,7 @@ def worker(worker_id, task_queue): # see: https://github.com/OpenModelica/OMPython/issues/261 # https://github.com/OpenModelica/OpenModelica/issues/13829 if resultfile.is_file() and resultfile.stat().st_size > 0: - self._sim_dict[resultfilename][self.DF_COLUMNS_RESULTS_AVAILABLE] = True + self._sim_dict[resultfilename][self.DICT_RESULT_AVAILABLE] = True sim_dict_done += 1 logger.info(f"All workers finished ({sim_dict_done} of {sim_dict_total} simulations with a result file).") @@ -2143,7 +2143,7 @@ def get_solutions( sol_dict[resultfilename] = {} - if self._sim_dict[resultfilename][self.DF_COLUMNS_RESULTS_AVAILABLE] != True: + if self._sim_dict[resultfilename][self.DICT_RESULT_AVAILABLE] != True: sol_dict[resultfilename]['msg'] = 'No result file available!' sol_dict[resultfilename]['data'] = {} continue From b0eee45522763ebcbc92901214e7a2c2416a995f Mon Sep 17 00:00:00 2001 From: syntron Date: Sat, 28 Jun 2025 21:12:10 +0200 Subject: [PATCH 21/29] [ModelicaSystemDoE] fix bool comparison --- OMPython/ModelicaSystem.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/OMPython/ModelicaSystem.py b/OMPython/ModelicaSystem.py index 3624b28b..d2acea13 100644 --- a/OMPython/ModelicaSystem.py +++ b/OMPython/ModelicaSystem.py @@ -2143,7 +2143,7 @@ def get_solutions( sol_dict[resultfilename] = {} - if self._sim_dict[resultfilename][self.DICT_RESULT_AVAILABLE] != True: + if not self._sim_dict[resultfilename][self.DICT_RESULT_AVAILABLE]: sol_dict[resultfilename]['msg'] = 'No result file available!' sol_dict[resultfilename]['data'] = {} continue From 26db60e583f761a7b1c36293476d4cab077f8b92 Mon Sep 17 00:00:00 2001 From: syntron Date: Sat, 28 Jun 2025 21:13:17 +0200 Subject: [PATCH 22/29] [ModelicaSystemDoE] remove unused code --- OMPython/ModelicaSystem.py | 1 - 1 file changed, 1 deletion(-) diff --git a/OMPython/ModelicaSystem.py b/OMPython/ModelicaSystem.py index d2acea13..3844a6ab 100644 --- a/OMPython/ModelicaSystem.py +++ b/OMPython/ModelicaSystem.py @@ -2047,7 +2047,6 @@ def simulate( def worker(worker_id, task_queue): while True: - mscmd: Optional[ModelicaSystemCmd] = None try: # Get the next task from the queue mscmd = task_queue.get(block=False) From ba930d69f28acece941def2c6aedd0eca373df97 Mon Sep 17 00:00:00 2001 From: syntron Date: Wed, 9 Jul 2025 21:26:57 +0200 Subject: [PATCH 23/29] [ModelicaSystemDoE] fix rebase fallout --- OMPython/ModelicaSystem.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/OMPython/ModelicaSystem.py b/OMPython/ModelicaSystem.py index 3844a6ab..1083df1e 100644 --- a/OMPython/ModelicaSystem.py +++ b/OMPython/ModelicaSystem.py @@ -2000,7 +2000,7 @@ def prepare(self) -> int: self._sim_dict[resfilename] = df_data mscmd = mod_structure.simulate_cmd( - resultfile=resultfile.absolute().resolve(), + result_file=resultfile.absolute().resolve(), timeout=self._timeout, ) if self._simargs is not None: @@ -2148,12 +2148,12 @@ def get_solutions( continue if var_list is None: - var_list_row = list(self._mod.getSolutions(resultfile=resultfile)) + var_list_row = list(self._mod.getSolutions(resultfile=resultfile.as_posix())) else: var_list_row = var_list try: - sol = self._mod.getSolutions(varList=var_list_row, resultfile=resultfile) + sol = self._mod.getSolutions(varList=var_list_row, resultfile=resultfile.as_posix()) sol_data = {var: sol[idx] for idx, var in enumerate(var_list_row)} sol_dict[resultfilename]['msg'] = 'Simulation available' sol_dict[resultfilename]['data'] = sol_data From 468a095e6dedb392c3057e99d54ef669b2df75e8 Mon Sep 17 00:00:00 2001 From: syntron Date: Wed, 15 Oct 2025 23:33:56 +0200 Subject: [PATCH 24/29] [ModelicaSystemDoE] fix rebase fallout --- OMPython/ModelicaSystem.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/OMPython/ModelicaSystem.py b/OMPython/ModelicaSystem.py index 1083df1e..6f3d5e6e 100644 --- a/OMPython/ModelicaSystem.py +++ b/OMPython/ModelicaSystem.py @@ -42,7 +42,7 @@ import queue import textwrap import threading -from typing import Any, Optional +from typing import Any, cast, Optional import warnings import xml.etree.ElementTree as ET @@ -1866,12 +1866,12 @@ def __init__( fileName: Optional[str | os.PathLike | pathlib.Path] = None, modelName: Optional[str] = None, lmodel: Optional[list[str | tuple[str, str]]] = None, - commandLineOptions: Optional[str] = None, + commandLineOptions: Optional[list[str]] = None, variableFilter: Optional[str] = None, customBuildDirectory: Optional[str | os.PathLike | pathlib.Path] = None, omhome: Optional[str] = None, - simargs: Optional[dict[str, Optional[str | dict[str, str]]]] = None, + simargs: Optional[dict[str, Optional[str | dict[str, str] | numbers.Number]]] = None, timeout: Optional[int] = None, resultpath: Optional[pathlib.Path] = None, @@ -1975,7 +1975,7 @@ def prepare(self) -> int: for idx_pc_simple, pc_simple in enumerate(param_simple_combinations): sim_param_simple = {} for idx_simple, pk_simple in enumerate(param_simple.keys()): - sim_param_simple[pk_simple] = pc_simple[idx_simple] + sim_param_simple[pk_simple] = cast(Any, pc_simple[idx_simple]) resfilename = f"DOE_{idx_pc_structure:09d}_{idx_pc_simple:09d}.mat" logger.info(f"use result file {repr(resfilename)} " From f5517cf5cc330d1681a8ef14f7ffa4cf847e0eba Mon Sep 17 00:00:00 2001 From: syntron Date: Tue, 4 Nov 2025 11:23:52 +0100 Subject: [PATCH 25/29] [ModelicaSystemDoE] cleanup & extend & document dict key constants * remove DICT_RESULT_FILENAME * add comment * add DICT_ID_STRUCTURE and DICT_ID_NON_STRUCTURE * rename param_simple => param_non_structure --- OMPython/ModelicaSystem.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/OMPython/ModelicaSystem.py b/OMPython/ModelicaSystem.py index 6f3d5e6e..af8af29b 100644 --- a/OMPython/ModelicaSystem.py +++ b/OMPython/ModelicaSystem.py @@ -1858,7 +1858,10 @@ def run_doe(): """ - DICT_RESULT_FILENAME: str = 'result filename' + # Dictionary keys used in simulation dict (see _sim_dict or get_doe()). These dict keys contain a space and, thus, + # cannot be used as OM variable identifiers. They are defined here as reference for any evaluation of the data. + DICT_ID_STRUCTURE: str = 'ID structure' + DICT_ID_NON_STRUCTURE: str = 'ID non-structure' DICT_RESULT_AVAILABLE: str = 'result available' def __init__( @@ -1927,18 +1930,18 @@ def prepare(self) -> int: """ param_structure = {} - param_simple = {} + param_non_structure = {} for param_name in self._parameters.keys(): changeable = self._mod.isParameterChangeable(name=param_name) logger.info(f"Parameter {repr(param_name)} is changeable? {changeable}") if changeable: - param_simple[param_name] = self._parameters[param_name] + param_non_structure[param_name] = self._parameters[param_name] else: param_structure[param_name] = self._parameters[param_name] param_structure_combinations = list(itertools.product(*param_structure.values())) - param_simple_combinations = list(itertools.product(*param_simple.values())) + param_simple_combinations = list(itertools.product(*param_non_structure.values())) self._sim_dict = {} for idx_pc_structure, pc_structure in enumerate(param_structure_combinations): @@ -1974,7 +1977,7 @@ def prepare(self) -> int: for idx_pc_simple, pc_simple in enumerate(param_simple_combinations): sim_param_simple = {} - for idx_simple, pk_simple in enumerate(param_simple.keys()): + for idx_simple, pk_simple in enumerate(param_non_structure.keys()): sim_param_simple[pk_simple] = cast(Any, pc_simple[idx_simple]) resfilename = f"DOE_{idx_pc_structure:09d}_{idx_pc_simple:09d}.mat" @@ -1985,11 +1988,11 @@ def prepare(self) -> int: df_data = ( { - 'ID structure': idx_pc_structure, + self.DICT_ID_STRUCTURE: idx_pc_structure, } | sim_param_structure | { - 'ID non-structure': idx_pc_simple, + self.DICT_ID_NON_STRUCTURE: idx_pc_simple, } | sim_param_simple | { From 94d04138e8e0b018fc60cbbc3202afb79ef97e94 Mon Sep 17 00:00:00 2001 From: syntron Date: Tue, 4 Nov 2025 11:25:27 +0100 Subject: [PATCH 26/29] [ModelicaSystemDoE] ensure any double quote in string variables is escaped --- OMPython/ModelicaSystem.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/OMPython/ModelicaSystem.py b/OMPython/ModelicaSystem.py index af8af29b..eb6b1d27 100644 --- a/OMPython/ModelicaSystem.py +++ b/OMPython/ModelicaSystem.py @@ -1962,7 +1962,8 @@ def prepare(self) -> int: pk_value = pc_structure[idx_structure] if isinstance(pk_value, str): - expression = f"setParameterValue({self._modelName}, {pk_structure}, \"{pk_value}\")" + pk_value_str = pk_value.replace('"', '\\"') + expression = f"setParameterValue({self._modelName}, {pk_structure}, \"{pk_value_str}\")" elif isinstance(pk_value, bool): pk_value_bool_str = "true" if pk_value else "false" expression = f"setParameterValue({self._modelName}, {pk_structure}, {pk_value_bool_str});" From be9842a6398bee481273fafc4965ea64996a4c39 Mon Sep 17 00:00:00 2001 From: syntron Date: Tue, 4 Nov 2025 11:41:11 +0100 Subject: [PATCH 27/29] [ModelicaSystemDoE] replace pathlib by OMCPath --- OMPython/ModelicaSystem.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/OMPython/ModelicaSystem.py b/OMPython/ModelicaSystem.py index eb6b1d27..899492f2 100644 --- a/OMPython/ModelicaSystem.py +++ b/OMPython/ModelicaSystem.py @@ -1866,18 +1866,18 @@ def run_doe(): def __init__( self, - fileName: Optional[str | os.PathLike | pathlib.Path] = None, + fileName: Optional[str | os.PathLike] = None, modelName: Optional[str] = None, lmodel: Optional[list[str | tuple[str, str]]] = None, commandLineOptions: Optional[list[str]] = None, variableFilter: Optional[str] = None, - customBuildDirectory: Optional[str | os.PathLike | pathlib.Path] = None, + customBuildDirectory: Optional[str | os.PathLike] = None, omhome: Optional[str] = None, simargs: Optional[dict[str, Optional[str | dict[str, str] | numbers.Number]]] = None, timeout: Optional[int] = None, - resultpath: Optional[pathlib.Path] = None, + resultpath: Optional[str | os.PathLike] = None, parameters: Optional[dict[str, list[str] | list[int] | list[float]]] = None, ) -> None: """ @@ -1908,10 +1908,13 @@ def __init__( self._simargs = simargs self._timeout = timeout - if isinstance(resultpath, pathlib.Path): - self._resultpath = resultpath + if resultpath is not None: + self._resultpath = self._mod._getconn.omcpath(resultpath) else: - self._resultpath = pathlib.Path('.') + self._resultpath = self._mod._getconn.omcpath_tempdir() + + if not self._resultpath.is_dir(): + raise ModelicaSystemError(f"Resultpath {self._resultpath.as_posix()} does not exists!") if isinstance(parameters, dict): self._parameters = parameters @@ -2062,7 +2065,7 @@ def worker(worker_id, task_queue): raise ModelicaSystemError("Missing simulation definition!") resultfile = mscmd.arg_get(key='r') - resultpath = pathlib.Path(resultfile) + resultpath = self._mod._getconn.omcpath(resultfile) logger.info(f"[Worker {worker_id}] Performing task: {resultpath.name}") @@ -2100,7 +2103,7 @@ def worker(worker_id, task_queue): # include check for an empty (=> 0B) result file which indicates a crash of the model executable # see: https://github.com/OpenModelica/OMPython/issues/261 # https://github.com/OpenModelica/OpenModelica/issues/13829 - if resultfile.is_file() and resultfile.stat().st_size > 0: + if resultfile.is_file() and resultfile.size() > 0: self._sim_dict[resultfilename][self.DICT_RESULT_AVAILABLE] = True sim_dict_done += 1 From ad916299d45784de6c5db64b35eb5abeead7bbaa Mon Sep 17 00:00:00 2001 From: syntron Date: Tue, 4 Nov 2025 11:46:37 +0100 Subject: [PATCH 28/29] [ModelicaSystem/ModelicaSystemDoE] improve session handling * add ModelicaSystem.session() - returns _getconn * add ModelicaSystemDoE.session() - returns _mod.session() reasoning: * do not access private variables of a class * limit chain access to (sub)data --- OMPython/ModelicaSystem.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/OMPython/ModelicaSystem.py b/OMPython/ModelicaSystem.py index 899492f2..aa399426 100644 --- a/OMPython/ModelicaSystem.py +++ b/OMPython/ModelicaSystem.py @@ -440,6 +440,12 @@ def __init__( if build: self.buildModel(variableFilter) + def session(self) -> OMCSessionZMQ: + """ + Return the OMC session used for this class. + """ + return self._getconn + def setCommandLineOptions(self, commandLineOptions: str): """ Set the provided command line option via OMC setCommandLineOptions(). @@ -1909,9 +1915,9 @@ def __init__( self._timeout = timeout if resultpath is not None: - self._resultpath = self._mod._getconn.omcpath(resultpath) + self._resultpath = self.session().omcpath(resultpath) else: - self._resultpath = self._mod._getconn.omcpath_tempdir() + self._resultpath = self.session().omcpath_tempdir() if not self._resultpath.is_dir(): raise ModelicaSystemError(f"Resultpath {self._resultpath.as_posix()} does not exists!") @@ -1924,6 +1930,12 @@ def __init__( self._sim_dict: Optional[dict[str, dict[str, Any]]] = None self._sim_task_query: queue.Queue = queue.Queue() + def session(self) -> OMCSessionZMQ: + """ + Return the OMC session used for this class. + """ + return self._mod.session() + def prepare(self) -> int: """ Prepare the DoE by evaluating the parameters. Each structural parameter requires a new instance of @@ -2065,7 +2077,7 @@ def worker(worker_id, task_queue): raise ModelicaSystemError("Missing simulation definition!") resultfile = mscmd.arg_get(key='r') - resultpath = self._mod._getconn.omcpath(resultfile) + resultpath = self.session().omcpath(resultfile) logger.info(f"[Worker {worker_id}] Performing task: {resultpath.name}") From d37f6ffcce354b15b77c645d1f402c4928a3b496 Mon Sep 17 00:00:00 2001 From: syntron Date: Tue, 4 Nov 2025 18:16:43 +0100 Subject: [PATCH 29/29] [ModelicaSystemDoE] fix path to resultfile it does not exists at this point thus, resolve() and absolute() will fail --- OMPython/ModelicaSystem.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/OMPython/ModelicaSystem.py b/OMPython/ModelicaSystem.py index aa399426..8ab8c901 100644 --- a/OMPython/ModelicaSystem.py +++ b/OMPython/ModelicaSystem.py @@ -2019,7 +2019,7 @@ def prepare(self) -> int: self._sim_dict[resfilename] = df_data mscmd = mod_structure.simulate_cmd( - result_file=resultfile.absolute().resolve(), + result_file=resultfile, timeout=self._timeout, ) if self._simargs is not None: