feat: RF GUI <-> python client interoperabilty
#2738
Merged
Conversation
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
daquinteroflex
force-pushed
the
dario/rf_endpoints_update
branch
2 times, most recently
from
1758e24 to
69b80ea
Compare
daquinteroflex
force-pushed
the
dario/rf_endpoints_update
branch
5 times, most recently
from
d995c70 to
52a51fa
Compare
4 tasks
daquinteroflex
changed the title
RF GUI <-> python client interoperabilty
daquinteroflex
force-pushed
the
dario/rf_endpoints_update
branch
2 times, most recently
from
425409d to
c0043f2
Compare
daquinteroflex
requested review from
yaugenst-flex and
momchil-flex
and removed request for
yaugenst-flex
daquinteroflex
force-pushed
the
dario/rf_endpoints_update
branch
5 times, most recently
from
25eccbf to
d14430d
Compare
dmarek-flex
approved these changes
Aug 27, 2025
|
Time for the greptile barrage |
![greptile-apps[bot]](https://avatars.githubusercontent.com/in/867647?s=60&v=4){kind=small}
Diff CoverageDiff: origin/develop...HEAD, staged and unstaged changes
Summary
tidy3d/components/index.pyLines 64-72 64 If the lengths of the 'keys' and 'values' tuples are not equal.
65 """
66 keys, values = data.get("keys"), data.get("values")
67 if keys is not None and values is not None and len(keys) != len(values):
! 68 raise ValueError("Length of 'keys' and 'values' must be the same.")
69 return data
70
71 def __getitem__(self, key: str) -> Any:
72 """Retrieves a `Simulation` object by its corresponding key.tidy3d/plugins/smatrix/data/data_array.pyLines 25-37 25 _dims = ("f", "port")
26
27 @pd.root_validator(pre=False)
28 def _warn_rf_license(cls, values):
! 29 log.warning(
30 "ℹ️ ⚠️ RF simulations are subject to new license requirements in the future. You have instantiated at least one RF-specific component.",
31 log_once=True,
32 )
! 33 return values
34
35
36 class ModalPortDataArray(DataArray):
37 """Port parameter matrix elements for modal ports.Lines 78-87 78 _data_attrs = {"long_name": "terminal-based port matrix element"}
79
80 @pd.root_validator(pre=False)
81 def _warn_rf_license(cls, values):
! 82 log.warning(
83 "ℹ️ ⚠️ RF simulations are subject to new license requirements in the future. You have instantiated at least one RF-specific component.",
84 log_once=True,
85 )
! 86 return valuestidy3d/plugins/smatrix/run.pyLines 20-28 20
21 def compose_simulation_data_index(port_task_map: dict[str, str]) -> SimulationDataMap:
22 port_data_dict = {}
23 for _, _ in port_task_map.items():
! 24 pass
25 # FIXME: get simulationdata for each port
26 # port_data_dict[port] = sim_data_i
27
28 return SimulationDataMap(Lines 73-82 73 -------
74 ModalComponentModelerData
75 An object containing the results mapped to their respective ports.
76 """
! 77 port_simulation_data = compose_simulation_data_index(port_task_map)
! 78 return ModalComponentModelerData(modeler=modeler, data=port_simulation_data)
79
80
81 def compose_modeler(
82 modeler_file: str,Lines 106-119 106 model_dict = json.loads(json_str)
107 modeler_type = model_dict["type"]
108
109 if modeler_type == "ModalComponentModeler":
! 110 modeler = ModalComponentModeler.from_file(modeler_file)
111 elif modeler_type == "TerminalComponentModeler":
! 112 modeler = TerminalComponentModeler.from_file(modeler_file)
113 else:
114 raise TypeError(f"Unsupported modeler type: {type(modeler_type).__name__}")
! 115 return modeler
116
117
118 def compose_modeler_data(
119 modeler: ModalComponentModeler | TerminalComponentModeler,Lines 142-155 142 TypeError
143 If the provided `modeler` is not a recognized type.
144 """
145 if isinstance(modeler, ModalComponentModeler):
! 146 modeler_data = ModalComponentModelerData(modeler=modeler, data=indexed_sim_data)
147 elif isinstance(modeler, TerminalComponentModeler):
! 148 modeler_data = TerminalComponentModelerData(modeler=modeler, data=indexed_sim_data)
149 else:
150 raise TypeError(f"Unsupported modeler type: {type(modeler).__name__}")
! 151 return modeler_data
152
153
154 def compose_terminal_modeler_data_from_batch_data(
155 modeler: TerminalComponentModeler,Lines 171-182 171 -------
172 TerminalComponentModelerData
173 An object containing the results mapped to their respective ports.
174 """
! 175 ports = [modeler.get_task_name(port=port_i) for port_i in modeler.ports]
! 176 data = [batch_data[modeler.get_task_name(port=port_i)] for port_i in modeler.ports]
! 177 port_simulation_data = SimulationDataMap(keys=tuple(ports), values=tuple(data))
! 178 return TerminalComponentModelerData(modeler=modeler, data=port_simulation_data)
179
180
181 def compose_modal_modeler_data_from_batch_data(
182 modeler: ModalComponentModeler,Lines 198-209 198 -------
199 ModalComponentModelerData
200 An object containing the results mapped to their respective ports.
201 """
! 202 ports = [modeler.get_task_name(port=port_i) for port_i in modeler.ports]
! 203 data = [batch_data[modeler.get_task_name(port=port_i)] for port_i in modeler.ports]
! 204 port_simulation_data = SimulationDataMap(keys=tuple(ports), values=tuple(data))
! 205 return ModalComponentModelerData(modeler=modeler, data=port_simulation_data)
206
207
208 def compose_modeler_data_from_batch_data(
209 modeler: ComponentModelerType,Lines 232-251 232 ------
233 TypeError
234 If the provided `modeler` is not a recognized type.
235 """
! 236 if isinstance(modeler, ModalComponentModeler):
! 237 modeler_data = compose_modal_modeler_data_from_batch_data(
238 modeler=modeler, batch_data=batch_data
239 )
! 240 elif isinstance(modeler, TerminalComponentModeler):
! 241 modeler_data = compose_terminal_modeler_data_from_batch_data(
242 modeler=modeler, batch_data=batch_data
243 )
244 else:
! 245 raise TypeError(f"Unsupported modeler type: {type(modeler)}")
246
! 247 return modeler_data
248
249
250 def create_batch(
251 modeler: ComponentModelerType,tidy3d/web/api/tidy3d_stub.pyLines 94-105 94 elif type_ == "ModeSimulation":
95 sim = ModeSimulation.from_file(file_path)
96 elif type_ == "VolumeMesher":
97 sim = VolumeMesher.from_file(file_path)
! 98 elif type_ == "ModalComponentModeler":
! 99 sim = ModalComponentModeler.from_file(file_path)
! 100 elif type_ == "TerminalComponentModeler":
! 101 sim = TerminalComponentModeler.from_file(file_path)
102
103 return sim
104
105 def to_file(Lines 160-171 160 if isinstance(self.simulation, ModeSimulation):
161 return TaskType.MODE.name
162 elif isinstance(self.simulation, VolumeMesher):
163 return TaskType.VOLUME_MESH.name
! 164 elif isinstance(self.simulation, ModalComponentModeler):
! 165 return TaskType.COMPONENT_MODELER.name
! 166 elif isinstance(self.simulation, TerminalComponentModeler):
! 167 return TaskType.TERMINAL_COMPONENT_MODELER.name
168
169 def validate_pre_upload(self, source_required) -> None:
170 """Perform some pre-checks on instances of component"""
171 if isinstance(self.simulation, Simulation):Lines 218-229 218 elif type_ == "ModeSimulationData":
219 sim_data = ModeSimulationData.from_file(file_path)
220 elif type_ == "VolumeMesherData":
221 sim_data = VolumeMesherData.from_file(file_path)
! 222 elif type_ == "ModalComponentModelerData":
! 223 sim_data = ModalComponentModelerData.from_file(file_path)
! 224 elif type_ == "TerminalComponentModelerData":
! 225 sim_data = TerminalComponentModelerData.from_file(file_path)
226
227 return sim_data
228
229 def to_file(self, file_path: str):tidy3d/web/api/webapi.pyLines 74-82 74
75
76 def _get_url_rf(resource_id: str) -> str:
77 """Get the RF GUI URL for a modeler/batch group."""
! 78 return f"{Env.current.website_endpoint}/rf?taskId={resource_id}"
79
80
81 def _is_modeler_batch(resource_id: str) -> bool:
82 """Detect whether the given id corresponds to a modeler batch resource."""Lines 83-91 83 return BatchTask.is_batch(resource_id, batch_type="RF_SWEEP")
84
85
86 def _batch_detail(resource_id: str):
! 87 return BatchTask(resource_id).detail(batch_type="RF_SWEEP")
88
89
90 @wait_for_connection
91 def run(Lines 284-301 284 task_type = stub.get_type()
285 # Component modeler compatibility: map to RF task type
286 port_name_list = None
287 if task_type in ("COMPONENT_MODELER", "TERMINAL_COMPONENT_MODELER"):
! 288 task_type = "RF"
289 # Collect port names for modeler tasks if available
! 290 try:
! 291 ports = getattr(simulation, "ports", None)
! 292 if ports is not None:
! 293 port_name_list = [
294 getattr(p, "name", None) for p in ports if getattr(p, "name", None)
295 ]
! 296 except Exception:
! 297 port_name_list = None
298
299 task = SimulationTask.create(
300 task_type,
301 task_name,Lines 319-339 319 )
320 if task_type in GUI_SUPPORTED_TASK_TYPES:
321 if task_type == "RF":
322 # Prefer the group id if present in the creation response; avoid extra GET.
! 323 group_id = getattr(task, "groupId", None) or getattr(task, "group_id", None)
! 324 if not group_id:
! 325 try:
! 326 detail_task = SimulationTask.get(task.task_id, verbose=False)
! 327 group_id = getattr(detail_task, "groupId", None) or getattr(
328 detail_task, "group_id", None
329 )
! 330 except Exception:
! 331 group_id = None
! 332 url = _get_url_rf(group_id or task.task_id)
! 333 folder_url = _get_folder_url(task.folder_id)
! 334 console.log(f"View task using web UI at [link={url}]'{url}'[/link].")
! 335 console.log(f"Task folder: [link={folder_url}]'{task.folder_name}'[/link].")
336 else:
337 url = _get_url(task.task_id)
338 folder_url = _get_folder_url(task.folder_id)
339 console.log(f"View task using web UI at [link={url}]'{url}'[/link].")Lines 342-350 342 remote_sim_file = SIM_FILE_HDF5_GZ
343 if task_type == "MODE_SOLVER":
344 remote_sim_file = MODE_FILE_HDF5_GZ
345 elif task_type == "RF":
! 346 remote_sim_file = MODELER_FILE_HDF5_GZ
347
348 task.upload_simulation(
349 stub=stub,
350 verbose=verbose,Lines 361-370 361 # log the url for the task in the web UI
362 log.debug(f"{Env.current.website_endpoint}/folders/{task.folder_id}/tasks/{task.task_id}")
363 if task_type == "RF":
364 # Prefer returning batch/group id for downstream batch endpoints
! 365 batch_id = getattr(task, "batchId", None) or getattr(task, "batch_id", None)
! 366 return batch_id or task.task_id
367 return task.task_id
368
369
370 def get_reduced_simulation(simulation, reduce_simulation):Lines 475-483 475 A string with each key-value pair as a bullet point.
476 """
477 # Use a list comprehension to format each key-value pair
478 # and then join them together with newline characters.
! 479 return "\n".join([f"- {key}: {value}" for key, value in data_dict.items()])
480
481 console = get_logging_console()
482
483 # Component modeler batch path: hide split/check/submitLines 482-491 482
483 # Component modeler batch path: hide split/check/submit
484 if _is_modeler_batch(task_id):
485 # split (modeler-specific)
! 486 split_path = "tidy3d/projects/terminal-component-modeler-split"
! 487 payload = {
488 "batchType": "RF_SWEEP",
489 "batchId": task_id,
490 "fileName": "modeler.hdf5.gz",
491 "protocolVersion": _get_protocol_version(),Lines 489-523 489 "batchId": task_id,
490 "fileName": "modeler.hdf5.gz",
491 "protocolVersion": _get_protocol_version(),
492 }
! 493 resp = http.post(split_path, payload)
494
! 495 console.log(
496 f"Child simulation subtasks are being uploaded to \n{dict_to_bullet_list(resp)}"
497 )
498
! 499 batch = BatchTask(task_id)
500 # Kick off server-side validation for the RF batch.
! 501 batch.check(solver_version=solver_version, batch_type="RF_SWEEP")
502 # Validation phase
! 503 console.log("Validating RF batch...")
! 504 detail = batch.wait_for_validate(batch_type="RF_SWEEP")
! 505 status = detail.totalStatus
! 506 status_str = status.value
! 507 if status_str in ("validate_success", "validate_warn"):
! 508 console.log("Batch validation completed.")
509
510 # Show estimated FlexCredit cost from batch detail
! 511 est_fc = detail.estFlexUnit
! 512 console.log(f"Estimated FlexCredit cost: {est_fc:1.3f} for RF batch.")
! 513 if status_str not in ("validate_success", "validate_warn"):
! 514 raise WebError(f"Batch task {task_id} is blocked: {status_str}")
515 # Submit batch to start runs after validation
! 516 batch.submit(
517 solver_version=solver_version, batch_type="RF_SWEEP", worker_group=worker_group
518 )
! 519 return
520
521 if priority is not None and (priority < 1 or priority > 10):
522 raise ValueError("Priority must be between '1' and '10' if specified.")
523 task = SimulationTask.get(task_id)Lines 608-617 608 """
609
610 # Batch/modeler monitoring path
611 if _is_modeler_batch(task_id):
! 612 _monitor_modeler_batch(task_id, verbose=verbose, worker_group=worker_group)
! 613 return
614
615 console = get_logging_console() if verbose else None
616
617 task_info = get_info(task_id)Lines 779-800 779 console.log(
780 f"Task is aborting. View task using web UI at [link={url}]'{url}'[/link] to check the result."
781 )
782 return TaskInfo(**{"taskId": task.task_id, **task.dict()})
! 783 except WebNotFoundError:
! 784 pass # Task not found, might be a batch task
785
! 786 is_batch = BatchTask.is_batch(task_id, batch_type="RF_SWEEP")
! 787 if is_batch:
! 788 url = _get_url_rf(task_id)
! 789 console.log(
790 f"Batch task abortion is not yet supported, contact customer support."
791 f" View task using web UI at [link={url}]'{url}'[/link]."
792 )
! 793 return
794
! 795 console.log("Task ID cannot be found to be aborted.")
! 796 return
797
798
799 @wait_for_connection
800 def download(Lines 820-834 820 # Component modeler batch download path
821 if _is_modeler_batch(task_id):
822 # Use a more descriptive default filename for component modeler downloads.
823 # If the caller left the default as 'simulation_data.hdf5', prefer 'cm_data.hdf5'.
! 824 if os.path.basename(path) == "simulation_data.hdf5":
! 825 base_dir = os.path.dirname(path) or "."
! 826 path = os.path.join(base_dir, "cm_data.hdf5")
827
! 828 def _download_cm() -> bool:
! 829 try:
! 830 BatchTask(task_id).get_data_hdf5(
831 remote_data_file_gz=CM_DATA_HDF5_GZ,
832 to_file=path,
833 verbose=verbose,
834 progress_callback=progress_callback,Lines 832-862 832 to_file=path,
833 verbose=verbose,
834 progress_callback=progress_callback,
835 )
! 836 return True
! 837 except Exception:
! 838 return False
839
! 840 if not _download_cm():
! 841 BatchTask(task_id).postprocess(batch_type="RF_SWEEP")
842 # wait for postprocess to finish
! 843 while True:
! 844 resp = BatchTask(task_id).detail(batch_type="RF_SWEEP")
! 845 total = resp.totalTask or 0
! 846 post_succ = resp.postprocessSuccess or 0
! 847 status = resp.totalStatus
! 848 status_str = status.value
! 849 if status_str in {"error", "diverged", "blocked", "aborted", "aborting"}:
! 850 raise WebError(
851 f"Batch task {task_id} failed during postprocess: {status_str}"
852 ) from None
! 853 if total > 0 and post_succ >= total:
! 854 break
! 855 time.sleep(REFRESH_TIME)
! 856 if not _download_cm():
! 857 raise WebError("Failed to download 'cm_data' after postprocess completion.")
! 858 return
859
860 # Regular single-task download
861 task_info = get_info(task_id)
862 task_type = task_info.taskTypeLines 997-1006 997 Object containing simulation data.
998 """
999 # For component modeler batches, default to a clearer filename if the default was used.
1000 if _is_modeler_batch(task_id) and os.path.basename(path) == "simulation_data.hdf5":
! 1001 base_dir = os.path.dirname(path) or "."
! 1002 path = os.path.join(base_dir, "cm_data.hdf5")
1003
1004 if not os.path.exists(path) or replace_existing:
1005 download(task_id=task_id, path=path, verbose=verbose, progress_callback=progress_callback)Lines 1005-1016 1005 download(task_id=task_id, path=path, verbose=verbose, progress_callback=progress_callback)
1006
1007 if verbose:
1008 console = get_logging_console()
! 1009 if _is_modeler_batch(task_id):
! 1010 console.log(f"loading component modeler data from {path}")
1011 else:
! 1012 console.log(f"loading simulation from {path}")
1013
1014 stub_data = Tidy3dStubData.postprocess(path)
1015 return stub_dataLines 1021-1065 1021 max_detail_tasks: int = 20,
1022 worker_group: Optional[str] = None,
1023 ) -> None:
1024 """Monitor modeler batch progress with aggregate and per-task views."""
! 1025 console = get_logging_console() if verbose else None
1026
! 1027 def _status_to_stage(status: str) -> tuple[str, int]:
! 1028 s = (status or "").lower()
1029 # Map a broader set of statuses to monotonic stages for progress bars
! 1030 if s in ("draft", "created"):
! 1031 return ("draft", 0)
! 1032 if s in ("queue", "queued"):
! 1033 return ("queued", 1)
! 1034 if s in ("preprocess",):
! 1035 return ("preprocess", 1)
! 1036 if s in ("validating",):
! 1037 return ("validating", 2)
! 1038 if s in ("validate_success", "validate_warn"):
! 1039 return ("Validate", 3)
! 1040 if s in ("running",):
! 1041 return ("running", 4)
! 1042 if s in ("postprocess",):
! 1043 return ("postprocess", 5)
! 1044 if s in ("run_success", "success"):
! 1045 return ("Success", 6)
1046 # Unknown statuses map to earliest stage to avoid showing 100% prematurely
! 1047 return (s or "unknown", 0)
1048
! 1049 detail = _batch_detail(batch_id)
! 1050 name = detail.name or "modeler_batch"
! 1051 group_id = detail.groupId
1052
! 1053 header = f"Modeler Batch: {name}"
! 1054 if group_id:
! 1055 header += f" (group {group_id})"
! 1056 if console is not None:
! 1057 console.log(header)
1058
1059 # Non-verbose path: poll without progress bars then return
! 1060 if not verbose:
! 1061 terminal_errors = {
1062 "validate_fail",
1063 "error",
1064 "diverged",
1065 "blocked",Lines 1066-1099 1066 "aborting",
1067 "aborted",
1068 }
1069 # Run phase
! 1070 while True:
! 1071 d = _batch_detail(batch_id)
! 1072 s = d.totalStatus.value
! 1073 total = d.totalTask or 0
! 1074 r = d.runSuccess or 0
! 1075 if s in terminal_errors:
! 1076 raise WebError(f"Batch {batch_id} terminated: {s}")
! 1077 if total and r >= total:
! 1078 break
! 1079 time.sleep(REFRESH_TIME)
1080 # Postprocess phase
! 1081 BatchTask(batch_id).postprocess(batch_type="RF_SWEEP", worker_group=worker_group)
! 1082 while True:
! 1083 d = _batch_detail(batch_id)
! 1084 s = d.totalStatus.value
! 1085 total = d.totalTask or 0
! 1086 p = d.postprocessSuccess or 0
! 1087 postprocess_status = d.postprocessStatus
! 1088 if s in terminal_errors:
! 1089 raise WebError(f"Batch {batch_id} terminated: {s}")
! 1090 if postprocess_status == "success":
! 1091 break
! 1092 time.sleep(REFRESH_TIME)
! 1093 return
1094
! 1095 progress_columns = (
1096 TextColumn("[progress.description]{task.description}"),
1097 BarColumn(bar_width=25),
1098 TaskProgressColumn(),
1099 TimeElapsedColumn(),Lines 1097-1111 1097 BarColumn(bar_width=25),
1098 TaskProgressColumn(),
1099 TimeElapsedColumn(),
1100 )
! 1101 with Progress(*progress_columns, console=console, transient=False) as progress:
! 1102 terminal_errors = {"validate_fail", "error", "diverged", "blocked", "aborting", "aborted"}
1103
1104 # Phase: Run (aggregate + per-task)
! 1105 p_run = progress.add_task("Run", total=1.0)
! 1106 task_bars: dict[str, int] = {}
! 1107 run_statuses = [
1108 "draft",
1109 "preprocess",
1110 "validating",
1111 "Validate",Lines 1113-1192 1113 "postprocess",
1114 "Success",
1115 ]
1116
! 1117 while True:
! 1118 detail = _batch_detail(batch_id)
! 1119 status = detail.totalStatus.value
! 1120 total = detail.totalTask or 0
! 1121 r = detail.runSuccess or 0
1122
1123 # Create per-task bars as soon as tasks appear
! 1124 if total and total <= max_detail_tasks and detail.tasks:
! 1125 name_to_task = {(t.taskName or t.taskId): t for t in (detail.tasks or [])}
! 1126 for name, t in name_to_task.items():
! 1127 if name not in task_bars:
! 1128 tstatus = (t.status or "draft").lower()
! 1129 _, idx = _status_to_stage(tstatus)
! 1130 pbar = progress.add_task(
1131 f" {name}",
1132 total=len(run_statuses) - 1,
1133 completed=min(idx, len(run_statuses) - 1),
1134 )
! 1135 task_bars[name] = pbar
1136
1137 # Aggregate run progress: average stage fraction across tasks
! 1138 if detail.tasks:
! 1139 acc = 0.0
! 1140 n_members = 0
! 1141 for t in detail.tasks or []:
! 1142 n_members += 1
! 1143 tstatus = (t.status or "draft").lower()
! 1144 _, idx = _status_to_stage(tstatus)
! 1145 acc += max(0.0, min(1.0, idx / 6.0))
! 1146 run_frac = (acc / float(n_members)) if n_members else 0.0
1147 else:
! 1148 run_frac = (r / total) if total else 0.0
! 1149 progress.update(p_run, completed=run_frac)
1150
1151 # Update per-task bars
! 1152 if task_bars and detail.tasks:
! 1153 name_to_task = {(t.taskName or t.taskId): t for t in (detail.tasks or [])}
! 1154 for tname, pbar in task_bars.items():
! 1155 t = name_to_task.get(tname)
! 1156 if not t:
! 1157 continue
! 1158 tstatus = (t.status or "draft").lower()
! 1159 _, idx = _status_to_stage(tstatus)
! 1160 completed = min(idx, 6)
! 1161 desc = f" {tname} [{tstatus or 'draft'}]"
! 1162 progress.update(pbar, completed=completed, description=desc, refresh=False)
1163
! 1164 if total and r >= total:
! 1165 break
! 1166 if status in terminal_errors:
! 1167 raise WebError(f"Batch {batch_id} terminated: {status}")
! 1168 progress.refresh()
! 1169 time.sleep(REFRESH_TIME)
1170
1171 # Phase: Postprocess
! 1172 BatchTask(batch_id).postprocess(batch_type="RF_SWEEP", worker_group=worker_group)
! 1173 p_post = progress.add_task("Postprocess", total=1.0)
! 1174 while True:
! 1175 detail = _batch_detail(batch_id)
! 1176 status = detail.totalStatus.value
! 1177 postprocess_status = detail.postprocessStatus
! 1178 total = detail.totalTask or 0
! 1179 p = detail.postprocessSuccess or 0
! 1180 progress.update(p_post, completed=(p / total) if total else 0.0)
! 1181 if postprocess_status == "success":
! 1182 break
! 1183 if status in terminal_errors:
! 1184 raise WebError(f"Batch {batch_id} terminated: {status}")
! 1185 progress.refresh()
! 1186 time.sleep(REFRESH_TIME)
! 1187 if console is not None:
! 1188 console.log("Postprocess completed.")
1189
1190
1191 @wait_for_connection
1192 def delete(task_id: TaskId, versions: bool = False) -> TaskInfo:Lines 1232-1248 1232 progress_callback : Callable[[float], None] = None
1233 Optional callback function called when downloading file with ``bytes_in_chunk`` as argument.
1234
1235 """
! 1236 task_info = get_info(task_id)
! 1237 task_type = task_info.taskType
1238
! 1239 remote_sim_file = SIM_FILE_HDF5_GZ
! 1240 if task_type == "MODE_SOLVER":
! 1241 remote_sim_file = MODE_FILE_HDF5_GZ
1242
! 1243 task = SimulationTask(taskId=task_id)
! 1244 task.get_simulation_hdf5(
1245 path, verbose=verbose, progress_callback=progress_callback, remote_sim_file=remote_sim_file
1246 )
1247 Lines 1518-1526 1518 console = get_logging_console()
1519 console.log("Authentication configured successfully!")
1520 except (WebError, HTTPError) as e:
1521 url = "https://docs.flexcompute.com/projects/tidy3d/en/latest/index.html"
! 1522 msg = (
1523 str(e)
1524 + "\n\n"
1525 + "It looks like the Tidy3D Python interface is not configured with your "
1526 "unique API key. "Lines 1531-1536 1531 "'.tidy3d/config' (windows) with content like: \n\n"
1532 "apikey = 'XXX' \n\nHere XXX is your API key copied from your account page within quotes.\n\n"
1533 f"For details, check the instructions at {url}."
1534 )
! 1535 raise WebError(msg) from etidy3d/web/core/http_util.pyLines 133-188 133
134 if resp.status_code != ResponseCodes.OK.value:
135 if resp.status_code == ResponseCodes.NOT_FOUND.value:
136 raise WebNotFoundError("Resource not found (HTTP 404).")
! 137 try:
! 138 json_resp = resp.json()
! 139 except Exception:
! 140 json_resp = None
141
142 # Build a helpful error message using available fields
! 143 err_msg = None
! 144 if isinstance(json_resp, dict):
! 145 parts = []
! 146 for key in ("error", "message", "msg", "detail", "code", "httpStatus", "warning"):
! 147 val = json_resp.get(key)
! 148 if not val:
! 149 continue
! 150 if key == "error":
151 # Always include the raw 'error' payload for debugging. Also try to extract a nested message.
! 152 if isinstance(val, str):
! 153 try:
! 154 nested = json.loads(val)
! 155 if isinstance(nested, dict):
! 156 nested_msg = (
157 nested.get("message")
158 or nested.get("error")
159 or nested.get("msg")
160 )
! 161 if nested_msg:
! 162 parts.append(str(nested_msg))
! 163 except Exception:
! 164 pass
! 165 parts.append(f"error={val}")
166 else:
! 167 parts.append(f"error={val!s}")
! 168 continue
! 169 parts.append(str(val))
! 170 if parts:
! 171 err_msg = "; ".join(parts)
! 172 if not err_msg:
173 # Fallback to response text or status code
! 174 err_msg = resp.text or f"HTTP {resp.status_code}"
175
176 # Append request context to aid debugging
! 177 try:
! 178 method = getattr(resp.request, "method", "")
! 179 url = getattr(resp.request, "url", "")
! 180 err_msg = f"{err_msg} [HTTP {resp.status_code} {method} {url}]"
! 181 except Exception:
! 182 pass
183
! 184 raise WebError(err_msg)
185
186 if not resp.text:
187 return None
188 result = resp.json()tidy3d/web/core/task_core.pyLines 90-98 90 resp = http.get(project_endpoint, params={"projectName": folder_name})
91 if resp:
92 folder = Folder(**resp)
93 if create and not folder:
! 94 resp = http.post(projects_endpoint, {"projectName": folder_name})
95 if resp:
96 folder = Folder(**resp)
97 FOLDER_CACHE[folder_name] = folder
98 return folderLines 269-277 269 }
270 # Component modeler: include port names if provided
271 if port_name_list:
272 # Align with backend contract: expect 'portNames' (not 'portNameList')
! 273 payload["portNames"] = port_name_list
274
275 resp = http.post(f"{projects_endpoint}/{folder.folder_id}/tasks", payload)
276 # RF group creation may return group-level info without 'taskId'.
277 # Use 'groupId' (or 'batchId' as fallback) as the resource id for subsequent uploads.Lines 276-289 276 # RF group creation may return group-level info without 'taskId'.
277 # Use 'groupId' (or 'batchId' as fallback) as the resource id for subsequent uploads.
278 if "taskId" not in resp and task_type == "RF":
279 # Prefer using 'batchId' as the resource id for uploads (S3 STS expects a task-like id).
! 280 if "batchId" in resp:
! 281 resp["taskId"] = resp["batchId"]
! 282 elif "groupId" in resp:
! 283 resp["taskId"] = resp["groupId"]
284 else:
! 285 raise WebError("Missing resource ID for task creation. Contact customer support.")
286 return SimulationTask(**resp, taskType=task_type, folder_name=folder_name)
287
288 @classmethod
289 def get(cls, task_id: str, verbose: bool = True) -> SimulationTask:Lines 749-757 749 most methods, as it dictates which backend service handles the request.
750 """
751
752 def __init__(self, batch_id: str):
! 753 self.batch_id = batch_id
754
755 @staticmethod
756 def is_batch(resource_id: str, batch_type: str) -> bool:
757 """Checks if a given resource ID corresponds to a valid batch task.Lines 774-783 774 try:
775 resp = http.get(
776 f"tidy3d/tasks/{resource_id}/batch-detail", params={"batchType": batch_type}
777 )
! 778 status = bool(resp and isinstance(resp, dict) and "status" in resp)
! 779 return status
780 except Exception:
781 return False
782
783 def detail(self, batch_type: str) -> BatchDetail:Lines 792-807 792 -------
793 BatchDetail
794 An object containing the batch's latest data.
795 """
! 796 resp = http.get(
797 f"tidy3d/tasks/{self.batch_id}/batch-detail", params={"batchType": batch_type}
798 )
799 # Some backends may return null for collection fields; coerce to sensible defaults
! 800 if isinstance(resp, dict):
! 801 if resp.get("tasks") is None:
! 802 resp["tasks"] = []
! 803 return BatchDetail(**(resp or {}))
804
805 def check(
806 self,
807 solver_version: Optional[str] = None,Lines 823-833 823 -------
824 Any
825 The server's response to the check request.
826 """
! 827 if protocol_version is None:
! 828 protocol_version = _get_protocol_version()
! 829 return http.post(
830 f"tidy3d/projects/{self.batch_id}/batch-check",
831 {
832 "batchType": batch_type,
833 "solverVersion": solver_version,Lines 859-869 859 -------
860 Any
861 The server's response to the submit request.
862 """
! 863 if protocol_version is None:
! 864 protocol_version = _get_protocol_version()
! 865 return http.post(
866 f"tidy3d/projects/{self.batch_id}/batch-submit",
867 {
868 "batchType": batch_type,
869 "solverVersion": solver_version,Lines 896-906 896 -------
897 Any
898 The server's response to the post-process request.
899 """
! 900 if protocol_version is None:
! 901 protocol_version = _get_protocol_version()
! 902 return http.post(
903 f"tidy3d/projects/{self.batch_id}/postprocess",
904 {
905 "batchType": batch_type,
906 "solverVersion": solver_version,Lines 931-949 931 This method blocks until the batch status is 'validate_success',
932 'validate_warn', 'validate_fail', or another terminal state like 'blocked'
933 or 'aborted', or until the timeout is reached.
934 """
! 935 start = datetime.now().timestamp()
! 936 while True:
! 937 d = self.detail(batch_type=batch_type)
! 938 status = d.totalStatus
! 939 if status in ("validate_success", "validate_warn", "validate_fail"):
! 940 return d
! 941 if status in ("blocked", "aborting", "aborted"):
! 942 return d
! 943 if timeout is not None and (datetime.now().timestamp() - start) > timeout:
! 944 return d
! 945 time.sleep(REFRESH_TIME)
946
947 def wait_for_run(self, timeout: Optional[float] = None, batch_type: str = "") -> BatchDetail:
948 """Waits for the batch to complete the execution stage by polling its status.Lines 964-976 964 This method blocks until the batch status reaches a terminal run state like
965 'run_success', 'run_failed', 'diverged', 'blocked', or 'aborted',
966 or until the timeout is reached.
967 """
! 968 start = datetime.now().timestamp()
! 969 while True:
! 970 d = self.detail(batch_type=batch_type)
! 971 status = d.totalStatus
! 972 if status in (
973 "run_success",
974 "run_failed",
975 "diverged",
976 "blocked",Lines 976-987 976 "blocked",
977 "aborting",
978 "aborted",
979 ):
! 980 return d
! 981 if timeout is not None and (datetime.now().timestamp() - start) > timeout:
! 982 return d
! 983 time.sleep(REFRESH_TIME)
984
985 def get_data_hdf5(
986 self,
987 remote_data_file_gz: str,Lines 1017-1027 1017 -----
1018 This method first attempts to download the gzipped version of a file.
1019 If that fails, it falls back to downloading the uncompressed version.
1020 """
! 1021 file = None
! 1022 try:
! 1023 file = download_gz_file(
1024 resource_id=self.batch_id,
1025 remote_filename=remote_data_file_gz,
1026 to_file=to_file,
1027 verbose=verbose,Lines 1026-1041 1026 to_file=to_file,
1027 verbose=verbose,
1028 progress_callback=progress_callback,
1029 )
! 1030 except ClientError:
! 1031 if verbose:
! 1032 console = get_logger_console()
! 1033 console.log(f"Unable to download '{remote_data_file_gz}'.")
1034
! 1035 if not file:
! 1036 try:
! 1037 file = download_file(
1038 resource_id=self.batch_id,
1039 remote_filename=remote_data_file_gz[:-3],
1040 to_file=to_file,
1041 verbose=verbose,Lines 1040-1051 1040 to_file=to_file,
1041 verbose=verbose,
1042 progress_callback=progress_callback,
1043 )
! 1044 except Exception as e:
! 1045 raise WebError(
1046 "Failed to download the batch data file from the server. "
1047 "Please confirm that the batch has been successfully postprocessed."
1048 ) from e
1049
! 1050 return file |
weiliangjin2021
approved these changes
Aug 28, 2025
There was a problem hiding this comment.
Wow, this is massive amount of effort! The flow looks good. A few minor comments and questions:
- How do we decide which function goes to analysis folder and utils.py?
- To make sure I understand correctly, now we'll need two lines to get smatrix: run and then compute smatrix from the data method?
27 tasks
In a way it can be noted by the "types" that the function accepts. The idea of the refactor is to have a more structured extensible flow for complex workflows. In a way the We can also concieve other previous class-methods as really pure operations onto the state of a given class. In a way, this is what So ultimately, I'd suggest, going forward with RF we need to have a clear separation of the "schema / information containers" and the "operations / functions / methods" onto those containers. This should both make the API design a lot easier, and also make it more extensible and powerful moving onwards. You could argue utils functions have a local scope limited to either the function itself, or used across multiple scopes so they should be standalone functions without any relation to the data containers or higher level functions they get used in.
Yep exactly! Again this conceptual separation between operations and data containers / schema. |
|
I've listed all pending items #2773 but this should be stable enough to merge to develop and do the RF GUI solver upgrade |
daquinteroflex
force-pushed
the
dario/rf_endpoints_update
branch
3 times, most recently
from
fd7e01e to
bad068a
Compare
momchil-flex
approved these changes
Aug 28, 2025
daquinteroflex
force-pushed
the
dario/rf_endpoints_update
branch
4 times, most recently
from
bf98e54 to
9321f2b
Compare
Co-authored-by: yaugenst-flex <[email protected]> Co-authored-by: dmarek-flex <[email protected]>
daquinteroflex
force-pushed
the
dario/rf_endpoints_update
branch
from
9321f2b to
1745219
Compare
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Greptile Summary
This PR implements RF GUI <-> Python client interoperability, enabling seamless integration between RF microwave simulation tools and the Python API. The changes introduce significant architectural improvements to the S-matrix plugin, including:
Core Architecture Changes:
ComponentModelerclass intoModalComponentModelerandTerminalComponentModelerfor better separation of concerns between modal and terminal-based RF analysisModalComponentModelerDataandTerminalComponentModelerDataclasses that separate simulation orchestration from results processing, following the established pattern in other Tidy3D solversBatchTask,BatchStatus,BatchDetail) to handle RF component modeling workflows that operate on collections of simulationsType System Reorganization:
types.pyfile into a package withbase.py,third_party.py,utils.py, andsimulation.pysubmodulestyping.Literal,typing.Union) instead of internal re-exportsVoltageArray,CurrentArray,ImpedanceArray) and RF-specific port data structuresRF-Specific Features:
analysis/modal.py,analysis/terminal.py) containing the mathematical core for scattering parameter calculationsSimulationMapandSimulationDataMapfor managing collections of simulations and their results with dictionary-like access patternsWeb API Enhancements:
COMPONENT_MODELER,TERMINAL_COMPONENT_MODELER) to the web infrastructurecm_data.hdf5)The changes maintain backward compatibility where possible while introducing a deprecation warning for the
TerminalComponentModelerclass indicating breaking changes in version 2.10. The implementation follows established patterns from other Tidy3D solvers, ensuring consistency across the ecosystem.Important Files Changed
Click to expand file changes table
tidy3d/plugins/microwave/custom_path_integrals.pytidy3d/plugins/microwave/path_integrals.pytidy3d/plugins/smatrix/analysis/__init__.pytidy3d/plugins/smatrix/run.pyschemas/TerminalComponentModeler.jsontidy3d/components/types/simulation.pytidy3d/components/tcad/generation_recombination.pytidy3d/plugins/smatrix/utils.pytests/utils.pytidy3d/components/data/index.pytidy3d/components/index.pytidy3d/components/types/base.pytidy3d/components/types/third_party.pytidy3d/components/types/__init__.pytidy3d/components/types/utils.pydocs/faqdocs/notebookstidy3d/plugins/smatrix/analysis/antenna.pytidy3d/plugins/smatrix/analysis/modal.pytidy3d/plugins/smatrix/analysis/terminal.pytidy3d/plugins/smatrix/component_modelers/base.pytidy3d/plugins/smatrix/component_modelers/modal.pytidy3d/plugins/smatrix/component_modelers/terminal.pytidy3d/plugins/smatrix/data/data_array.pytidy3d/plugins/smatrix/data/terminal.pytidy3d/plugins/smatrix/ports/types.pytidy3d/plugins/smatrix/smatrix.pytidy3d/plugins/smatrix/__init__.pytidy3d/web/api/webapi.pytidy3d/web/core/http_util.pytidy3d/web/core/task_core.pytidy3d/web/core/task_info.pytidy3d/web/api/tidy3d_stub.pytests/test_components/test_map.pytests/test_data/test_map_data.pytests/test_plugins/smatrix/test_component_modeler.pytests/test_plugins/smatrix/test_terminal_component_modeler.pytidy3d/components/base.pytidy3d/components/data/monitor_data.pytidy3d/components/autograd/types.pytidy3d/components/data/data_array.pytidy3d/components/geometry/mesh.pytidy3d/components/material/tcad/charge.pytidy3d/components/medium.pytidy3d/components/monitor.pytidy3d/components/tcad/doping.pytidy3d/components/tcad/simulation/heat_charge.pytidy3d/components/tcad/types.pytidy3d/plugins/adjoint/components/types.pytidy3d/plugins/adjoint/web.pytidy3d/plugins/design/design.pytidy3d/plugins/microwave/impedance_calculator.pytidy3d/plugins/smatrix/component_modelers/types.pytidy3d/plugins/smatrix/data/modal.pytidy3d/plugins/smatrix/data/types.pytidy3d/plugins/smatrix/network.pytidy3d/plugins/smatrix/ports/modal.pytidy3d/web/api/connect_util.pytidy3d/web/api/container.pytidy3d/web/common.pytidy3d/web/core/constants.pytidy3d/web/core/types.pytests/test_plugins/test_array_factor.pytests/test_plugins/smatrix/terminal_component_modeler_def.pytidy3d/plugins/autograd/README.mddocs/api/plugins/smatrix.rsttidy3d/__init__.pyschemas/Simulation.jsonConfidence score: 2/5