From f7002bec508b1136877823727d4a2a4a1d4f1202 Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Mon, 26 May 2025 23:34:16 +0200 Subject: [PATCH 01/97] enhance esp-idf-size use --- builder/main.py | 91 +++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 77 insertions(+), 14 deletions(-) diff --git a/builder/main.py b/builder/main.py index 6c6dca5f2..8a21d8334 100644 --- a/builder/main.py +++ b/builder/main.py @@ -360,28 +360,66 @@ def check_lib_archive_exists(): def firmware_metrics(target, source, env): + """ + Custom target to run esp-idf-size with support for command line parameters + Usage: pio run -t metrics -- [esp-idf-size arguments] + """ if terminal_cp != "utf-8": print("Firmware metrics can not be shown. Set the terminal codepage to \"utf-8\"") return + map_file = os.path.join(env.subst("$BUILD_DIR"), env.subst("$PROGNAME") + ".map") if not os.path.isfile(map_file): # map file can be in project dir map_file = os.path.join(get_project_dir(), env.subst("$PROGNAME") + ".map") - if os.path.isfile(map_file): - try: - import subprocess - python_exe = env.subst("$PYTHONEXE") - run_env = os.environ.copy() - run_env["PYTHONIOENCODING"] = "utf-8" - run_env["PYTHONUTF8"] = "1" - # Show output of esp_idf_size, but suppresses the command echo - subprocess.run([ - python_exe, "-m", "esp_idf_size", "--ng", map_file - ], env=run_env, check=False) - except Exception: - print("Warning: Failed to run firmware metrics. Is esp-idf-size installed?") - pass + if not os.path.isfile(map_file): + print(f"Error: Map file not found: {map_file}") + print("Make sure the project is built first with 'pio run'") + return + + try: + import subprocess + import sys + import shlex + + cmd = [env.subst("$PYTHONEXE"), "-m", "esp_idf_size", "--ng"] + + # Parameters from platformio.ini + extra_args = env.GetProjectOption("custom_esp_idf_size_args", "") + if extra_args: + cmd.extend(shlex.split(extra_args)) + + # Command Line Parameter, after -- + cli_args = [] + if "--" in sys.argv: + dash_index = sys.argv.index("--") + if dash_index + 1 < len(sys.argv): + cli_args = sys.argv[dash_index + 1:] + cmd.extend(cli_args) + + # Map-file as last argument + cmd.append(map_file) + + # Debug-Info if wanted + if env.GetProjectOption("custom_esp_idf_size_verbose", False): + print(f"Running command: {' '.join(cmd)}") + + # Call esp-idf-size + result = subprocess.run(cmd, check=False, capture_output=False) + + if result.returncode != 0: + print(f"Warning: esp-idf-size exited with code {result.returncode}") + + except ImportError: + print("Error: esp-idf-size module not found.") + print("Install with: pip install esp-idf-size") + except FileNotFoundError: + print("Error: Python executable not found.") + print("Check your Python installation.") + except Exception as e: + print(f"Error: Failed to run firmware metrics: {e}") + print("Make sure esp-idf-size is installed: pip install esp-idf-size") # # Target: Build executable and linkable firmware or FS image @@ -618,6 +656,31 @@ def firmware_metrics(target, source, env): "Erase Flash", ) +# +# Register Custom Target +# +env.AddCustomTarget( + name="metrics", + dependencies="$BUILD_DIR/${PROGNAME}.elf", + actions=firmware_metrics, + title="Firmware Size Metrics", + description="Analyze firmware size using esp-idf-size (supports CLI args after --)", + always_build=True +) + +# +# Additional Target without Build-Dependency when already compiled +# +env.AddCustomTarget( + name="metrics-only", + dependencies=None, + actions=firmware_metrics, + title="Firmware Size Metrics (No Build)", + description="Analyze firmware size without building first", + always_build=True +) + + # # Override memory inspection behavior # From 2d9ff886762f1c1850c38eb4ca5e686f433aee8d Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Mon, 26 May 2025 23:54:51 +0200 Subject: [PATCH 02/97] Build the map file always for espidf too --- builder/frameworks/espidf.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/builder/frameworks/espidf.py b/builder/frameworks/espidf.py index e464e64f5..3ed93fe03 100644 --- a/builder/frameworks/espidf.py +++ b/builder/frameworks/espidf.py @@ -1799,13 +1799,10 @@ def get_python_exe(): "-DSDKCONFIG=" + SDKCONFIG_PATH, ] -if "CPPDEFINES" in env: - flatten_cppdefines = env.Flatten(env['CPPDEFINES']) - if "SHOW_METRICS" in flatten_cppdefines: - # This will add the linker flag for the map file - extra_cmake_args.append( - f'-DCMAKE_EXE_LINKER_FLAGS=-Wl,-Map={os.path.join(BUILD_DIR, env.subst("$PROGNAME") + ".map")}' - ) +# This will add the linker flag for the map file +extra_cmake_args.append( + f'-DCMAKE_EXE_LINKER_FLAGS=-Wl,-Map={os.path.join(BUILD_DIR, env.subst("$PROGNAME") + ".map")}' +) # Add any extra args from board config extra_cmake_args += click.parser.split_arg_string(board.get("build.cmake_extra_args", "")) From 4f9e28e0e65197bc49abb57c1771dceb8c33ca5d Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Thu, 29 May 2025 00:07:14 +0200 Subject: [PATCH 03/97] Update espidf.py --- builder/frameworks/espidf.py | 342 +++++++++++++++++++---------------- 1 file changed, 187 insertions(+), 155 deletions(-) diff --git a/builder/frameworks/espidf.py b/builder/frameworks/espidf.py index 3ed93fe03..191e7280e 100644 --- a/builder/frameworks/espidf.py +++ b/builder/frameworks/espidf.py @@ -135,6 +135,7 @@ def _get_installed_standard_pip_packages(): flag_custom_sdkonfig = False flag_custom_component_add = False flag_custom_component_remove = False +removed_components = set() IDF5 = ( platform.get_package_version("framework-espidf") @@ -171,6 +172,7 @@ def _get_installed_standard_pip_packages(): os.rename(ARDUINO_FRAMEWORK_DIR, new_path) ARDUINO_FRAMEWORK_DIR = new_path assert ARDUINO_FRAMEWORK_DIR and os.path.isdir(ARDUINO_FRAMEWORK_DIR) + arduino_libs_mcu = join(ARDUINO_FRAMEWORK_DIR,"tools","esp32-arduino-libs",mcu) BUILD_DIR = env.subst("$BUILD_DIR") PROJECT_DIR = env.subst("$PROJECT_DIR") @@ -195,177 +197,193 @@ def _get_installed_standard_pip_packages(): flag_custom_sdkonfig = True def HandleArduinoIDFsettings(env): + """ + Handles Arduino IDF settings configuration with custom sdkconfig support. + """ + def get_MD5_hash(phrase): + """Generate MD5 hash for checksum validation.""" import hashlib - return hashlib.md5((phrase).encode('utf-8')).hexdigest()[:16] + return hashlib.md5(phrase.encode('utf-8')).hexdigest()[:16] - def custom_sdkconfig_file(string): - if not config.has_option("env:"+env["PIOENV"], "custom_sdkconfig"): + def load_custom_sdkconfig_file(): + """Load custom sdkconfig from file or URL if specified.""" + if not config.has_option("env:" + env["PIOENV"], "custom_sdkconfig"): return "" - sdkconfig_entrys = env.GetProjectOption("custom_sdkconfig").splitlines() - for file in sdkconfig_entrys: - if "http" in file and "://" in file: - response = requests.get(file.split(" ")[0]) - if response.ok: - target = str(response.content.decode('utf-8')) - else: - print("Failed to download:", file) + + sdkconfig_entries = env.GetProjectOption("custom_sdkconfig").splitlines() + + for file_entry in sdkconfig_entries: + # Handle HTTP/HTTPS URLs + if "http" in file_entry and "://" in file_entry: + try: + response = requests.get(file_entry.split(" ")[0]) + if response.ok: + return response.content.decode('utf-8') + except Exception as e: + print(f"Error downloading {file_entry}: {e}") return "" - return target - if "file://" in file: - file_path = join(PROJECT_DIR,file.lstrip("file://").split(os.path.sep)[-1]) + + # Handle local files + if "file://" in file_entry: + file_path = join(PROJECT_DIR, file_entry.lstrip("file://").split(os.path.sep)[-1]) if os.path.exists(file_path): - with open(file_path, 'r') as file: - target = file.read() + try: + with open(file_path, 'r') as f: + return f.read() + except Exception as e: + print(f"Error reading file {file_path}: {e}") + return "" else: - print("File not found:", file_path) + print("File not found, check path:", file_path) return "" - return target + return "" + def extract_flag_name(line): + """Extract flag name from sdkconfig line.""" + line = line.strip() + if line.startswith("#") and "is not set" in line: + return line.split(" ")[1] + elif not line.startswith("#") and "=" in line: + return line.split("=")[0] + return None - custom_sdk_config_flags = "" - board_idf_config_flags = "" - sdkconfig_file_flags = "" - custom_sdkconfig_file_str = "" - - if config.has_option("env:"+env["PIOENV"], "custom_sdkconfig"): - flag_custom_sdkonfig = True - custom_sdk_config_flags = (env.GetProjectOption("custom_sdkconfig").rstrip("\n")) + "\n" - custom_sdkconfig_file_str = custom_sdkconfig_file(sdkconfig_file_flags) - - if "espidf.custom_sdkconfig" in board: - board_idf_config_flags = ('\n'.join([element for element in board.get("espidf.custom_sdkconfig", "")])).rstrip("\n") + "\n" - flag_custom_sdkonfig = True - - if flag_custom_sdkonfig == True: # TDOO duplicated - print("*** Add \"custom_sdkconfig\" settings to IDF sdkconfig.defaults ***") - idf_config_flags = custom_sdk_config_flags - if custom_sdkconfig_file_str != "": - sdkconfig_file_flags = custom_sdkconfig_file_str + "\n" - idf_config_flags = sdkconfig_file_flags + idf_config_flags - idf_config_flags = board_idf_config_flags + idf_config_flags + def build_idf_config_flags(): + """Build complete IDF configuration flags from all sources.""" + flags = [] + + # Add board-specific flags first + if "espidf.custom_sdkconfig" in board: + board_flags = board.get("espidf.custom_sdkconfig", []) + if board_flags: + flags.extend(board_flags) + + # Add custom sdkconfig file content + custom_file_content = load_custom_sdkconfig_file() + if custom_file_content: + flags.append(custom_file_content) + + # Add project-level custom sdkconfig + if config.has_option("env:" + env["PIOENV"], "custom_sdkconfig"): + custom_flags = env.GetProjectOption("custom_sdkconfig").rstrip("\n") + if custom_flags: + flags.append(custom_flags) + + return "\n".join(flags) + "\n" if flags else "" + + def add_flash_configuration(config_flags): + """Add flash frequency and mode configuration.""" if flash_frequency != "80m": - idf_config_flags = idf_config_flags + "# CONFIG_ESPTOOLPY_FLASHFREQ_80M is not set\n" - esptool_flashfreq_y = "CONFIG_ESPTOOLPY_FLASHFREQ_%s=y\n" % flash_frequency.upper() - esptool_flashfreq_M = "CONFIG_ESPTOOLPY_FLASHFREQ=\"%s\"\n" % flash_frequency - idf_config_flags = idf_config_flags + esptool_flashfreq_y + esptool_flashfreq_M + config_flags += "# CONFIG_ESPTOOLPY_FLASHFREQ_80M is not set\n" + config_flags += f"CONFIG_ESPTOOLPY_FLASHFREQ_{flash_frequency.upper()}=y\n" + config_flags += f"CONFIG_ESPTOOLPY_FLASHFREQ=\"{flash_frequency}\"\n" + if flash_mode != "qio": - idf_config_flags = idf_config_flags + "# CONFIG_ESPTOOLPY_FLASHMODE_QIO is not set\n" - esptool_flashmode = "CONFIG_ESPTOOLPY_FLASHMODE_%s=y\n" % flash_mode.upper() - if esptool_flashmode not in idf_config_flags: - idf_config_flags = idf_config_flags + esptool_flashmode - if mcu in ("esp32") and "CONFIG_FREERTOS_UNICORE=y" in idf_config_flags: - idf_config_flags = idf_config_flags + "# CONFIG_SPIRAM is not set\n" - - idf_config_flags = idf_config_flags.splitlines() - sdkconfig_src = join(ARDUINO_FRMWRK_LIB_DIR,mcu,"sdkconfig") - - def get_flag(line): - if line.startswith("#") and "is not set" in line: - return line.split(" ")[1] - elif not line.startswith("#") and len(line.split("=")) > 1: - return line.split("=")[0] - else: - return None - - with open(sdkconfig_src) as src: - sdkconfig_dst = os.path.join(PROJECT_DIR, "sdkconfig.defaults") - dst = open(sdkconfig_dst,"w") - dst.write("# TASMOTA__"+ get_MD5_hash(''.join(custom_sdk_config_flags).strip() + mcu) +"\n") - while line := src.readline(): - flag = get_flag(line) - if flag is None: + config_flags += "# CONFIG_ESPTOOLPY_FLASHMODE_QIO is not set\n" + + flash_mode_flag = f"CONFIG_ESPTOOLPY_FLASHMODE_{flash_mode.upper()}=y\n" + if flash_mode_flag not in config_flags: + config_flags += flash_mode_flag + + # ESP32 specific SPIRAM configuration + if mcu == "esp32" and "CONFIG_FREERTOS_UNICORE=y" in config_flags: + config_flags += "# CONFIG_SPIRAM is not set\n" + + return config_flags + + def write_sdkconfig_file(idf_config_flags, checksum_source): + """Write the final sdkconfig.defaults file with checksum.""" + sdkconfig_src = join(ARDUINO_FRAMEWORK_DIR, "tools", "esp32-arduino-libs", mcu, "sdkconfig") + sdkconfig_dst = join(PROJECT_DIR, "sdkconfig.defaults") + + # Generate checksum for validation (maintains original logic) + checksum = get_MD5_hash(checksum_source.strip() + mcu) + + with open(sdkconfig_src, 'r') as src, open(sdkconfig_dst, 'w') as dst: + # Write checksum header (critical for compilation decision logic) + dst.write(f"# TASMOTA__{checksum}\n") + + processed_flags = set() + + # Process each line from source sdkconfig + for line in src: + flag_name = extract_flag_name(line) + + if flag_name is None: dst.write(line) - else: - no_match = True - for item in idf_config_flags: - if flag == get_flag(item.replace("\'", "")): - dst.write(item.replace("\'", "")+"\n") - no_match = False - print("Replace:",line,"with:",item.replace("\'", "")) - idf_config_flags.remove(item) - if no_match: - dst.write(line) - for item in idf_config_flags: # are there new flags? - print("Add:",item.replace("\'", "")) - dst.write(item.replace("\'", "")+"\n") - dst.close() - return - else: + continue + + # Check if we have a custom replacement for this flag + flag_replaced = False + for custom_flag in idf_config_flags[:]: # Create copy for safe removal + custom_flag_name = extract_flag_name(custom_flag.replace("'", "")) + + if flag_name == custom_flag_name: + cleaned_flag = custom_flag.replace("'", "") + dst.write(cleaned_flag + "\n") + print(f"Replace: {line.strip()} with: {cleaned_flag}") + idf_config_flags.remove(custom_flag) + processed_flags.add(custom_flag_name) + flag_replaced = True + break + + if not flag_replaced: + dst.write(line) + + # Add any remaining new flags + for remaining_flag in idf_config_flags: + cleaned_flag = remaining_flag.replace("'", "") + print(f"Add: {cleaned_flag}") + dst.write(cleaned_flag + "\n") + + # Main execution logic + has_custom_config = ( + config.has_option("env:" + env["PIOENV"], "custom_sdkconfig") or + "espidf.custom_sdkconfig" in board + ) + + if not has_custom_config: return + + print("*** Add \"custom_sdkconfig\" settings to IDF sdkconfig.defaults ***") + + # Build complete configuration + idf_config_flags = build_idf_config_flags() + idf_config_flags = add_flash_configuration(idf_config_flags) + + # Convert to list for processing + idf_config_list = [line for line in idf_config_flags.splitlines() if line.strip()] + + # Write final configuration file with checksum + custom_sdk_config_flags = "" + if config.has_option("env:" + env["PIOENV"], "custom_sdkconfig"): + custom_sdk_config_flags = env.GetProjectOption("custom_sdkconfig").rstrip("\n") + "\n" + + write_sdkconfig_file(idf_config_list, custom_sdk_config_flags) + + def HandleCOMPONENTsettings(env): - if flag_custom_component_add == True or flag_custom_component_remove == True: # todo remove duplicated - import yaml - from yaml import SafeLoader - print("*** \"custom_component\" is used to (de)select managed idf components ***") - if flag_custom_component_remove == True: - idf_custom_component_remove = env.GetProjectOption("custom_component_remove").splitlines() - else: - idf_custom_component_remove = "" - if flag_custom_component_add == True: - idf_custom_component_add = env.GetProjectOption("custom_component_add").splitlines() - else: - idf_custom_component_add = "" - - # search "idf_component.yml" file - try: # 1.st in Arduino framework - idf_component_yml_src = os.path.join(ARDUINO_FRAMEWORK_DIR, "idf_component.yml") - shutil.copy(join(ARDUINO_FRAMEWORK_DIR,"idf_component.yml"),join(ARDUINO_FRAMEWORK_DIR,"idf_component.yml.orig")) - yml_file_dir = idf_component_yml_src - except: # 2.nd Project source - try: - idf_component_yml_src = os.path.join(PROJECT_SRC_DIR, "idf_component.yml") - shutil.copy(join(PROJECT_SRC_DIR,"idf_component.yml"),join(PROJECT_SRC_DIR,"idf_component.yml.orig")) - yml_file_dir = idf_component_yml_src - except: # no idf_component.yml in Project source -> create - idf_component_yml_src = os.path.join(PROJECT_SRC_DIR, "idf_component.yml") - yml_file_dir = idf_component_yml_src - idf_component_yml_str = """ - dependencies: - idf: \">=5.1\" - """ - idf_component_yml = yaml.safe_load(idf_component_yml_str) - with open(idf_component_yml_src, 'w',) as f : - yaml.dump(idf_component_yml,f) - - yaml_file=open(idf_component_yml_src,"r") - idf_component=yaml.load(yaml_file, Loader=SafeLoader) - idf_component_str=json.dumps(idf_component) # convert to json string - idf_component_json=json.loads(idf_component_str) # convert string to json dict - - if idf_custom_component_remove != "": - for entry in idf_custom_component_remove: - # checking if the entry exists before removing - if entry in idf_component_json["dependencies"]: - print("*** Removing component:",entry) - del idf_component_json["dependencies"][entry] - - if idf_custom_component_add != "": - for entry in idf_custom_component_add: - if len(str(entry)) > 4: # too short or empty entry - # add new entrys to json - if "@" in entry: - idf_comp_entry = str(entry.split("@")[0]).replace(" ", "") - idf_comp_vers = str(entry.split("@")[1]).replace(" ", "") - else: - idf_comp_entry = str(entry).replace(" ", "") - idf_comp_vers = "*" - if idf_comp_entry not in idf_component_json["dependencies"]: - print("*** Adding component:", idf_comp_entry, idf_comp_vers) - new_entry = {idf_comp_entry: {"version": idf_comp_vers}} - idf_component_json["dependencies"].update(new_entry) - - idf_component_yml_file = open(yml_file_dir,"w") - yaml.dump(idf_component_json, idf_component_yml_file) - idf_component_yml_file.close() - # print("JSON from modified idf_component.yml:") - # print(json.dumps(idf_component_json)) + from component_manager import ComponentManager + component_manager = ComponentManager(env) + + if flag_custom_component_add or flag_custom_component_remove: + actions = [action for flag, action in [ + (flag_custom_component_add, "select"), + (flag_custom_component_remove, "deselect") + ] if flag] + action_text = " and ".join(actions) + print(f"*** \"custom_component\" is used to {action_text} managed idf components ***") + + component_manager.handle_component_settings( + add_components=flag_custom_component_add, + remove_components=flag_custom_component_remove + ) return return -if flag_custom_component_add == True or flag_custom_component_remove == True: +if "arduino" in env.subst("$PIOFRAMEWORK"): HandleCOMPONENTsettings(env) if flag_custom_sdkonfig == True and "arduino" in env.subst("$PIOFRAMEWORK") and "espidf" not in env.subst("$PIOFRAMEWORK"): @@ -1799,6 +1817,7 @@ def get_python_exe(): "-DSDKCONFIG=" + SDKCONFIG_PATH, ] + # This will add the linker flag for the map file extra_cmake_args.append( f'-DCMAKE_EXE_LINKER_FLAGS=-Wl,-Map={os.path.join(BUILD_DIR, env.subst("$PROGNAME") + ".map")}' @@ -2170,7 +2189,13 @@ def idf_lib_copy(source, target, env): print("*** Original Arduino \"idf_component.yml\" restored ***") except: print("*** Original Arduino \"idf_component.yml\" couldnt be restored ***") - env.AddPostAction("checkprogsize", idf_lib_copy) + # Restore original pioarduino-build.py + from component_manager import ComponentManager + component_manager = ComponentManager(env) + component_manager.restore_pioarduino_build_py() + silent_action = env.Action(idf_lib_copy) + silent_action.strfunction = lambda target, source, env: '' # hack to silence scons command output + env.AddPostAction("checkprogsize", silent_action) if "espidf" in env.subst("$PIOFRAMEWORK") and (flag_custom_component_add == True or flag_custom_component_remove == True): def idf_custom_component(source, target, env): @@ -2186,8 +2211,15 @@ def idf_custom_component(source, target, env): os.remove(join(PROJECT_SRC_DIR,"idf_component.yml")) print("*** pioarduino generated \"idf_component.yml\" removed ***") except: - print("*** \"idf_component.yml\" couldnt be removed ***") - env.AddPostAction("checkprogsize", idf_custom_component) + print("*** no custom \"idf_component.yml\" found for removing ***") + if "arduino" in env.subst("$PIOFRAMEWORK"): + # Restore original pioarduino-build.py, only used with Arduino + from component_manager import ComponentManager + component_manager = ComponentManager(env) + component_manager.restore_pioarduino_build_py() + silent_action = env.Action(idf_custom_component) + silent_action.strfunction = lambda target, source, env: '' # hack to silence scons command output + env.AddPostAction("checkprogsize", silent_action) # # Process OTA partition and image # @@ -2242,7 +2274,7 @@ def _parse_size(value): partitions_csv = env.subst("$PARTITIONS_TABLE_CSV") result = [] next_offset = 0 -bound = int(board.get("upload.offset_address", "0x10000"), 16) # default 0x10000 +bound = 0x10000 with open(partitions_csv) as fp: for line in fp.readlines(): line = line.strip() From 3c0209e2b8cea7fe18481efc1ae56d3c00a30c1b Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Thu, 29 May 2025 00:08:33 +0200 Subject: [PATCH 04/97] Create component_manager.py --- builder/frameworks/component_manager.py | 510 ++++++++++++++++++++++++ 1 file changed, 510 insertions(+) create mode 100644 builder/frameworks/component_manager.py diff --git a/builder/frameworks/component_manager.py b/builder/frameworks/component_manager.py new file mode 100644 index 000000000..f188bbb32 --- /dev/null +++ b/builder/frameworks/component_manager.py @@ -0,0 +1,510 @@ +# component_manager.py +import os +import shutil +import re +import yaml +from yaml import SafeLoader +from os.path import join +from typing import Set, Optional, Dict, Any, List + + +class ComponentManager: + """Manages IDF components for ESP32 Arduino framework builds.""" + + def __init__(self, env): + self.env = env + self.platform = env.PioPlatform() + self.config = env.GetProjectConfig() + self.board = env.BoardConfig() + self.mcu = self.board.get("build.mcu", "esp32").lower() + self.project_src_dir = env.subst("$PROJECT_SRC_DIR") + self.removed_components: Set[str] = set() + self.ignored_libs: Set[str] = set() + + self.arduino_framework_dir = self.platform.get_package_dir("framework-arduinoespressif32") + self.arduino_libs_mcu = join(self.arduino_framework_dir, "tools", "esp32-arduino-libs", self.mcu) + + def handle_component_settings(self, add_components: bool = False, remove_components: bool = False) -> None: + """Handle adding and removing IDF components based on project configuration.""" + if not (add_components or remove_components): + return + + # Create backup before first component removal + if remove_components and not self.removed_components: + self._backup_pioarduino_build_py() + + component_yml_path = self._get_or_create_component_yml() + component_data = self._load_component_yml(component_yml_path) + + if remove_components: + try: + components_to_remove = self.env.GetProjectOption("custom_component_remove").splitlines() + self._remove_components(component_data, components_to_remove) + except: + pass + + if add_components: + try: + components_to_add = self.env.GetProjectOption("custom_component_add").splitlines() + self._add_components(component_data, components_to_add) + except: + pass + + self._save_component_yml(component_yml_path, component_data) + + # Clean up removed components + if self.removed_components: + self._cleanup_removed_components() + + self.handle_lib_ignore() + + def handle_lib_ignore(self) -> None: + """Handle lib_ignore entries from platformio.ini and remove corresponding includes.""" + # Create backup before processing lib_ignore + if not self.ignored_libs: + self._backup_pioarduino_build_py() + + # Get lib_ignore entries from current environment only + lib_ignore_entries = self._get_lib_ignore_entries() + + if lib_ignore_entries: + self.ignored_libs.update(lib_ignore_entries) + self._remove_ignored_lib_includes() + print(f"Removed include paths for {len(lib_ignore_entries)} ignored libraries") + + def _get_lib_ignore_entries(self) -> List[str]: + """Get lib_ignore entries from current environment configuration only.""" + try: + # Get lib_ignore from current environment only + lib_ignore = self.env.GetProjectOption("lib_ignore", []) + + if isinstance(lib_ignore, str): + lib_ignore = [lib_ignore] + elif lib_ignore is None: + lib_ignore = [] + + # Clean and normalize entries + cleaned_entries = [] + for entry in lib_ignore: + entry = str(entry).strip() + if entry: + # Convert library names to potential include directory names + include_name = self._convert_lib_name_to_include(entry) + cleaned_entries.append(include_name) + + # Filter out critical ESP32 components that should never be ignored + critical_components = [ + 'lwip', # Network stack + 'freertos', # Real-time OS + 'esp_system', # System functions + 'esp_common', # Common ESP functions + 'driver', # Hardware drivers + 'nvs_flash', # Non-volatile storage + 'spi_flash', # Flash memory access + 'esp_timer', # Timer functions + 'esp_event', # Event system + 'log' # Logging system + ] + + filtered_entries = [] + for entry in cleaned_entries: + if entry not in critical_components: + filtered_entries.append(entry) + + return filtered_entries + + except Exception: + return [] + + def _get_arduino_core_libraries(self) -> Dict[str, str]: + """Get all Arduino core libraries and their corresponding include paths.""" + libraries_mapping = {} + + # Path to Arduino Core Libraries + arduino_libs_dir = join(self.arduino_framework_dir, "libraries") + + if not os.path.exists(arduino_libs_dir): + return libraries_mapping + + try: + for entry in os.listdir(arduino_libs_dir): + lib_path = join(arduino_libs_dir, entry) + if os.path.isdir(lib_path): + lib_name = self._get_library_name_from_properties(lib_path) + if lib_name: + include_path = self._map_library_to_include_path(lib_name, entry) + libraries_mapping[lib_name.lower()] = include_path + libraries_mapping[entry.lower()] = include_path # Also use directory name as key + except Exception: + pass + + return libraries_mapping + + def _get_library_name_from_properties(self, lib_dir: str) -> Optional[str]: + """Extract library name from library.properties file.""" + prop_path = join(lib_dir, "library.properties") + if not os.path.isfile(prop_path): + return None + + try: + with open(prop_path, 'r', encoding='utf-8') as f: + for line in f: + line = line.strip() + if line.startswith('name='): + return line.split('=', 1)[1].strip() + except Exception: + pass + + return None + + def _map_library_to_include_path(self, lib_name: str, dir_name: str) -> str: + """Map library name to corresponding include path.""" + lib_name_lower = lib_name.lower().replace(' ', '').replace('-', '_') + dir_name_lower = dir_name.lower() + + # Extended mapping list with Arduino Core Libraries + extended_mapping = { + # Core ESP32 mappings + 'wifi': 'esp_wifi', + 'bluetooth': 'bt', + 'bluetoothserial': 'bt', + 'ble': 'bt', + 'bt': 'bt', + 'ethernet': 'esp_eth', + 'websocket': 'esp_websocket_client', + 'http': 'esp_http_client', + 'https': 'esp_https_ota', + 'ota': 'esp_https_ota', + 'spiffs': 'spiffs', + 'fatfs': 'fatfs', + 'mesh': 'esp_wifi_mesh', + 'smartconfig': 'esp_smartconfig', + 'mdns': 'mdns', + 'coap': 'coap', + 'mqtt': 'mqtt', + 'json': 'cjson', + 'mbedtls': 'mbedtls', + 'openssl': 'openssl', + + # Arduino Core specific mappings (safe mappings that don't conflict with critical components) + 'esp32blearduino': 'bt', + 'esp32_ble_arduino': 'bt', + 'esp32': 'esp32', + 'wire': 'driver', + 'spi': 'driver', + 'i2c': 'driver', + 'uart': 'driver', + 'serial': 'driver', + 'analogwrite': 'driver', + 'ledc': 'driver', + 'pwm': 'driver', + 'dac': 'driver', + 'adc': 'driver', + 'touch': 'driver', + 'hall': 'driver', + 'rtc': 'driver', + 'timer': 'esp_timer', + 'preferences': 'arduino_preferences', + 'eeprom': 'arduino_eeprom', + 'update': 'esp_https_ota', + 'httpupdate': 'esp_https_ota', + 'httpclient': 'esp_http_client', + 'httpsclient': 'esp_https_ota', + 'wifimanager': 'esp_wifi', + 'wificlientsecure': 'esp_wifi', + 'wifiserver': 'esp_wifi', + 'wifiudp': 'esp_wifi', + 'wificlient': 'esp_wifi', + 'wifiap': 'esp_wifi', + 'wifimulti': 'esp_wifi', + 'esp32webserver': 'esp_http_server', + 'webserver': 'esp_http_server', + 'asyncwebserver': 'esp_http_server', + 'dnsserver': 'lwip', + 'netbios': 'netbios', + 'simpletime': 'lwip', + 'fs': 'vfs', + 'sd': 'fatfs', + 'sd_mmc': 'fatfs', + 'littlefs': 'esp_littlefs', + 'ffat': 'fatfs', + 'camera': 'esp32_camera', + 'esp_camera': 'esp32_camera', + 'arducam': 'esp32_camera', + 'rainmaker': 'esp_rainmaker', + 'esp_rainmaker': 'esp_rainmaker', + 'provisioning': 'wifi_provisioning', + 'wifiprovisioning': 'wifi_provisioning', + 'espnow': 'esp_now', + 'esp_now': 'esp_now', + 'esptouch': 'esp_smartconfig', + 'ping': 'lwip', + 'netif': 'lwip', + 'tcpip': 'lwip' + } + + # Check extended mapping first + if lib_name_lower in extended_mapping: + return extended_mapping[lib_name_lower] + + # Check directory name + if dir_name_lower in extended_mapping: + return extended_mapping[dir_name_lower] + + # Fallback: Use directory name as include path + return dir_name_lower + + def _convert_lib_name_to_include(self, lib_name: str) -> str: + """Convert library name to potential include directory name.""" + # Load Arduino Core Libraries on first call + if not hasattr(self, '_arduino_libraries_cache'): + self._arduino_libraries_cache = self._get_arduino_core_libraries() + + lib_name_lower = lib_name.lower() + + # Check Arduino Core Libraries first + if lib_name_lower in self._arduino_libraries_cache: + return self._arduino_libraries_cache[lib_name_lower] + + # Remove common prefixes and suffixes + cleaned_name = lib_name_lower + + # Remove common prefixes + prefixes_to_remove = ['lib', 'arduino-', 'esp32-', 'esp-'] + for prefix in prefixes_to_remove: + if cleaned_name.startswith(prefix): + cleaned_name = cleaned_name[len(prefix):] + + # Remove common suffixes + suffixes_to_remove = ['-lib', '-library', '.h'] + for suffix in suffixes_to_remove: + if cleaned_name.endswith(suffix): + cleaned_name = cleaned_name[:-len(suffix)] + + # Check again with cleaned name + if cleaned_name in self._arduino_libraries_cache: + return self._arduino_libraries_cache[cleaned_name] + + # Direct mapping for common cases not in Arduino libraries + direct_mapping = { + 'ble': 'bt', + 'bluetooth': 'bt', + 'bluetoothserial': 'bt' + } + + if cleaned_name in direct_mapping: + return direct_mapping[cleaned_name] + + return cleaned_name + + def _remove_ignored_lib_includes(self) -> None: + """Remove include entries for ignored libraries from pioarduino-build.py.""" + build_py_path = join(self.arduino_libs_mcu, "pioarduino-build.py") + + if not os.path.exists(build_py_path): + return + + try: + with open(build_py_path, 'r') as f: + content = f.read() + + original_content = content + total_removed = 0 + + # Remove CPPPATH entries for each ignored library + for lib_name in self.ignored_libs: + # Multiple patterns to catch different include formats + patterns = [ + rf'.*join\([^,]*,\s*"include",\s*"{re.escape(lib_name)}"[^)]*\),?\n', + rf'.*"include/{re.escape(lib_name)}"[^,\n]*,?\n', + rf'.*"[^"]*include[^"]*{re.escape(lib_name)}[^"]*"[^,\n]*,?\n', + rf'.*"[^"]*/{re.escape(lib_name)}/include[^"]*"[^,\n]*,?\n', + rf'.*"[^"]*{re.escape(lib_name)}[^"]*include[^"]*"[^,\n]*,?\n', + rf'.*join\([^)]*"include"[^)]*"{re.escape(lib_name)}"[^)]*\),?\n', + rf'.*"{re.escape(lib_name)}/include"[^,\n]*,?\n', + rf'\s*"[^"]*/{re.escape(lib_name)}/[^"]*",?\n' + ] + + for pattern in patterns: + matches = re.findall(pattern, content) + if matches: + content = re.sub(pattern, '', content) + total_removed += len(matches) + + # Clean up empty lines and trailing commas + content = re.sub(r'\n\s*\n', '\n', content) + content = re.sub(r',\s*\n\s*\]', '\n]', content) + + # Validate and write changes + if self._validate_changes(original_content, content) and content != original_content: + with open(build_py_path, 'w') as f: + f.write(content) + + except Exception: + pass + + def _validate_changes(self, original_content: str, new_content: str) -> bool: + """Validate that the changes are reasonable.""" + original_lines = len(original_content.splitlines()) + new_lines = len(new_content.splitlines()) + removed_lines = original_lines - new_lines + + # Don't allow removing more than 50% of the file or negative changes + return not (removed_lines > original_lines * 0.5 or removed_lines < 0) + + def _get_or_create_component_yml(self) -> str: + """Get path to idf_component.yml, creating it if necessary.""" + # Try Arduino framework first + framework_yml = join(self.arduino_framework_dir, "idf_component.yml") + if os.path.exists(framework_yml): + self._create_backup(framework_yml) + return framework_yml + + # Try project source directory + project_yml = join(self.project_src_dir, "idf_component.yml") + if os.path.exists(project_yml): + self._create_backup(project_yml) + return project_yml + + # Create new file in project source + self._create_default_component_yml(project_yml) + return project_yml + + def _create_backup(self, file_path: str) -> None: + """Create backup of a file.""" + backup_path = f"{file_path}.orig" + if not os.path.exists(backup_path): + shutil.copy(file_path, backup_path) + + def _create_default_component_yml(self, file_path: str) -> None: + """Create a default idf_component.yml file.""" + default_content = { + "dependencies": { + "idf": ">=5.1" + } + } + + with open(file_path, 'w') as f: + yaml.dump(default_content, f) + + def _load_component_yml(self, file_path: str) -> Dict[str, Any]: + """Load and parse idf_component.yml file.""" + try: + with open(file_path, "r") as f: + return yaml.load(f, Loader=SafeLoader) or {"dependencies": {}} + except Exception: + return {"dependencies": {}} + + def _save_component_yml(self, file_path: str, data: Dict[str, Any]) -> None: + """Save component data to YAML file.""" + try: + with open(file_path, "w") as f: + yaml.dump(data, f) + except Exception: + pass + + def _remove_components(self, component_data: Dict[str, Any], components_to_remove: list) -> None: + """Remove specified components from the configuration.""" + dependencies = component_data.setdefault("dependencies", {}) + + for component in components_to_remove: + component = component.strip() + if not component: + continue + + if component in dependencies: + del dependencies[component] + + # Track for cleanup + filesystem_name = self._convert_component_name_to_filesystem(component) + self.removed_components.add(filesystem_name) + + def _add_components(self, component_data: Dict[str, Any], components_to_add: list) -> None: + """Add specified components to the configuration.""" + dependencies = component_data.setdefault("dependencies", {}) + + for component in components_to_add: + component = component.strip() + if len(component) <= 4: # Skip too short entries + continue + + component_name, version = self._parse_component_entry(component) + + if component_name not in dependencies: + dependencies[component_name] = {"version": version} + + def _parse_component_entry(self, entry: str) -> tuple[str, str]: + """Parse component entry into name and version.""" + if "@" in entry: + name, version = entry.split("@", 1) + return (name.strip(), version.strip()) + return (entry.strip(), "*") + + def _convert_component_name_to_filesystem(self, component_name: str) -> str: + """Convert component name from registry format to filesystem format.""" + return component_name.replace("/", "__") + + def _backup_pioarduino_build_py(self) -> None: + """Create backup of the original pioarduino-build.py.""" + if "arduino" not in self.env.subst("$PIOFRAMEWORK"): + return + + build_py_path = join(self.arduino_libs_mcu, "pioarduino-build.py") + backup_path = join(self.arduino_libs_mcu, f"pioarduino-build.py.{self.mcu}") + + if os.path.exists(build_py_path) and not os.path.exists(backup_path): + shutil.copy2(build_py_path, backup_path) + + def _cleanup_removed_components(self) -> None: + """Clean up removed components and restore original build file.""" + for component in self.removed_components: + self._remove_include_directory(component) + + self._remove_cpppath_entries() + + def _remove_include_directory(self, component: str) -> None: + """Remove include directory for a component.""" + include_path = join(self.arduino_libs_mcu, "include", component) + + if os.path.exists(include_path): + shutil.rmtree(include_path) + + def _remove_cpppath_entries(self) -> None: + """Remove CPPPATH entries for removed components from pioarduino-build.py.""" + build_py_path = join(self.arduino_libs_mcu, "pioarduino-build.py") + + if not os.path.exists(build_py_path): + return + + try: + with open(build_py_path, 'r') as f: + content = f.read() + + original_content = content + + # Remove CPPPATH entries for each removed component + for component in self.removed_components: + patterns = [ + rf'.*join\([^,]*,\s*"include",\s*"{re.escape(component)}"[^)]*\),?\n', + rf'.*"include/{re.escape(component)}"[^,\n]*,?\n', + rf'.*"[^"]*include[^"]*{re.escape(component)}[^"]*"[^,\n]*,?\n' + ] + + for pattern in patterns: + content = re.sub(pattern, '', content) + + if content != original_content: + with open(build_py_path, 'w') as f: + f.write(content) + + except Exception: + pass + + def restore_pioarduino_build_py(self, source=None, target=None, env=None) -> None: + """Restore the original pioarduino-build.py from backup.""" + build_py_path = join(self.arduino_libs_mcu, "pioarduino-build.py") + backup_path = join(self.arduino_libs_mcu, f"pioarduino-build.py.{self.mcu}") + + if os.path.exists(backup_path): + shutil.copy2(backup_path, build_py_path) + os.remove(backup_path) From b9dba06da1cfeeb2912ba7b308fcf876ff8ef06d Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Thu, 29 May 2025 00:13:29 +0200 Subject: [PATCH 05/97] Update component_manager.py --- builder/frameworks/component_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/builder/frameworks/component_manager.py b/builder/frameworks/component_manager.py index f188bbb32..0bf816b95 100644 --- a/builder/frameworks/component_manager.py +++ b/builder/frameworks/component_manager.py @@ -22,7 +22,7 @@ def __init__(self, env): self.ignored_libs: Set[str] = set() self.arduino_framework_dir = self.platform.get_package_dir("framework-arduinoespressif32") - self.arduino_libs_mcu = join(self.arduino_framework_dir, "tools", "esp32-arduino-libs", self.mcu) + self.arduino_libs_mcu = join(self.platform.get_package_dir("framework-arduinoespressif32-libs", self.mcu) def handle_component_settings(self, add_components: bool = False, remove_components: bool = False) -> None: """Handle adding and removing IDF components based on project configuration.""" From 6d0d39979c949a4fc56346f3f1b0005600651274 Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Thu, 29 May 2025 00:53:23 +0200 Subject: [PATCH 06/97] Update arduino.py --- builder/frameworks/arduino.py | 743 ++++++++++++++++++++++++++-------- 1 file changed, 575 insertions(+), 168 deletions(-) diff --git a/builder/frameworks/arduino.py b/builder/frameworks/arduino.py index 41430f1e6..657278cde 100644 --- a/builder/frameworks/arduino.py +++ b/builder/frameworks/arduino.py @@ -28,258 +28,665 @@ import os import sys import shutil -from os.path import join, exists - -from SCons.Script import COMMAND_LINE_TARGETS, DefaultEnvironment, SConscript +import hashlib +import logging +from contextlib import suppress +from os.path import join, exists, isabs, splitdrive, commonpath, relpath +from pathlib import Path +from typing import Union, List + +from SCons.Script import DefaultEnvironment, SConscript from platformio import fs from platformio.package.version import pepver_to_semver -from platformio.project.config import ProjectConfig from platformio.package.manager.tool import ToolPackageManager +IS_WINDOWS = sys.platform.startswith("win") + +# Include path length threshold for path shortening, only valid and needed for Windows +# Windows has a path length limit of ~260 characters per path, but the total command line +# length is also limited to ~32000 characters +INCLUDE_PATH_LENGTH_THRESHOLD = 31500 # Total character count threshold + +python_deps = { + "wheel": ">=0.35.1", + "rich-click": ">=1.8.6", + "PyYAML": ">=6.0.2", + "intelhex": ">=2.3.0", + "esp-idf-size": ">=1.6.1" +} + +def setup_logging(): + """Setup logging with optional file output""" + handlers = [logging.StreamHandler()] + + # Only add file handler if writable and not disabled + log_file = os.environ.get('ARDUINO_FRAMEWORK_LOG_FILE') + if log_file: + with suppress(OSError, PermissionError): + handlers.append(logging.FileHandler(log_file)) + + logging.basicConfig( + level=logging.INFO, + format='%(asctime)s - %(levelname)s - %(message)s', + handlers=handlers + ) + +# Only setup logging if enabled via environment variable +if os.environ.get('ARDUINO_FRAMEWORK_ENABLE_LOGGING'): + setup_logging() + +# Constants for better performance +UNICORE_FLAGS = { + "CORE32SOLO1", + "CONFIG_FREERTOS_UNICORE=y" +} + +# Global flags to prevent message spam +_PATH_SHORTENING_MESSAGES = { + 'shortening_applied': False, + 'no_framework_paths_warning': False, + 'long_path_warning_shown': False +} + + +# Cache class for frequently used paths +class PathCache: + def __init__(self, platform, mcu): + self.platform = platform + self.mcu = mcu + self._framework_dir = None + self._sdk_dir = None + + @property + def framework_dir(self): + if self._framework_dir is None: + self._framework_dir = self.platform.get_package_dir("framework-arduinoespressif32") + return self._framework_dir + + @property + def framework_dir(self): + if self._framework_lib_dir is None: + self._framework_lib_dir = self.platform.get_package_dir("framework-arduinoespressif32-libs") + return self._framework_lib_dir + + @property + def sdk_dir(self): + if self._sdk_dir is None: + self._sdk_dir = fs.to_unix_path( + join(self.framework_dir, "tools", "esp32-arduino-libs", self.mcu, "include") + ) + return self._sdk_dir + +def check_and_warn_long_path_support(): + """Checks Windows long path support and issues warning if disabled""" + if not IS_WINDOWS or _PATH_SHORTENING_MESSAGES['long_path_warning_shown']: + return + + try: + import winreg + key = winreg.OpenKey( + winreg.HKEY_LOCAL_MACHINE, + r"SYSTEM\CurrentControlSet\Control\FileSystem" + ) + value, _ = winreg.QueryValueEx(key, "LongPathsEnabled") + winreg.CloseKey(key) + + if value != 1: + print("*** WARNING: Windows Long Path Support is disabled ***") + print("*** Enable it for better performance: ***") + print("*** 1. Run as Administrator: gpedit.msc ***") + print("*** 2. Navigate to: Computer Configuration > Administrative Templates > System > Filesystem ***") + print("*** 3. Enable 'Enable Win32 long paths' ***") + print("*** OR run PowerShell as Admin: ***") + print("*** New-ItemProperty -Path 'HKLM:\\SYSTEM\\CurrentControlSet\\Control\\FileSystem' -Name 'LongPathsEnabled' -Value 1 -PropertyType DWORD -Force ***") + print("*** Restart required after enabling ***") + except Exception: + print("*** WARNING: Could not check Long Path Support status ***") + print("*** Consider enabling Windows Long Path Support for better performance ***") + + _PATH_SHORTENING_MESSAGES['long_path_warning_shown'] = True + +# Secure deletion functions +def safe_delete_file(file_path: Union[str, Path], + force: bool = False) -> bool: + """ + Secure file deletion + + Args: + file_path: Path to file to be deleted + force: Forces deletion even for write-protected files + + Returns: + bool: True if successfully deleted + """ + file_path = Path(file_path) + + try: + # Check existence + if not file_path.exists(): + logging.warning(f"File does not exist: {file_path}") + return False + + # Remove write protection if necessary + if force and not os.access(file_path, os.W_OK): + file_path.chmod(0o666) + + # Delete file + file_path.unlink() + logging.info(f"File deleted: {file_path}") + return True + + except PermissionError: + logging.error(f"No permission to delete: {file_path}") + return False + except Exception as e: + logging.error(f"Error deleting {file_path}: {e}") + return False + +def safe_delete_directory(dir_path: Union[str, Path]) -> bool: + """ + Secure directory deletion + """ + dir_path = Path(dir_path) + + try: + if not dir_path.exists(): + logging.warning(f"Directory does not exist: {dir_path}") + return False + + shutil.rmtree(dir_path) + logging.info(f"Directory deleted: {dir_path}") + return True + + except Exception as e: + logging.error(f"Error deleting {dir_path}: {e}") + return False + +def validate_platformio_path(path: Union[str, Path]) -> bool: + """ + Special validation for PlatformIO package paths + """ + path = Path(path).resolve() + path_str = str(path) + + # Must be within .platformio directory structure + if ".platformio" not in path_str: + return False + + # Must be a packages directory + if "packages" not in path_str: + return False + + # Must be framework-related + framework_indicators = [ + "framework-arduinoespressif32", + "esp32-arduino-libs", + ".platformio/packages", + "packages/framework-arduinoespressif32" + ] + + if not any(indicator in path_str for indicator in framework_indicators): + return False + + # Must not be a critical system path + critical_paths = ["/usr", "/bin", "/sbin", "/etc", "/boot"] + return not any(critical in path_str for critical in critical_paths) + +def validate_deletion_path(path: Union[str, Path], + allowed_patterns: List[str]) -> bool: + """ + Validates if a path can be safely deleted + + Args: + path: Path to be checked + allowed_patterns: Allowed path patterns + + Returns: + bool: True if deletion is safe + """ + path = Path(path).resolve() + + # Check against critical system paths + critical_paths = [ + Path.home(), + Path("/"), + Path("C:\\") if IS_WINDOWS else None, + Path("/usr"), + Path("/etc"), + Path("/bin"), + Path("/sbin") + ] + + for critical in filter(None, critical_paths): + try: + if path == critical or critical in path.parents: + logging.error(f"Critical system path detected: {path}") + return False + except (OSError, ValueError): + # Path comparison failed, reject for safety + logging.error(f"Path comparison failed for: {path}") + return False + + # Check against allowed patterns + path_str = str(path) + is_allowed = any(pattern in path_str for pattern in allowed_patterns) + + if not is_allowed: + logging.error(f"Path does not match allowed patterns: {path}") + logging.error(f"Allowed patterns: {allowed_patterns}") + else: + logging.info(f"Path validation successful: {path}") + + return is_allowed + +def safe_framework_cleanup(): + """Secure cleanup of Arduino Framework""" + + # Secure deletion of framework directories + if exists(FRAMEWORK_DIR): + logging.info(f"Attempting to validate framework path: {FRAMEWORK_DIR}") + + # Use specialized PlatformIO path validation + if validate_platformio_path(FRAMEWORK_DIR): + #print("*** Secure framework cleanup ***") + logging.info(f"Framework path validated successfully: {FRAMEWORK_DIR}") + + if safe_delete_directory(FRAMEWORK_DIR): + #print("Framework successfully removed") + else: + logging.error(f"PlatformIO path validation failed: {FRAMEWORK_DIR}") + return False + + logging.info(f"Attempting to validate framework path: {FRAMEWORK_LIB_DIR}") + + # Use specialized PlatformIO path validation + if validate_platformio_path(FRAMEWORK_LIB_DIR): + #print("*** Secure framework cleanup ***") + logging.info(f"Framework lib path validated successfully: {FRAMEWORK_LIB_DIR}") + + if safe_delete_directory(FRAMEWORK_LIB_DIR): + #print("Framework successfully removed") + return True + else: + print("Error removing framework") + return False + else: + logging.error(f"PlatformIO path validation failed: {FRAMEWORK_LIB_DIR}") + return False + return True + +def safe_remove_sdkconfig_files(): + """Secure removal of SDKConfig files""" + envs = [section.replace("env:", "") for section in config.sections() if section.startswith("env:")] + for env_name in envs: + file_path = join(project_dir, f"sdkconfig.{env_name}") + if exists(file_path): + safe_delete_file(file_path) + +# Initialization env = DefaultEnvironment() pm = ToolPackageManager() platform = env.PioPlatform() config = env.GetProjectConfig() board = env.BoardConfig() + +# Cached values mcu = board.get("build.mcu", "esp32") +pioenv = env["PIOENV"] +project_dir = env.subst("$PROJECT_DIR") +path_cache = PathCache(platform, mcu) +current_env_section = f"env:{pioenv}" + +# Board configuration board_sdkconfig = board.get("espidf.custom_sdkconfig", "") entry_custom_sdkconfig = "\n" flag_custom_sdkconfig = False -IS_WINDOWS = sys.platform.startswith("win") +flag_custom_component_remove = False +flag_custom_component_add = False +flag_lib_ignore = False + +# pio lib_ignore check +if config.has_option(current_env_section, "lib_ignore"): + flag_lib_ignore = True + +# Custom Component remove check +if config.has_option(current_env_section, "custom_component_remove"): + flag_custom_component_remove = True -if config.has_option("env:"+env["PIOENV"], "custom_sdkconfig"): +# Custom SDKConfig check +if config.has_option(current_env_section, "custom_sdkconfig"): entry_custom_sdkconfig = env.GetProjectOption("custom_sdkconfig") flag_custom_sdkconfig = True -if len(str(board_sdkconfig)) > 2: +if len(board_sdkconfig) > 2: flag_custom_sdkconfig = True -extra_flags = (''.join([element for element in board.get("build.extra_flags", "")])).replace("-D", " ") +extra_flags_raw = board.get("build.extra_flags", []) +if isinstance(extra_flags_raw, list): + extra_flags = " ".join(extra_flags_raw).replace("-D", " ") +else: + extra_flags = str(extra_flags_raw).replace("-D", " ") + framework_reinstall = False -flag_any_custom_sdkconfig = False -FRAMEWORK_LIB_DIR = platform.get_package_dir("framework-arduinoespressif32-libs") +FRAMEWORK_DIR = path_cache.framework_dir +FRAMEWORK_LIB_DIR = path_cache.framework_libs_dir SConscript("_embed_files.py", exports="env") -flag_any_custom_sdkconfig = os.path.exists(join(platform.get_package_dir("framework-arduinoespressif32-libs"),"sdkconfig")) +flag_any_custom_sdkconfig = exists(join(platform.get_package_dir("framework-arduinoespressif32-libs"),"sdkconfig")) + +def has_unicore_flags(): + """Check if any UNICORE flags are present in configuration""" + return any(flag in extra_flags or flag in entry_custom_sdkconfig + or flag in board_sdkconfig for flag in UNICORE_FLAGS) -# Esp32-solo1 libs needs adopted settings -if flag_custom_sdkconfig == True and ("CORE32SOLO1" in extra_flags or "CONFIG_FREERTOS_UNICORE=y" in entry_custom_sdkconfig or "CONFIG_FREERTOS_UNICORE=y" in board_sdkconfig): - if len(str(env.GetProjectOption("build_unflags"))) == 2: # No valid env, needs init +# Esp32-solo1 libs settings +if flag_custom_sdkconfig and has_unicore_flags(): + if len(str(env.GetProjectOption("build_unflags"))) == 2: # No valid env, needs init env['BUILD_UNFLAGS'] = {} - build_unflags = " ".join(env['BUILD_UNFLAGS']) - build_unflags = build_unflags + " -mdisable-hardware-atomics -ustart_app_other_cores" + + build_unflags = " ".join(env['BUILD_UNFLAGS']) + " -mdisable-hardware-atomics -ustart_app_other_cores" new_build_unflags = build_unflags.split() - env.Replace( - BUILD_UNFLAGS=new_build_unflags - ) + env.Replace(BUILD_UNFLAGS=new_build_unflags) + +def get_packages_to_install(deps, installed_packages): + """Generator for packages to install""" + for package, spec in deps.items(): + if package not in installed_packages: + yield package + else: + version_spec = semantic_version.Spec(spec) + if not version_spec.match(installed_packages[package]): + yield package def install_python_deps(): def _get_installed_pip_packages(): result = {} - packages = {} - pip_output = subprocess.check_output( - [ - env.subst("$PYTHONEXE"), - "-m", - "pip", - "list", - "--format=json", - "--disable-pip-version-check", - ] - ) try: + pip_output = subprocess.check_output([ + env.subst("$PYTHONEXE"), + "-m", "pip", "list", "--format=json", "--disable-pip-version-check" + ]) packages = json.loads(pip_output) - except: + for p in packages: + result[p["name"]] = pepver_to_semver(p["version"]) + except Exception: print("Warning! Couldn't extract the list of installed Python packages.") - return {} - for p in packages: - result[p["name"]] = pepver_to_semver(p["version"]) - + return result - deps = { - "wheel": ">=0.35.1", - "rich-click": ">=1.8.6", - "PyYAML": ">=6.0.2", - "intelhex": ">=2.3.0", - "esp-idf-size": ">=1.6.1" - } - installed_packages = _get_installed_pip_packages() - packages_to_install = [] - for package, spec in deps.items(): - if package not in installed_packages: - packages_to_install.append(package) - else: - version_spec = semantic_version.Spec(spec) - if not version_spec.match(installed_packages[package]): - packages_to_install.append(package) + packages_to_install = list(get_packages_to_install(python_deps, installed_packages)) if packages_to_install: + packages_str = " ".join(f'"{p}{python_deps[p]}"' for p in packages_to_install) env.Execute( env.VerboseAction( - ( - '"$PYTHONEXE" -m pip install -U -q -q -q ' - + " ".join( - [ - '"%s%s"' % (p, deps[p]) - for p in packages_to_install - ] - ) - ), + f'"$PYTHONEXE" -m pip install -U -q -q -q {packages_str}', "Installing Arduino Python dependencies", ) ) - return install_python_deps() def get_MD5_hash(phrase): - import hashlib - return hashlib.md5((phrase).encode('utf-8')).hexdigest()[:16] - + return hashlib.md5(phrase.encode('utf-8')).hexdigest()[:16] def matching_custom_sdkconfig(): - # check if current env is matching to existing sdkconfig + """Checks if current environment matches existing sdkconfig""" cust_sdk_is_present = False - matching_sdkconfig = False - last_sdkconfig_path = join(env.subst("$PROJECT_DIR"),"sdkconfig.defaults") - if flag_any_custom_sdkconfig == False: - matching_sdkconfig = True - return matching_sdkconfig, cust_sdk_is_present - if os.path.exists(last_sdkconfig_path) == False: - return matching_sdkconfig, cust_sdk_is_present - if flag_custom_sdkconfig == False: - matching_sdkconfig = False - return matching_sdkconfig, cust_sdk_is_present - with open(last_sdkconfig_path) as src: - line = src.readline() - if line.startswith("# TASMOTA__"): - cust_sdk_is_present = True; - costum_options = entry_custom_sdkconfig - if (line.split("__")[1]).strip() == get_MD5_hash((costum_options).strip() + mcu): - matching_sdkconfig = True - - return matching_sdkconfig, cust_sdk_is_present + + if not flag_any_custom_sdkconfig: + return True, cust_sdk_is_present + + last_sdkconfig_path = join(project_dir, "sdkconfig.defaults") + if not exists(last_sdkconfig_path): + return False, cust_sdk_is_present + + if not flag_custom_sdkconfig: + return False, cust_sdk_is_present + + try: + with open(last_sdkconfig_path) as src: + line = src.readline() + if line.startswith("# TASMOTA__"): + cust_sdk_is_present = True + custom_options = entry_custom_sdkconfig + expected_hash = get_MD5_hash(custom_options.strip() + mcu) + if line.split("__")[1].strip() == expected_hash: + return True, cust_sdk_is_present + except (IOError, IndexError): + pass + + return False, cust_sdk_is_present def check_reinstall_frwrk(): - framework_reinstall = False - cust_sdk_is_present = False - matching_sdkconfig = False - if flag_custom_sdkconfig == True: - matching_sdkconfig, cust_sdk_is_present = matching_custom_sdkconfig() - if flag_custom_sdkconfig == False and flag_any_custom_sdkconfig == True: - # case custom sdkconfig exists and a env without "custom_sdkconfig" - framework_reinstall = True - if flag_custom_sdkconfig == True and matching_sdkconfig == False: - # check if current custom sdkconfig is different from existing - framework_reinstall = True - return framework_reinstall - - -FRAMEWORK_SDK_DIR = fs.to_unix_path( - os.path.join( - FRAMEWORK_LIB_DIR, - mcu, - "include", - ) -) + if not flag_custom_sdkconfig and flag_any_custom_sdkconfig: + # case custom sdkconfig exists and an env without "custom_sdkconfig" + return True + + if flag_custom_sdkconfig: + matching_sdkconfig, _ = matching_custom_sdkconfig() + if not matching_sdkconfig: + # check if current custom sdkconfig is different from existing + return True + + return False -IS_INTEGRATION_DUMP = env.IsIntegrationDump() +def call_compile_libs(): + if mcu == "esp32c2": + ARDUINO_FRMWRK_C2_LIB_DIR = join(platform.get_package_dir("framework-arduinoespressif32-libs"),mcu) + if not os.path.exists(ARDUINO_FRMWRK_C2_LIB_DIR): + ARDUINO_C2_DIR = join(platform.get_package_dir("framework-arduino-c2-skeleton-lib"),mcu) + shutil.copytree(ARDUINO_C2_DIR, ARDUINO_FRMWRK_C2_LIB_DIR, dirs_exist_ok=True) + print(f"*** Compile Arduino IDF libs for {pioenv} ***") + SConscript("espidf.py") +FRAMEWORK_SDK_DIR = path_cache.sdk_dir +IS_INTEGRATION_DUMP = env.IsIntegrationDump() def is_framework_subfolder(potential_subfolder): - if not os.path.isabs(potential_subfolder): + """Check if a path is a subfolder of the framework SDK directory""" + # carefully check before change this function + if not isabs(potential_subfolder): return False - if ( - os.path.splitdrive(FRAMEWORK_SDK_DIR)[0] - != os.path.splitdrive(potential_subfolder)[0] - ): + if splitdrive(FRAMEWORK_SDK_DIR)[0] != splitdrive(potential_subfolder)[0]: return False - return os.path.commonpath([FRAMEWORK_SDK_DIR]) == os.path.commonpath( - [FRAMEWORK_SDK_DIR, potential_subfolder] - ) + return commonpath([FRAMEWORK_SDK_DIR]) == commonpath([FRAMEWORK_SDK_DIR, potential_subfolder]) +def calculate_include_path_length(includes): + """Calculate total character count of all include paths""" + return sum(len(str(inc)) for inc in includes) -def shorthen_includes(env, node): - if IS_INTEGRATION_DUMP: - # Don't shorten include paths for IDE integrations - return node +def analyze_path_distribution(includes): + """Analyze the distribution of include path lengths for optimization insights""" + if not includes: + return {} + + lengths = [len(str(inc)) for inc in includes] + framework_lengths = [len(str(inc)) for inc in includes if is_framework_subfolder(inc)] + + return { + 'total_paths': len(includes), + 'total_length': sum(lengths), + 'average_length': sum(lengths) / len(lengths), + 'max_length': max(lengths), + 'min_length': min(lengths), + 'framework_paths': len(framework_lengths), + 'framework_total_length': sum(framework_lengths), + 'framework_avg_length': sum(framework_lengths) / len(framework_lengths) if framework_lengths else 0 + } - includes = [fs.to_unix_path(inc) for inc in env.get("CPPPATH", [])] +def debug_framework_paths(env, include_count, total_length): + """Debug framework paths to understand the issue (verbose mode only)""" + if not env.get("VERBOSE"): + return + + print("*** Debug Framework Paths ***") + print(f"*** MCU: {mcu} ***") + print(f"*** FRAMEWORK_DIR: {FRAMEWORK_DIR} ***") + print(f"*** FRAMEWORK_SDK_DIR: {FRAMEWORK_SDK_DIR} ***") + print(f"*** SDK exists: {exists(FRAMEWORK_SDK_DIR)} ***") + print(f"*** Include count: {include_count} ***") + print(f"*** Total path length: {total_length} (threshold: {INCLUDE_PATH_LENGTH_THRESHOLD}) ***") + + includes = env.get("CPPPATH", []) + framework_count = 0 + longest_paths = sorted(includes, key=len, reverse=True)[:5] + + print("*** Longest include paths: ***") + for i, inc in enumerate(longest_paths): + is_fw = is_framework_subfolder(inc) + if is_fw: + framework_count += 1 + print(f"*** {i+1}: {inc} (length: {len(str(inc))}) -> Framework: {is_fw} ***") + + print(f"*** Framework includes found: {framework_count}/{len(includes)} ***") + + # Show path distribution analysis + analysis = analyze_path_distribution(includes) + print(f"*** Path Analysis: Avg={analysis.get('average_length', 0):.1f}, Max={analysis.get('max_length', 0)}, Framework Avg={analysis.get('framework_avg_length', 0):.1f} ***") + +def apply_include_shortening(env, node, includes, total_length): + """Applies include path shortening technique""" + env_get = env.get + to_unix_path = fs.to_unix_path + ccflags = env["CCFLAGS"] + asflags = env["ASFLAGS"] + + includes = [to_unix_path(inc) for inc in env_get("CPPPATH", [])] shortened_includes = [] generic_includes = [] + + original_length = total_length + saved_chars = 0 + for inc in includes: if is_framework_subfolder(inc): - shortened_includes.append( - "-iwithprefix/" - + fs.to_unix_path(os.path.relpath(inc, FRAMEWORK_SDK_DIR)) - ) + relative_path = to_unix_path(relpath(inc, FRAMEWORK_SDK_DIR)) + shortened_path = "-iwithprefix/" + relative_path + shortened_includes.append(shortened_path) + + # Calculate character savings + # Original: full path in -I flag + # New: -iprefix + shortened relative path + original_chars = len(f"-I{inc}") + new_chars = len(shortened_path) + saved_chars += max(0, original_chars - new_chars) else: generic_includes.append(inc) + # Show result message only once + if not _PATH_SHORTENING_MESSAGES['shortening_applied']: + if shortened_includes: + new_total_length = original_length - saved_chars + len(f"-iprefix{FRAMEWORK_SDK_DIR}") + print(f"*** Applied include path shortening for {len(shortened_includes)} framework paths ***") + print(f"*** Path length reduced from {original_length} to ~{new_total_length} characters ***") + print(f"*** Estimated savings: {saved_chars} characters ***") + else: + if not _PATH_SHORTENING_MESSAGES['no_framework_paths_warning']: + print("*** Warning: Path length high but no framework paths found for shortening ***") + print("*** This may indicate an architecture-specific issue ***") + print("*** Run with -v (verbose) for detailed path analysis ***") + _PATH_SHORTENING_MESSAGES['no_framework_paths_warning'] = True + _PATH_SHORTENING_MESSAGES['shortening_applied'] = True + + common_flags = ["-iprefix", FRAMEWORK_SDK_DIR] + shortened_includes + return env.Object( node, CPPPATH=generic_includes, - CCFLAGS=env["CCFLAGS"] - + ["-iprefix", FRAMEWORK_SDK_DIR] - + shortened_includes, - ASFLAGS=env["ASFLAGS"] - + ["-iprefix", FRAMEWORK_SDK_DIR] - + shortened_includes, + CCFLAGS=ccflags + common_flags, + ASFLAGS=asflags + common_flags, ) -# Check if framework = arduino, espidf is set -> compile Arduino as an component of IDF -# using platformio.ini entry since we modify the framework env var for Hybrid Compile! +def smart_include_length_shorten(env, node): + """Include path shortening based on total path length threshold""" + if IS_INTEGRATION_DUMP: + # Don't shorten include paths for IDE integrations + return node + + if not IS_WINDOWS: + return env.Object(node) + + # Check long path support once + check_and_warn_long_path_support() + + includes = env.get("CPPPATH", []) + include_count = len(includes) + total_path_length = calculate_include_path_length(includes) + + # Debug output in verbose mode + debug_framework_paths(env, include_count, total_path_length) + + # Apply shortening only if total path length exceeds threshold + # This is more accurate than just counting includes, as it considers + # the actual command line length impact + if total_path_length <= INCLUDE_PATH_LENGTH_THRESHOLD: + return env.Object(node) # Normal compilation + + # Apply include path shortening + return apply_include_shortening(env, node, includes, total_path_length) + def get_frameworks_in_current_env(): - current_env_section = "env:" + env["PIOENV"] + """Determines the frameworks of the current environment""" if "framework" in config.options(current_env_section): - frameworks = config.get(current_env_section, "framework", "") - return frameworks + return config.get(current_env_section, "framework", "") return [] +# Framework check current_env_frameworks = get_frameworks_in_current_env() if "arduino" in current_env_frameworks and "espidf" in current_env_frameworks: # Arduino as component is set, switch off Hybrid compile flag_custom_sdkconfig = False -def call_compile_libs(): - if mcu == "esp32c2": - ARDUINO_FRMWRK_C2_LIB_DIR = join(platform.get_package_dir("framework-arduinoespressif32-libs"),mcu) - if not os.path.exists(ARDUINO_FRMWRK_C2_LIB_DIR): - ARDUINO_C2_DIR = join(platform.get_package_dir("framework-arduino-c2-skeleton-lib"),mcu) - shutil.copytree(ARDUINO_C2_DIR, ARDUINO_FRMWRK_C2_LIB_DIR, dirs_exist_ok=True) - print("*** Compile Arduino IDF libs for %s ***" % env["PIOENV"]) - SConscript("espidf.py") - -if check_reinstall_frwrk() == True: - envs = [section.replace("env:", "") for section in config.sections() if section.startswith("env:")] - for env_name in envs: - file_path = join(env.subst("$PROJECT_DIR"), f"sdkconfig.{env_name}") - if exists(file_path): - os.remove(file_path) +# Framework reinstallation if required - IMPROVED WITH SECURE DELETION +if check_reinstall_frwrk(): + # Secure removal of SDKConfig files + safe_remove_sdkconfig_files() + print("*** Reinstall Arduino framework ***") - shutil.rmtree(platform.get_package_dir("framework-arduinoespressif32")) - shutil.rmtree(platform.get_package_dir("framework-arduinoespressif32-libs")) - ARDUINO_FRMWRK_URL = str(platform.get_package_spec("framework-arduinoespressif32")).split("uri=",1)[1][:-1] - ARDUINO_FRMWRK_LIB_URL = str(platform.get_package_spec("framework-arduinoespressif32-libs")).split("uri=",1)[1][:-1] - pm.install(ARDUINO_FRMWRK_URL) - pm.install(ARDUINO_FRMWRK_LIB_URL) - if flag_custom_sdkconfig == True: - call_compile_libs() - flag_custom_sdkconfig = False - -if flag_custom_sdkconfig == True and flag_any_custom_sdkconfig == False: + + # Secure framework cleanup + if safe_framework_cleanup(): + arduino_frmwrk_url = str(platform.get_package_spec("framework-arduinoespressif32")).split("uri=", 1)[1][:-1] + arduino_frmwrk_lib_url = str(platform.get_package_spec("framework-arduinoespressif32-libs")).split("uri=",1)[1][:-1] + pm.install(arduino_frmwrk_url) + pm.install(arduino_frmwrk_lib_url) + + if flag_custom_sdkconfig: + call_compile_libs() + flag_custom_sdkconfig = False + else: + logging.error("Framework cleanup failed - installation aborted") + sys.exit(1) + +if flag_custom_sdkconfig and not flag_any_custom_sdkconfig: call_compile_libs() -if "arduino" in env.subst("$PIOFRAMEWORK") and "espidf" not in env.subst("$PIOFRAMEWORK") and env.subst("$ARDUINO_LIB_COMPILE_FLAG") in ("Inactive", "True"): +# Main logic for Arduino Framework +pioframework = env.subst("$PIOFRAMEWORK") +arduino_lib_compile_flag = env.subst("$ARDUINO_LIB_COMPILE_FLAG") + +if ("arduino" in pioframework and "espidf" not in pioframework and + arduino_lib_compile_flag in ("Inactive", "True")): + + if flag_custom_component_remove or flag_lib_ignore: + flag_remove = True + from component_manager import ComponentManager + component_manager = ComponentManager(env) + component_manager.handle_component_settings( + add_components=flag_custom_component_add, + remove_components=flag_remove + ) + silent_action = env.Action(component_manager.restore_pioarduino_build_py) + silent_action.strfunction = lambda target, source, env: '' # hack to silence scons command output + env.AddPostAction("checkprogsize", silent_action) + if IS_WINDOWS: - env.AddBuildMiddleware(shorthen_includes) - if os.path.exists(join(platform.get_package_dir( - "framework-arduinoespressif32"), "tools", "platformio-build.py")): - PIO_BUILD = "platformio-build.py" - else: - PIO_BUILD = "pioarduino-build.py" - SConscript(join(platform.get_package_dir("framework-arduinoespressif32"), "tools", PIO_BUILD)) + # Smart include path optimization based on total path length + env.AddBuildMiddleware(smart_include_length_shorten) + + build_script_path = join(FRAMEWORK_DIR, "tools", "pioarduino-build.py") + SConscript(build_script_path) From b93c3bf7ad0d7975ddcfb521f0e66f4985f26b5f Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Thu, 29 May 2025 00:56:08 +0200 Subject: [PATCH 07/97] Update arduino.py --- builder/frameworks/arduino.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/builder/frameworks/arduino.py b/builder/frameworks/arduino.py index 657278cde..8a7866bab 100644 --- a/builder/frameworks/arduino.py +++ b/builder/frameworks/arduino.py @@ -220,9 +220,10 @@ def validate_platformio_path(path: Union[str, Path]) -> bool: # Must be framework-related framework_indicators = [ "framework-arduinoespressif32", - "esp32-arduino-libs", ".platformio/packages", - "packages/framework-arduinoespressif32" + "packages/framework-arduinoespressif32", + "packages/framework-arduinoespressif32-libs", + "packages/framework-arduino-c2-skeleton-lib" ] if not any(indicator in path_str for indicator in framework_indicators): From c287ddbdbb5a09bc222f7ae01df12e8eb9ef7ad3 Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Thu, 29 May 2025 01:00:38 +0200 Subject: [PATCH 08/97] Update arduino.py --- builder/frameworks/arduino.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/builder/frameworks/arduino.py b/builder/frameworks/arduino.py index 8a7866bab..28c93117d 100644 --- a/builder/frameworks/arduino.py +++ b/builder/frameworks/arduino.py @@ -293,12 +293,15 @@ def safe_framework_cleanup(): logging.info(f"Framework path validated successfully: {FRAMEWORK_DIR}") if safe_delete_directory(FRAMEWORK_DIR): - #print("Framework successfully removed") + print("Framework successfully removed") + else: + print("Error removing framework") + return False else: logging.error(f"PlatformIO path validation failed: {FRAMEWORK_DIR}") return False - logging.info(f"Attempting to validate framework path: {FRAMEWORK_LIB_DIR}") + logging.info(f"Attempting to validate framework lib path: {FRAMEWORK_LIB_DIR}") # Use specialized PlatformIO path validation if validate_platformio_path(FRAMEWORK_LIB_DIR): @@ -306,7 +309,7 @@ def safe_framework_cleanup(): logging.info(f"Framework lib path validated successfully: {FRAMEWORK_LIB_DIR}") if safe_delete_directory(FRAMEWORK_LIB_DIR): - #print("Framework successfully removed") + print("Framework libs successfully removed") return True else: print("Error removing framework") From c19d1a0cf043eee71431aee3ad4bb3b6fe623b2c Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Thu, 29 May 2025 14:02:24 +0200 Subject: [PATCH 09/97] Update arduino.py --- builder/frameworks/arduino.py | 183 ++++++++++++++++++---------------- 1 file changed, 99 insertions(+), 84 deletions(-) diff --git a/builder/frameworks/arduino.py b/builder/frameworks/arduino.py index 28c93117d..330ebe0f3 100644 --- a/builder/frameworks/arduino.py +++ b/builder/frameworks/arduino.py @@ -30,6 +30,7 @@ import shutil import hashlib import logging +import threading from contextlib import suppress from os.path import join, exists, isabs, splitdrive, commonpath, relpath from pathlib import Path @@ -81,7 +82,8 @@ def setup_logging(): "CONFIG_FREERTOS_UNICORE=y" } -# Global flags to prevent message spam +# Thread-safe global flags to prevent message spam +_PATH_SHORTENING_LOCK = threading.Lock() _PATH_SHORTENING_MESSAGES = { 'shortening_applied': False, 'no_framework_paths_warning': False, @@ -95,6 +97,7 @@ def __init__(self, platform, mcu): self.platform = platform self.mcu = mcu self._framework_dir = None + self._framework_lib_dir = None self._sdk_dir = None @property @@ -104,7 +107,7 @@ def framework_dir(self): return self._framework_dir @property - def framework_dir(self): + def framework_lib_dir(self): if self._framework_lib_dir is None: self._framework_lib_dir = self.platform.get_package_dir("framework-arduinoespressif32-libs") return self._framework_lib_dir @@ -119,32 +122,33 @@ def sdk_dir(self): def check_and_warn_long_path_support(): """Checks Windows long path support and issues warning if disabled""" - if not IS_WINDOWS or _PATH_SHORTENING_MESSAGES['long_path_warning_shown']: - return - - try: - import winreg - key = winreg.OpenKey( - winreg.HKEY_LOCAL_MACHINE, - r"SYSTEM\CurrentControlSet\Control\FileSystem" - ) - value, _ = winreg.QueryValueEx(key, "LongPathsEnabled") - winreg.CloseKey(key) + with _PATH_SHORTENING_LOCK: # Thread-safe access + if not IS_WINDOWS or _PATH_SHORTENING_MESSAGES['long_path_warning_shown']: + return + + try: + import winreg + key = winreg.OpenKey( + winreg.HKEY_LOCAL_MACHINE, + r"SYSTEM\CurrentControlSet\Control\FileSystem" + ) + value, _ = winreg.QueryValueEx(key, "LongPathsEnabled") + winreg.CloseKey(key) + + if value != 1: + print("*** WARNING: Windows Long Path Support is disabled ***") + print("*** Enable it for better performance: ***") + print("*** 1. Run as Administrator: gpedit.msc ***") + print("*** 2. Navigate to: Computer Configuration > Administrative Templates > System > Filesystem ***") + print("*** 3. Enable 'Enable Win32 long paths' ***") + print("*** OR run PowerShell as Admin: ***") + print("*** New-ItemProperty -Path 'HKLM:\\SYSTEM\\CurrentControlSet\\Control\\FileSystem' -Name 'LongPathsEnabled' -Value 1 -PropertyType DWORD -Force ***") + print("*** Restart required after enabling ***") + except Exception: + print("*** WARNING: Could not check Long Path Support status ***") + print("*** Consider enabling Windows Long Path Support for better performance ***") - if value != 1: - print("*** WARNING: Windows Long Path Support is disabled ***") - print("*** Enable it for better performance: ***") - print("*** 1. Run as Administrator: gpedit.msc ***") - print("*** 2. Navigate to: Computer Configuration > Administrative Templates > System > Filesystem ***") - print("*** 3. Enable 'Enable Win32 long paths' ***") - print("*** OR run PowerShell as Admin: ***") - print("*** New-ItemProperty -Path 'HKLM:\\SYSTEM\\CurrentControlSet\\Control\\FileSystem' -Name 'LongPathsEnabled' -Value 1 -PropertyType DWORD -Force ***") - print("*** Restart required after enabling ***") - except Exception: - print("*** WARNING: Could not check Long Path Support status ***") - print("*** Consider enabling Windows Long Path Support for better performance ***") - - _PATH_SHORTENING_MESSAGES['long_path_warning_shown'] = True + _PATH_SHORTENING_MESSAGES['long_path_warning_shown'] = True # Secure deletion functions def safe_delete_file(file_path: Union[str, Path], @@ -204,34 +208,37 @@ def safe_delete_directory(dir_path: Union[str, Path]) -> bool: def validate_platformio_path(path: Union[str, Path]) -> bool: """ - Special validation for PlatformIO package paths + Enhanced validation for PlatformIO package paths """ - path = Path(path).resolve() - path_str = str(path) - - # Must be within .platformio directory structure - if ".platformio" not in path_str: - return False - - # Must be a packages directory - if "packages" not in path_str: - return False + try: + path = Path(path).resolve() + path_str = str(path) - # Must be framework-related - framework_indicators = [ - "framework-arduinoespressif32", - ".platformio/packages", - "packages/framework-arduinoespressif32", - "packages/framework-arduinoespressif32-libs", - "packages/framework-arduino-c2-skeleton-lib" - ] - - if not any(indicator in path_str for indicator in framework_indicators): + # Must be within .platformio directory structure + if ".platformio" not in path_str: + return False + + # Must be a packages directory + if "packages" not in path_str: + return False + + # Must be framework-related + framework_indicators = [ + "framework-arduinoespressif32", + "framework-arduinoespressif32-libs", + "framework-arduino-c2-skeleton-lib" + ] + + if not any(indicator in path_str for indicator in framework_indicators): + return False + + # Must not be a critical system path + critical_paths = ["/usr", "/bin", "/sbin", "/etc", "/boot", "C:\\Windows", "C:\\Program Files"] + return not any(critical in path_str for critical in critical_paths) + + except Exception as e: + logging.error(f"Path validation error: {e}") return False - - # Must not be a critical system path - critical_paths = ["/usr", "/bin", "/sbin", "/etc", "/boot"] - return not any(critical in path_str for critical in critical_paths) def validate_deletion_path(path: Union[str, Path], allowed_patterns: List[str]) -> bool: @@ -281,43 +288,42 @@ def validate_deletion_path(path: Union[str, Path], return is_allowed def safe_framework_cleanup(): - """Secure cleanup of Arduino Framework""" + """Secure cleanup of Arduino Framework with enhanced error handling""" + success = True - # Secure deletion of framework directories + # Framework directory cleanup if exists(FRAMEWORK_DIR): logging.info(f"Attempting to validate framework path: {FRAMEWORK_DIR}") - # Use specialized PlatformIO path validation if validate_platformio_path(FRAMEWORK_DIR): - #print("*** Secure framework cleanup ***") logging.info(f"Framework path validated successfully: {FRAMEWORK_DIR}") if safe_delete_directory(FRAMEWORK_DIR): print("Framework successfully removed") else: print("Error removing framework") - return False + success = False else: logging.error(f"PlatformIO path validation failed: {FRAMEWORK_DIR}") - return False - + success = False + + # Framework libs directory cleanup + if exists(FRAMEWORK_LIB_DIR): logging.info(f"Attempting to validate framework lib path: {FRAMEWORK_LIB_DIR}") - # Use specialized PlatformIO path validation if validate_platformio_path(FRAMEWORK_LIB_DIR): - #print("*** Secure framework cleanup ***") logging.info(f"Framework lib path validated successfully: {FRAMEWORK_LIB_DIR}") if safe_delete_directory(FRAMEWORK_LIB_DIR): print("Framework libs successfully removed") - return True else: - print("Error removing framework") - return False + print("Error removing framework libs") + success = False else: logging.error(f"PlatformIO path validation failed: {FRAMEWORK_LIB_DIR}") - return False - return True + success = False + + return success def safe_remove_sdkconfig_files(): """Secure removal of SDKConfig files""" @@ -499,9 +505,17 @@ def is_framework_subfolder(potential_subfolder): return False return commonpath([FRAMEWORK_SDK_DIR]) == commonpath([FRAMEWORK_SDK_DIR, potential_subfolder]) +# Performance optimization with caching def calculate_include_path_length(includes): - """Calculate total character count of all include paths""" - return sum(len(str(inc)) for inc in includes) + """Calculate total character count of all include paths with caching""" + if not hasattr(calculate_include_path_length, '_cache'): + calculate_include_path_length._cache = {} + + cache_key = tuple(includes) + if cache_key not in calculate_include_path_length._cache: + calculate_include_path_length._cache[cache_key] = sum(len(str(inc)) for inc in includes) + + return calculate_include_path_length._cache[cache_key] def analyze_path_distribution(includes): """Analyze the distribution of include path lengths for optimization insights""" @@ -581,20 +595,21 @@ def apply_include_shortening(env, node, includes, total_length): else: generic_includes.append(inc) - # Show result message only once - if not _PATH_SHORTENING_MESSAGES['shortening_applied']: - if shortened_includes: - new_total_length = original_length - saved_chars + len(f"-iprefix{FRAMEWORK_SDK_DIR}") - print(f"*** Applied include path shortening for {len(shortened_includes)} framework paths ***") - print(f"*** Path length reduced from {original_length} to ~{new_total_length} characters ***") - print(f"*** Estimated savings: {saved_chars} characters ***") - else: - if not _PATH_SHORTENING_MESSAGES['no_framework_paths_warning']: - print("*** Warning: Path length high but no framework paths found for shortening ***") - print("*** This may indicate an architecture-specific issue ***") - print("*** Run with -v (verbose) for detailed path analysis ***") - _PATH_SHORTENING_MESSAGES['no_framework_paths_warning'] = True - _PATH_SHORTENING_MESSAGES['shortening_applied'] = True + # Show result message only once with thread safety + with _PATH_SHORTENING_LOCK: + if not _PATH_SHORTENING_MESSAGES['shortening_applied']: + if shortened_includes: + new_total_length = original_length - saved_chars + len(f"-iprefix{FRAMEWORK_SDK_DIR}") + print(f"*** Applied include path shortening for {len(shortened_includes)} framework paths ***") + print(f"*** Path length reduced from {original_length} to ~{new_total_length} characters ***") + print(f"*** Estimated savings: {saved_chars} characters ***") + else: + if not _PATH_SHORTENING_MESSAGES['no_framework_paths_warning']: + print("*** Warning: Path length high but no framework paths found for shortening ***") + print("*** This may indicate an architecture-specific issue ***") + print("*** Run with -v (verbose) for detailed path analysis ***") + _PATH_SHORTENING_MESSAGES['no_framework_paths_warning'] = True + _PATH_SHORTENING_MESSAGES['shortening_applied'] = True common_flags = ["-iprefix", FRAMEWORK_SDK_DIR] + shortened_includes @@ -645,14 +660,14 @@ def get_frameworks_in_current_env(): # Arduino as component is set, switch off Hybrid compile flag_custom_sdkconfig = False -# Framework reinstallation if required - IMPROVED WITH SECURE DELETION +# Framework reinstallation if required - Enhanced with secure deletion and error handling if check_reinstall_frwrk(): # Secure removal of SDKConfig files safe_remove_sdkconfig_files() print("*** Reinstall Arduino framework ***") - # Secure framework cleanup + # Secure framework cleanup with enhanced error handling if safe_framework_cleanup(): arduino_frmwrk_url = str(platform.get_package_spec("framework-arduinoespressif32")).split("uri=", 1)[1][:-1] arduino_frmwrk_lib_url = str(platform.get_package_spec("framework-arduinoespressif32-libs")).split("uri=",1)[1][:-1] From cb55e0027f93b1641c16c364467f4598fc0afae4 Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Thu, 29 May 2025 14:06:40 +0200 Subject: [PATCH 10/97] Update arduino.py --- builder/frameworks/arduino.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/builder/frameworks/arduino.py b/builder/frameworks/arduino.py index 330ebe0f3..c64b3dfb3 100644 --- a/builder/frameworks/arduino.py +++ b/builder/frameworks/arduino.py @@ -380,7 +380,7 @@ def safe_remove_sdkconfig_files(): framework_reinstall = False FRAMEWORK_DIR = path_cache.framework_dir -FRAMEWORK_LIB_DIR = path_cache.framework_libs_dir +FRAMEWORK_LIB_DIR = path_cache.framework_lib_dir SConscript("_embed_files.py", exports="env") From 5d892a2abab0e3c892b023e759cab4b8cef2e59e Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Thu, 29 May 2025 14:10:50 +0200 Subject: [PATCH 11/97] Update component_manager.py --- builder/frameworks/component_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/builder/frameworks/component_manager.py b/builder/frameworks/component_manager.py index 0bf816b95..d8b8985da 100644 --- a/builder/frameworks/component_manager.py +++ b/builder/frameworks/component_manager.py @@ -22,7 +22,7 @@ def __init__(self, env): self.ignored_libs: Set[str] = set() self.arduino_framework_dir = self.platform.get_package_dir("framework-arduinoespressif32") - self.arduino_libs_mcu = join(self.platform.get_package_dir("framework-arduinoespressif32-libs", self.mcu) + self.arduino_libs_mcu = join(self.platform.get_package_dir("framework-arduinoespressif32-libs", self.mcu)) def handle_component_settings(self, add_components: bool = False, remove_components: bool = False) -> None: """Handle adding and removing IDF components based on project configuration.""" From 7b4ea71b697d2e2b1fdc8dfc98b02311a0693ac4 Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Thu, 29 May 2025 14:17:55 +0200 Subject: [PATCH 12/97] Update component_manager.py --- builder/frameworks/component_manager.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/builder/frameworks/component_manager.py b/builder/frameworks/component_manager.py index d8b8985da..f28b8097b 100644 --- a/builder/frameworks/component_manager.py +++ b/builder/frameworks/component_manager.py @@ -22,7 +22,8 @@ def __init__(self, env): self.ignored_libs: Set[str] = set() self.arduino_framework_dir = self.platform.get_package_dir("framework-arduinoespressif32") - self.arduino_libs_mcu = join(self.platform.get_package_dir("framework-arduinoespressif32-libs", self.mcu)) + base_libs_dir = self.platform.get_package_dir("framework-arduinoespressif32-libs") + self.arduino_libs_mcu = join(base_libs_dir, self.mcu) def handle_component_settings(self, add_components: bool = False, remove_components: bool = False) -> None: """Handle adding and removing IDF components based on project configuration.""" From fcd889ef812dc943b9515802df4f7fa79d526e88 Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Thu, 29 May 2025 14:20:16 +0200 Subject: [PATCH 13/97] Update component_manager.py --- builder/frameworks/component_manager.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/builder/frameworks/component_manager.py b/builder/frameworks/component_manager.py index f28b8097b..0ea43f4df 100644 --- a/builder/frameworks/component_manager.py +++ b/builder/frameworks/component_manager.py @@ -22,8 +22,7 @@ def __init__(self, env): self.ignored_libs: Set[str] = set() self.arduino_framework_dir = self.platform.get_package_dir("framework-arduinoespressif32") - base_libs_dir = self.platform.get_package_dir("framework-arduinoespressif32-libs") - self.arduino_libs_mcu = join(base_libs_dir, self.mcu) + self.arduino_libs_mcu = join(self.platform.get_package_dir("framework-arduinoespressif32-libs"), self.mcu) def handle_component_settings(self, add_components: bool = False, remove_components: bool = False) -> None: """Handle adding and removing IDF components based on project configuration.""" From dff5544973f8d168ae3c7c8ca629e50dcf31f3b8 Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Thu, 29 May 2025 14:36:10 +0200 Subject: [PATCH 14/97] Update espidf.py --- builder/frameworks/espidf.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/builder/frameworks/espidf.py b/builder/frameworks/espidf.py index 191e7280e..c308bc33d 100644 --- a/builder/frameworks/espidf.py +++ b/builder/frameworks/espidf.py @@ -172,7 +172,7 @@ def _get_installed_standard_pip_packages(): os.rename(ARDUINO_FRAMEWORK_DIR, new_path) ARDUINO_FRAMEWORK_DIR = new_path assert ARDUINO_FRAMEWORK_DIR and os.path.isdir(ARDUINO_FRAMEWORK_DIR) - arduino_libs_mcu = join(ARDUINO_FRAMEWORK_DIR,"tools","esp32-arduino-libs",mcu) + arduino_libs_mcu = join(platform.get_package_dir("framework-arduinoespressif32-libs"),mcu) BUILD_DIR = env.subst("$BUILD_DIR") PROJECT_DIR = env.subst("$PROJECT_DIR") @@ -294,7 +294,7 @@ def add_flash_configuration(config_flags): def write_sdkconfig_file(idf_config_flags, checksum_source): """Write the final sdkconfig.defaults file with checksum.""" - sdkconfig_src = join(ARDUINO_FRAMEWORK_DIR, "tools", "esp32-arduino-libs", mcu, "sdkconfig") + sdkconfig_src = join(arduino_libs_mcu, "sdkconfig") sdkconfig_dst = join(PROJECT_DIR, "sdkconfig.defaults") # Generate checksum for validation (maintains original logic) From 810ff2f934b87336d63f07f8431a4c1b8a30b83a Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Thu, 29 May 2025 14:46:21 +0200 Subject: [PATCH 15/97] Update espidf.py --- builder/frameworks/espidf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/builder/frameworks/espidf.py b/builder/frameworks/espidf.py index c308bc33d..2fbd1d0e1 100644 --- a/builder/frameworks/espidf.py +++ b/builder/frameworks/espidf.py @@ -159,7 +159,7 @@ def _get_installed_standard_pip_packages(): ARDUINO_FRAMEWORK_DIR = platform.get_package_dir("framework-arduinoespressif32") ARDUINO_FRMWRK_LIB_DIR = platform.get_package_dir("framework-arduinoespressif32-libs") if mcu == "esp32c2": - ARDUINO_FRMWRK_C2_LIB_DIR = join(platform.get_package_dir("framework-arduinoespressif32-libs"),mcu) + ARDUINO_FRMWRK_C2_LIB_DIR = join(ARDUINO_FRMWRK_LIB_DIR, mcu) if not os.path.exists(ARDUINO_FRMWRK_C2_LIB_DIR): ARDUINO_C2_DIR = join(platform.get_package_dir("framework-arduino-c2-skeleton-lib"),mcu) shutil.copytree(ARDUINO_C2_DIR, ARDUINO_FRMWRK_C2_LIB_DIR, dirs_exist_ok=True) From 3063ee60db1e2542535721dea3ecf5c8fcc510cc Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Thu, 29 May 2025 15:17:50 +0200 Subject: [PATCH 16/97] C2 always Hybrid Compile --- builder/frameworks/arduino.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/builder/frameworks/arduino.py b/builder/frameworks/arduino.py index c64b3dfb3..e20399ab8 100644 --- a/builder/frameworks/arduino.py +++ b/builder/frameworks/arduino.py @@ -681,7 +681,9 @@ def get_frameworks_in_current_env(): logging.error("Framework cleanup failed - installation aborted") sys.exit(1) -if flag_custom_sdkconfig and not flag_any_custom_sdkconfig: +if mcu == "esp32c2": + call_compile_libs() +elif flag_custom_sdkconfig and not flag_any_custom_sdkconfig: call_compile_libs() # Main logic for Arduino Framework From d41ade92630a736c834564fc251660d360be5630 Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Thu, 29 May 2025 15:34:17 +0200 Subject: [PATCH 17/97] set flag for Hybrid Compile for the C2 --- builder/frameworks/arduino.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/builder/frameworks/arduino.py b/builder/frameworks/arduino.py index e20399ab8..887c709c0 100644 --- a/builder/frameworks/arduino.py +++ b/builder/frameworks/arduino.py @@ -682,7 +682,7 @@ def get_frameworks_in_current_env(): sys.exit(1) if mcu == "esp32c2": - call_compile_libs() + flag_custom_sdkconfig = True elif flag_custom_sdkconfig and not flag_any_custom_sdkconfig: call_compile_libs() From 68395ebc4f0f6217d1377ecc05b1702e5f9dedf0 Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Thu, 29 May 2025 15:35:17 +0200 Subject: [PATCH 18/97] Update arduino.py --- builder/frameworks/arduino.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/builder/frameworks/arduino.py b/builder/frameworks/arduino.py index 887c709c0..cf4cc05b5 100644 --- a/builder/frameworks/arduino.py +++ b/builder/frameworks/arduino.py @@ -683,7 +683,7 @@ def get_frameworks_in_current_env(): if mcu == "esp32c2": flag_custom_sdkconfig = True -elif flag_custom_sdkconfig and not flag_any_custom_sdkconfig: +if flag_custom_sdkconfig and not flag_any_custom_sdkconfig: call_compile_libs() # Main logic for Arduino Framework From cdcb5c257bb3c4c449677dcfe98d5019c24054ed Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Thu, 29 May 2025 15:38:10 +0200 Subject: [PATCH 19/97] Update component_manager.py --- builder/frameworks/component_manager.py | 56 +++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/builder/frameworks/component_manager.py b/builder/frameworks/component_manager.py index 0ea43f4df..16a58a988 100644 --- a/builder/frameworks/component_manager.py +++ b/builder/frameworks/component_manager.py @@ -297,6 +297,54 @@ def _convert_lib_name_to_include(self, lib_name: str) -> str: return cleaned_name + def _has_bt_ble_dependencies(self) -> bool: + """Check if lib_deps contains any BT/BLE related dependencies.""" + try: + # Get lib_deps from current environment + lib_deps = self.env.GetProjectOption("lib_deps", []) + + if isinstance(lib_deps, str): + lib_deps = [lib_deps] + elif lib_deps is None: + lib_deps = [] + + # Convert to string and check for BT/BLE keywords + lib_deps_str = ' '.join(str(dep) for dep in lib_deps).upper() + + bt_ble_keywords = ['BLE', 'BT', 'NIMBLE', 'BLUETOOTH'] + + for keyword in bt_ble_keywords: + if keyword in lib_deps_str: + return True + + return False + + except Exception: + return False + + def _is_bt_related_library(self, lib_name: str) -> bool: + """Check if a library name is related to Bluetooth/BLE functionality.""" + lib_name_upper = lib_name.upper() + + bt_related_names = [ + 'BT', + 'BLE', + 'BLUETOOTH', + 'NIMBLE', + 'ESP32_BLE', + 'ESP32BLE', + 'BLUETOOTHSERIAL', + 'BLE_ARDUINO', + 'ESP_BLE', + 'ESP_BT' + ] + + for bt_name in bt_related_names: + if bt_name in lib_name_upper: + return True + + return False + def _remove_ignored_lib_includes(self) -> None: """Remove include entries for ignored libraries from pioarduino-build.py.""" build_py_path = join(self.arduino_libs_mcu, "pioarduino-build.py") @@ -304,6 +352,9 @@ def _remove_ignored_lib_includes(self) -> None: if not os.path.exists(build_py_path): return + # Check if BT/BLE dependencies exist in lib_deps + bt_ble_protected = self._has_bt_ble_dependencies() + try: with open(build_py_path, 'r') as f: content = f.read() @@ -313,6 +364,11 @@ def _remove_ignored_lib_includes(self) -> None: # Remove CPPPATH entries for each ignored library for lib_name in self.ignored_libs: + # Skip BT-related libraries if BT/BLE dependencies are present + if bt_ble_protected and self._is_bt_related_library(lib_name): + print(f"Skipping removal of BT-related library '{lib_name}' due to BT/BLE dependency in lib_deps") + continue + # Multiple patterns to catch different include formats patterns = [ rf'.*join\([^,]*,\s*"include",\s*"{re.escape(lib_name)}"[^)]*\),?\n', From 19ab7504a92c9c020723cc29960516cab0430181 Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Thu, 29 May 2025 15:59:45 +0200 Subject: [PATCH 20/97] Update arduino.py --- builder/frameworks/arduino.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/builder/frameworks/arduino.py b/builder/frameworks/arduino.py index cf4cc05b5..a728ef055 100644 --- a/builder/frameworks/arduino.py +++ b/builder/frameworks/arduino.py @@ -355,6 +355,9 @@ def safe_remove_sdkconfig_files(): flag_custom_component_add = False flag_lib_ignore = False +if mcu == "esp32c2": + flag_custom_sdkconfig = True + # pio lib_ignore check if config.has_option(current_env_section, "lib_ignore"): flag_lib_ignore = True @@ -681,8 +684,6 @@ def get_frameworks_in_current_env(): logging.error("Framework cleanup failed - installation aborted") sys.exit(1) -if mcu == "esp32c2": - flag_custom_sdkconfig = True if flag_custom_sdkconfig and not flag_any_custom_sdkconfig: call_compile_libs() From f2880bc59a04bd7b1602d8ceb7ac82a274669de1 Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Thu, 29 May 2025 16:50:54 +0200 Subject: [PATCH 21/97] include path remove with smart components detection logic --- builder/frameworks/component_manager.py | 161 ++++++++++++++++-------- 1 file changed, 107 insertions(+), 54 deletions(-) diff --git a/builder/frameworks/component_manager.py b/builder/frameworks/component_manager.py index 16a58a988..07ae14050 100644 --- a/builder/frameworks/component_manager.py +++ b/builder/frameworks/component_manager.py @@ -116,6 +116,110 @@ def _get_lib_ignore_entries(self) -> List[str]: except Exception: return [] + def _analyze_project_dependencies(self) -> Set[str]: + """Analyze project files to detect actually used components/libraries.""" + used_components = set() + + try: + # Analyze project source files + src_dir = self.env.subst("$PROJECT_SRC_DIR") + if os.path.exists(src_dir): + for root, dirs, files in os.walk(src_dir): + for file in files: + if file.endswith(('.cpp', '.c', '.h', '.hpp', '.ino')): + file_path = os.path.join(root, file) + used_components.update(self._extract_components_from_file(file_path)) + + # Analyze lib_deps for explicit dependencies (if available) + lib_deps = self.env.GetProjectOption("lib_deps", []) + if isinstance(lib_deps, str): + lib_deps = [lib_deps] + + for dep in lib_deps: + used_components.update(self._extract_components_from_lib_dep(str(dep))) + + except Exception: + pass + + return used_components + + def _extract_components_from_file(self, file_path: str) -> Set[str]: + """Extract component usage from a single file by analyzing includes and function calls.""" + components = set() + + # Component detection patterns - maps component names to search patterns + component_patterns = { + 'bt': ['bluetooth', 'ble', 'nimble', 'bt_', 'esp_bt', 'esp_ble'], + 'esp_wifi': ['wifi', 'esp_wifi', 'tcpip_adapter'], + 'esp_dsp': ['dsps_', 'esp_dsp', 'fft2r'], + 'esp_http_client': ['esp_http_client', 'http_client'], + 'esp_https_ota': ['esp_https_ota', 'esp_ota'], + 'mdns': ['mdns', 'esp_mdns'], + 'mqtt': ['mqtt', 'esp_mqtt'], + 'spiffs': ['spiffs', 'esp_spiffs'], + 'fatfs': ['fatfs', 'ff.h'], + 'nvs_flash': ['nvs', 'nvs_flash'], + 'esp_timer': ['esp_timer', 'timer_'], + 'driver': ['gpio_', 'uart_', 'spi_', 'i2c_', 'adc_', 'dac_'], + 'esp_camera': ['esp_camera', 'camera.h'], + 'esp_now': ['esp_now', 'espnow'], + 'esp_smartconfig': ['smartconfig', 'esp_smartconfig'] + } + + try: + with open(file_path, 'r', encoding='utf-8', errors='ignore') as f: + content = f.read().lower() + + # Check each component pattern against file content + for component, patterns in component_patterns.items(): + if any(pattern in content for pattern in patterns): + components.add(component) + + except Exception: + pass + + return components + + def _extract_components_from_lib_dep(self, lib_dep: str) -> Set[str]: + """Extract components from lib_deps entry by mapping library names to components.""" + components = set() + lib_dep_upper = lib_dep.upper() + + # Map lib_deps entries to ESP-IDF components + lib_dep_mapping = { + 'bt': ['BLE', 'BT', 'BLUETOOTH', 'NIMBLE'], + 'esp_wifi': ['WIFI', 'ASYNCTCP', 'ESPASYNCWEBSERVER'], + 'esp_dsp': ['DSP', 'FFT', 'JPEG'], + 'esp_http_client': ['HTTP', 'HTTPCLIENT'], + 'mqtt': ['MQTT', 'PUBSUB'], + 'esp_camera': ['CAMERA', 'ESP32CAM'] + } + + for component, keywords in lib_dep_mapping.items(): + if any(keyword in lib_dep_upper for keyword in keywords): + components.add(component) + + return components + + def _is_component_used_in_project(self, lib_name: str) -> bool: + """Check if a component/library is actually used in the project.""" + # Cache project analysis results for performance + if not hasattr(self, '_project_components_cache'): + self._project_components_cache = self._analyze_project_dependencies() + + lib_name_lower = lib_name.lower() + + # Direct match + if lib_name_lower in self._project_components_cache: + return True + + # Partial match for related components + for used_component in self._project_components_cache: + if lib_name_lower in used_component or used_component in lib_name_lower: + return True + + return False + def _get_arduino_core_libraries(self) -> Dict[str, str]: """Get all Arduino core libraries and their corresponding include paths.""" libraries_mapping = {} @@ -297,54 +401,6 @@ def _convert_lib_name_to_include(self, lib_name: str) -> str: return cleaned_name - def _has_bt_ble_dependencies(self) -> bool: - """Check if lib_deps contains any BT/BLE related dependencies.""" - try: - # Get lib_deps from current environment - lib_deps = self.env.GetProjectOption("lib_deps", []) - - if isinstance(lib_deps, str): - lib_deps = [lib_deps] - elif lib_deps is None: - lib_deps = [] - - # Convert to string and check for BT/BLE keywords - lib_deps_str = ' '.join(str(dep) for dep in lib_deps).upper() - - bt_ble_keywords = ['BLE', 'BT', 'NIMBLE', 'BLUETOOTH'] - - for keyword in bt_ble_keywords: - if keyword in lib_deps_str: - return True - - return False - - except Exception: - return False - - def _is_bt_related_library(self, lib_name: str) -> bool: - """Check if a library name is related to Bluetooth/BLE functionality.""" - lib_name_upper = lib_name.upper() - - bt_related_names = [ - 'BT', - 'BLE', - 'BLUETOOTH', - 'NIMBLE', - 'ESP32_BLE', - 'ESP32BLE', - 'BLUETOOTHSERIAL', - 'BLE_ARDUINO', - 'ESP_BLE', - 'ESP_BT' - ] - - for bt_name in bt_related_names: - if bt_name in lib_name_upper: - return True - - return False - def _remove_ignored_lib_includes(self) -> None: """Remove include entries for ignored libraries from pioarduino-build.py.""" build_py_path = join(self.arduino_libs_mcu, "pioarduino-build.py") @@ -352,9 +408,6 @@ def _remove_ignored_lib_includes(self) -> None: if not os.path.exists(build_py_path): return - # Check if BT/BLE dependencies exist in lib_deps - bt_ble_protected = self._has_bt_ble_dependencies() - try: with open(build_py_path, 'r') as f: content = f.read() @@ -364,9 +417,9 @@ def _remove_ignored_lib_includes(self) -> None: # Remove CPPPATH entries for each ignored library for lib_name in self.ignored_libs: - # Skip BT-related libraries if BT/BLE dependencies are present - if bt_ble_protected and self._is_bt_related_library(lib_name): - print(f"Skipping removal of BT-related library '{lib_name}' due to BT/BLE dependency in lib_deps") + # Universal protection: Skip if component is actually used in project + if self._is_component_used_in_project(lib_name): + print(f"Skipping removal of library '{lib_name}' - detected as used in project") continue # Multiple patterns to catch different include formats From ccdb040a1ec9dc56d2372229600d159de722369f Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Thu, 29 May 2025 17:10:39 +0200 Subject: [PATCH 22/97] debug code for not detecting component --- builder/frameworks/component_manager.py | 108 ++++++++++++++++++------ 1 file changed, 83 insertions(+), 25 deletions(-) diff --git a/builder/frameworks/component_manager.py b/builder/frameworks/component_manager.py index 07ae14050..3b8745f43 100644 --- a/builder/frameworks/component_manager.py +++ b/builder/frameworks/component_manager.py @@ -120,34 +120,54 @@ def _analyze_project_dependencies(self) -> Set[str]: """Analyze project files to detect actually used components/libraries.""" used_components = set() + print(f"DEBUG: Starting project analysis") + try: - # Analyze project source files src_dir = self.env.subst("$PROJECT_SRC_DIR") + print(f"DEBUG: Scanning source directory: {src_dir}") + if os.path.exists(src_dir): + file_count = 0 for root, dirs, files in os.walk(src_dir): for file in files: if file.endswith(('.cpp', '.c', '.h', '.hpp', '.ino')): + file_count += 1 file_path = os.path.join(root, file) - used_components.update(self._extract_components_from_file(file_path)) + file_components = self._extract_components_from_file(file_path) + if file_components: + print(f"DEBUG: File {file} detected components: {file_components}") + used_components.update(file_components) + + print(f"DEBUG: Scanned {file_count} source files") - # Analyze lib_deps for explicit dependencies (if available) + # Check lib_deps lib_deps = self.env.GetProjectOption("lib_deps", []) - if isinstance(lib_deps, str): - lib_deps = [lib_deps] - - for dep in lib_deps: - used_components.update(self._extract_components_from_lib_dep(str(dep))) + if lib_deps: + print(f"DEBUG: Found lib_deps: {lib_deps}") + if isinstance(lib_deps, str): + lib_deps = [lib_deps] - except Exception: - pass + for dep in lib_deps: + dep_components = self._extract_components_from_lib_dep(str(dep)) + if dep_components: + print(f"DEBUG: lib_dep '{dep}' mapped to components: {dep_components}") + used_components.update(dep_components) + else: + print(f"DEBUG: No lib_deps found") + + except Exception as e: + print(f"DEBUG: Exception in project analysis: {e}") + import traceback + traceback.print_exc() + print(f"DEBUG: Final detected components: {used_components}") return used_components def _extract_components_from_file(self, file_path: str) -> Set[str]: """Extract component usage from a single file by analyzing includes and function calls.""" components = set() - # Component detection patterns - maps component names to search patterns + # Component detection patterns - maps component names to code patterns component_patterns = { 'bt': ['bluetooth', 'ble', 'nimble', 'bt_', 'esp_bt', 'esp_ble'], 'esp_wifi': ['wifi', 'esp_wifi', 'tcpip_adapter'], @@ -163,14 +183,18 @@ def _extract_components_from_file(self, file_path: str) -> Set[str]: 'driver': ['gpio_', 'uart_', 'spi_', 'i2c_', 'adc_', 'dac_'], 'esp_camera': ['esp_camera', 'camera.h'], 'esp_now': ['esp_now', 'espnow'], - 'esp_smartconfig': ['smartconfig', 'esp_smartconfig'] + 'esp_smartconfig': ['smartconfig', 'esp_smartconfig'], + 'esp_eth': ['esp_eth', 'ethernet'], + 'esp_websocket_client': ['websocket', 'esp_websocket'], + 'cjson': ['cjson', 'json'], + 'mbedtls': ['mbedtls', 'ssl'], + 'openssl': ['openssl'] } try: with open(file_path, 'r', encoding='utf-8', errors='ignore') as f: content = f.read().lower() - # Check each component pattern against file content for component, patterns in component_patterns.items(): if any(pattern in content for pattern in patterns): components.add(component) @@ -181,7 +205,7 @@ def _extract_components_from_file(self, file_path: str) -> Set[str]: return components def _extract_components_from_lib_dep(self, lib_dep: str) -> Set[str]: - """Extract components from lib_deps entry by mapping library names to components.""" + """Extract components from lib_deps entry by mapping library names to ESP-IDF components.""" components = set() lib_dep_upper = lib_dep.upper() @@ -192,7 +216,10 @@ def _extract_components_from_lib_dep(self, lib_dep: str) -> Set[str]: 'esp_dsp': ['DSP', 'FFT', 'JPEG'], 'esp_http_client': ['HTTP', 'HTTPCLIENT'], 'mqtt': ['MQTT', 'PUBSUB'], - 'esp_camera': ['CAMERA', 'ESP32CAM'] + 'esp_camera': ['CAMERA', 'ESP32CAM'], + 'esp_now': ['ESPNOW', 'ESP_NOW'], + 'mdns': ['MDNS'], + 'esp_eth': ['ETHERNET'] } for component, keywords in lib_dep_mapping.items(): @@ -203,7 +230,7 @@ def _extract_components_from_lib_dep(self, lib_dep: str) -> Set[str]: def _is_component_used_in_project(self, lib_name: str) -> bool: """Check if a component/library is actually used in the project.""" - # Cache project analysis results for performance + # Cache project analysis for performance if not hasattr(self, '_project_components_cache'): self._project_components_cache = self._analyze_project_dependencies() @@ -406,8 +433,16 @@ def _remove_ignored_lib_includes(self) -> None: build_py_path = join(self.arduino_libs_mcu, "pioarduino-build.py") if not os.path.exists(build_py_path): + print(f"DEBUG: Build file not found: {build_py_path}") return + print(f"DEBUG: Starting lib_ignore processing") + print(f"DEBUG: ignored_libs = {list(self.ignored_libs)}") + + # Force project analysis and show results + self._project_components_cache = self._analyze_project_dependencies() + print(f"DEBUG: Detected project components: {list(self._project_components_cache)}") + try: with open(build_py_path, 'r') as f: content = f.read() @@ -415,14 +450,19 @@ def _remove_ignored_lib_includes(self) -> None: original_content = content total_removed = 0 - # Remove CPPPATH entries for each ignored library + # Check each library individually for lib_name in self.ignored_libs: - # Universal protection: Skip if component is actually used in project - if self._is_component_used_in_project(lib_name): - print(f"Skipping removal of library '{lib_name}' - detected as used in project") + print(f"DEBUG: Processing lib_name = '{lib_name}'") + + # Check if component is used + is_used = self._is_component_used_in_project(lib_name) + print(f"DEBUG: Is '{lib_name}' used in project? {is_used}") + + if is_used: + print(f"DEBUG: SKIPPING '{lib_name}' - detected as used") continue - - # Multiple patterns to catch different include formats + + # Check what would be removed patterns = [ rf'.*join\([^,]*,\s*"include",\s*"{re.escape(lib_name)}"[^)]*\),?\n', rf'.*"include/{re.escape(lib_name)}"[^,\n]*,?\n', @@ -434,11 +474,24 @@ def _remove_ignored_lib_includes(self) -> None: rf'\s*"[^"]*/{re.escape(lib_name)}/[^"]*",?\n' ] + matches_found = [] for pattern in patterns: matches = re.findall(pattern, content) if matches: + matches_found.extend(matches) + + if matches_found: + print(f"DEBUG: REMOVING '{lib_name}' - found {len(matches_found)} matches:") + for match in matches_found: + print(f"DEBUG: - {match.strip()}") + + for pattern in patterns: content = re.sub(pattern, '', content) - total_removed += len(matches) + total_removed += len(re.findall(pattern, original_content)) + else: + print(f"DEBUG: No matches found for '{lib_name}'") + + print(f"DEBUG: Total lines removed: {total_removed}") # Clean up empty lines and trailing commas content = re.sub(r'\n\s*\n', '\n', content) @@ -446,11 +499,16 @@ def _remove_ignored_lib_includes(self) -> None: # Validate and write changes if self._validate_changes(original_content, content) and content != original_content: + print(f"DEBUG: Content changed, writing new file") with open(build_py_path, 'w') as f: f.write(content) + else: + print(f"DEBUG: No changes made to build file") - except Exception: - pass + except Exception as e: + print(f"DEBUG: Exception occurred: {e}") + import traceback + traceback.print_exc() def _validate_changes(self, original_content: str, new_content: str) -> bool: """Validate that the changes are reasonable.""" From 0a8330a5a5c94984d6f69d63923e3d7648f18c56 Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Thu, 29 May 2025 17:22:11 +0200 Subject: [PATCH 23/97] fix dsp detection --- builder/frameworks/component_manager.py | 91 ++++++------------------- 1 file changed, 20 insertions(+), 71 deletions(-) diff --git a/builder/frameworks/component_manager.py b/builder/frameworks/component_manager.py index 3b8745f43..a8187cb7c 100644 --- a/builder/frameworks/component_manager.py +++ b/builder/frameworks/component_manager.py @@ -120,47 +120,27 @@ def _analyze_project_dependencies(self) -> Set[str]: """Analyze project files to detect actually used components/libraries.""" used_components = set() - print(f"DEBUG: Starting project analysis") - try: + # Analyze project source files src_dir = self.env.subst("$PROJECT_SRC_DIR") - print(f"DEBUG: Scanning source directory: {src_dir}") - if os.path.exists(src_dir): - file_count = 0 for root, dirs, files in os.walk(src_dir): for file in files: if file.endswith(('.cpp', '.c', '.h', '.hpp', '.ino')): - file_count += 1 file_path = os.path.join(root, file) - file_components = self._extract_components_from_file(file_path) - if file_components: - print(f"DEBUG: File {file} detected components: {file_components}") - used_components.update(file_components) - - print(f"DEBUG: Scanned {file_count} source files") + used_components.update(self._extract_components_from_file(file_path)) - # Check lib_deps + # Analyze lib_deps for explicit dependencies (if present) lib_deps = self.env.GetProjectOption("lib_deps", []) - if lib_deps: - print(f"DEBUG: Found lib_deps: {lib_deps}") - if isinstance(lib_deps, str): - lib_deps = [lib_deps] - - for dep in lib_deps: - dep_components = self._extract_components_from_lib_dep(str(dep)) - if dep_components: - print(f"DEBUG: lib_dep '{dep}' mapped to components: {dep_components}") - used_components.update(dep_components) - else: - print(f"DEBUG: No lib_deps found") + if isinstance(lib_deps, str): + lib_deps = [lib_deps] + + for dep in lib_deps: + used_components.update(self._extract_components_from_lib_dep(str(dep))) - except Exception as e: - print(f"DEBUG: Exception in project analysis: {e}") - import traceback - traceback.print_exc() + except Exception: + pass - print(f"DEBUG: Final detected components: {used_components}") return used_components def _extract_components_from_file(self, file_path: str) -> Set[str]: @@ -171,7 +151,7 @@ def _extract_components_from_file(self, file_path: str) -> Set[str]: component_patterns = { 'bt': ['bluetooth', 'ble', 'nimble', 'bt_', 'esp_bt', 'esp_ble'], 'esp_wifi': ['wifi', 'esp_wifi', 'tcpip_adapter'], - 'esp_dsp': ['dsps_', 'esp_dsp', 'fft2r'], + 'esp_dsp': ['dsps_', 'esp_dsp', 'fft2r', 'dsps_fft2r'], # Enhanced DSP detection 'esp_http_client': ['esp_http_client', 'http_client'], 'esp_https_ota': ['esp_https_ota', 'esp_ota'], 'mdns': ['mdns', 'esp_mdns'], @@ -433,16 +413,8 @@ def _remove_ignored_lib_includes(self) -> None: build_py_path = join(self.arduino_libs_mcu, "pioarduino-build.py") if not os.path.exists(build_py_path): - print(f"DEBUG: Build file not found: {build_py_path}") return - print(f"DEBUG: Starting lib_ignore processing") - print(f"DEBUG: ignored_libs = {list(self.ignored_libs)}") - - # Force project analysis and show results - self._project_components_cache = self._analyze_project_dependencies() - print(f"DEBUG: Detected project components: {list(self._project_components_cache)}") - try: with open(build_py_path, 'r') as f: content = f.read() @@ -450,19 +422,14 @@ def _remove_ignored_lib_includes(self) -> None: original_content = content total_removed = 0 - # Check each library individually + # Remove CPPPATH entries for each ignored library for lib_name in self.ignored_libs: - print(f"DEBUG: Processing lib_name = '{lib_name}'") - - # Check if component is used - is_used = self._is_component_used_in_project(lib_name) - print(f"DEBUG: Is '{lib_name}' used in project? {is_used}") - - if is_used: - print(f"DEBUG: SKIPPING '{lib_name}' - detected as used") + # Universal protection: Skip if component is actually used in project + if self._is_component_used_in_project(lib_name): + print(f"Skipping removal of library '{lib_name}' - detected as used in project") continue - - # Check what would be removed + + # Multiple patterns to catch different include formats patterns = [ rf'.*join\([^,]*,\s*"include",\s*"{re.escape(lib_name)}"[^)]*\),?\n', rf'.*"include/{re.escape(lib_name)}"[^,\n]*,?\n', @@ -474,24 +441,11 @@ def _remove_ignored_lib_includes(self) -> None: rf'\s*"[^"]*/{re.escape(lib_name)}/[^"]*",?\n' ] - matches_found = [] for pattern in patterns: matches = re.findall(pattern, content) if matches: - matches_found.extend(matches) - - if matches_found: - print(f"DEBUG: REMOVING '{lib_name}' - found {len(matches_found)} matches:") - for match in matches_found: - print(f"DEBUG: - {match.strip()}") - - for pattern in patterns: content = re.sub(pattern, '', content) - total_removed += len(re.findall(pattern, original_content)) - else: - print(f"DEBUG: No matches found for '{lib_name}'") - - print(f"DEBUG: Total lines removed: {total_removed}") + total_removed += len(matches) # Clean up empty lines and trailing commas content = re.sub(r'\n\s*\n', '\n', content) @@ -499,16 +453,11 @@ def _remove_ignored_lib_includes(self) -> None: # Validate and write changes if self._validate_changes(original_content, content) and content != original_content: - print(f"DEBUG: Content changed, writing new file") with open(build_py_path, 'w') as f: f.write(content) - else: - print(f"DEBUG: No changes made to build file") - except Exception as e: - print(f"DEBUG: Exception occurred: {e}") - import traceback - traceback.print_exc() + except Exception: + pass def _validate_changes(self, original_content: str, new_content: str) -> bool: """Validate that the changes are reasonable.""" From c5c81355b718f974024d4a10befda0d70fb60d64 Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Thu, 29 May 2025 17:34:14 +0200 Subject: [PATCH 24/97] Update tasmota_platformio_override.ini --- examples/tasmota_platformio_override.ini | 1 - 1 file changed, 1 deletion(-) diff --git a/examples/tasmota_platformio_override.ini b/examples/tasmota_platformio_override.ini index 8b477e297..4ba00f20b 100644 --- a/examples/tasmota_platformio_override.ini +++ b/examples/tasmota_platformio_override.ini @@ -38,7 +38,6 @@ custom_sdkconfig = https://raw.githubusercontent.com/pioarduino/sdkconfig '# CONFIG_ETH_RMII_CLK_INPUT is not set' '# CONFIG_ETH_RMII_CLK_IN_GPIO is not set' custom_component_remove = - espressif/esp-dsp espressif/network_provisioning espressif/esp-zboss-lib espressif/esp-zigbee-lib From 1076c0d4b4d8ed505a6ae11427b5f0089562583d Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Thu, 29 May 2025 18:16:26 +0200 Subject: [PATCH 25/97] remove sophisticated analysis --- builder/frameworks/component_manager.py | 151 ++++++++---------------- 1 file changed, 48 insertions(+), 103 deletions(-) diff --git a/builder/frameworks/component_manager.py b/builder/frameworks/component_manager.py index a8187cb7c..66b006b5b 100644 --- a/builder/frameworks/component_manager.py +++ b/builder/frameworks/component_manager.py @@ -116,113 +116,50 @@ def _get_lib_ignore_entries(self) -> List[str]: except Exception: return [] - def _analyze_project_dependencies(self) -> Set[str]: - """Analyze project files to detect actually used components/libraries.""" - used_components = set() - + def _has_bt_ble_dependencies(self) -> bool: + """Check if lib_deps contains any BT/BLE related dependencies.""" try: - # Analyze project source files - src_dir = self.env.subst("$PROJECT_SRC_DIR") - if os.path.exists(src_dir): - for root, dirs, files in os.walk(src_dir): - for file in files: - if file.endswith(('.cpp', '.c', '.h', '.hpp', '.ino')): - file_path = os.path.join(root, file) - used_components.update(self._extract_components_from_file(file_path)) - - # Analyze lib_deps for explicit dependencies (if present) + # Get lib_deps from current environment lib_deps = self.env.GetProjectOption("lib_deps", []) + if isinstance(lib_deps, str): lib_deps = [lib_deps] + elif lib_deps is None: + lib_deps = [] + + # Convert to string and check for BT/BLE keywords + lib_deps_str = ' '.join(str(dep) for dep in lib_deps).upper() + + bt_ble_keywords = ['BLE', 'BT', 'NIMBLE', 'BLUETOOTH'] + + for keyword in bt_ble_keywords: + if keyword in lib_deps_str: + return True + + return False - for dep in lib_deps: - used_components.update(self._extract_components_from_lib_dep(str(dep))) - - except Exception: - pass - - return used_components - - def _extract_components_from_file(self, file_path: str) -> Set[str]: - """Extract component usage from a single file by analyzing includes and function calls.""" - components = set() - - # Component detection patterns - maps component names to code patterns - component_patterns = { - 'bt': ['bluetooth', 'ble', 'nimble', 'bt_', 'esp_bt', 'esp_ble'], - 'esp_wifi': ['wifi', 'esp_wifi', 'tcpip_adapter'], - 'esp_dsp': ['dsps_', 'esp_dsp', 'fft2r', 'dsps_fft2r'], # Enhanced DSP detection - 'esp_http_client': ['esp_http_client', 'http_client'], - 'esp_https_ota': ['esp_https_ota', 'esp_ota'], - 'mdns': ['mdns', 'esp_mdns'], - 'mqtt': ['mqtt', 'esp_mqtt'], - 'spiffs': ['spiffs', 'esp_spiffs'], - 'fatfs': ['fatfs', 'ff.h'], - 'nvs_flash': ['nvs', 'nvs_flash'], - 'esp_timer': ['esp_timer', 'timer_'], - 'driver': ['gpio_', 'uart_', 'spi_', 'i2c_', 'adc_', 'dac_'], - 'esp_camera': ['esp_camera', 'camera.h'], - 'esp_now': ['esp_now', 'espnow'], - 'esp_smartconfig': ['smartconfig', 'esp_smartconfig'], - 'esp_eth': ['esp_eth', 'ethernet'], - 'esp_websocket_client': ['websocket', 'esp_websocket'], - 'cjson': ['cjson', 'json'], - 'mbedtls': ['mbedtls', 'ssl'], - 'openssl': ['openssl'] - } - - try: - with open(file_path, 'r', encoding='utf-8', errors='ignore') as f: - content = f.read().lower() - - for component, patterns in component_patterns.items(): - if any(pattern in content for pattern in patterns): - components.add(component) - except Exception: - pass - - return components - - def _extract_components_from_lib_dep(self, lib_dep: str) -> Set[str]: - """Extract components from lib_deps entry by mapping library names to ESP-IDF components.""" - components = set() - lib_dep_upper = lib_dep.upper() - - # Map lib_deps entries to ESP-IDF components - lib_dep_mapping = { - 'bt': ['BLE', 'BT', 'BLUETOOTH', 'NIMBLE'], - 'esp_wifi': ['WIFI', 'ASYNCTCP', 'ESPASYNCWEBSERVER'], - 'esp_dsp': ['DSP', 'FFT', 'JPEG'], - 'esp_http_client': ['HTTP', 'HTTPCLIENT'], - 'mqtt': ['MQTT', 'PUBSUB'], - 'esp_camera': ['CAMERA', 'ESP32CAM'], - 'esp_now': ['ESPNOW', 'ESP_NOW'], - 'mdns': ['MDNS'], - 'esp_eth': ['ETHERNET'] - } - - for component, keywords in lib_dep_mapping.items(): - if any(keyword in lib_dep_upper for keyword in keywords): - components.add(component) - - return components + return False - def _is_component_used_in_project(self, lib_name: str) -> bool: - """Check if a component/library is actually used in the project.""" - # Cache project analysis for performance - if not hasattr(self, '_project_components_cache'): - self._project_components_cache = self._analyze_project_dependencies() - - lib_name_lower = lib_name.lower() - - # Direct match - if lib_name_lower in self._project_components_cache: - return True - - # Partial match for related components - for used_component in self._project_components_cache: - if lib_name_lower in used_component or used_component in lib_name_lower: + def _is_bt_related_library(self, lib_name: str) -> bool: + """Check if a library name is related to Bluetooth/BLE functionality.""" + lib_name_upper = lib_name.upper() + + bt_related_names = [ + 'BT', + 'BLE', + 'BLUETOOTH', + 'NIMBLE', + 'ESP32_BLE', + 'ESP32BLE', + 'BLUETOOTHSERIAL', + 'BLE_ARDUINO', + 'ESP_BLE', + 'ESP_BT' + ] + + for bt_name in bt_related_names: + if bt_name in lib_name_upper: return True return False @@ -415,6 +352,9 @@ def _remove_ignored_lib_includes(self) -> None: if not os.path.exists(build_py_path): return + # Check if BT/BLE dependencies exist in lib_deps + bt_ble_protected = self._has_bt_ble_dependencies() + try: with open(build_py_path, 'r') as f: content = f.read() @@ -424,9 +364,14 @@ def _remove_ignored_lib_includes(self) -> None: # Remove CPPPATH entries for each ignored library for lib_name in self.ignored_libs: - # Universal protection: Skip if component is actually used in project - if self._is_component_used_in_project(lib_name): - print(f"Skipping removal of library '{lib_name}' - detected as used in project") + # Skip BT-related libraries if BT/BLE dependencies are present + if bt_ble_protected and self._is_bt_related_library(lib_name): + print(f"Skipping removal of BT-related library '{lib_name}' due to BT/BLE dependency in lib_deps") + continue + + # Hard protection for DSP components + if lib_name.lower() in ['dsp', 'esp_dsp', 'dsps', 'fft2r', 'dsps_fft2r']: + print(f"Hard-protected DSP component '{lib_name}' from removal") continue # Multiple patterns to catch different include formats From 433bb537d7337e1764baf9a8e92c6565c86ec890 Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Thu, 29 May 2025 18:39:48 +0200 Subject: [PATCH 26/97] Update platformio.ini --- examples/arduino-blink/platformio.ini | 28 +++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/examples/arduino-blink/platformio.ini b/examples/arduino-blink/platformio.ini index 5a1d46f12..cc820b5a6 100644 --- a/examples/arduino-blink/platformio.ini +++ b/examples/arduino-blink/platformio.ini @@ -12,6 +12,11 @@ platform = espressif32 framework = arduino board = esp32-solo1 build_flags = -DLED_BUILTIN=2 +lib_ignore = wifi + bt + spiffs + NetworkClientSecure + custom_component_remove = espressif/esp_hosted espressif/esp_wifi_remote @@ -31,6 +36,10 @@ platform = espressif32 framework = arduino board = esp32-c2-devkitm-1 monitor_speed = 115200 +lib_ignore = wifi + bt + spiffs + NetworkClientSecure custom_component_remove = espressif/esp_hosted espressif/esp_wifi_remote espressif/esp-dsp @@ -50,6 +59,10 @@ platform = espressif32 framework = arduino board = arduino_nano_esp32 monitor_speed = 115200 +lib_ignore = wifi + bt + spiffs + NetworkClientSecure custom_component_remove = espressif/esp_hosted espressif/esp_wifi_remote espressif/esp-dsp @@ -68,6 +81,9 @@ custom_component_remove = espressif/esp_hosted platform = espressif32 framework = arduino board = esp32s3_120_16_8-qio_opi +lib_ignore = bt + spiffs + NetworkClientSecure custom_sdkconfig = CONFIG_SPIRAM_MODE_OCT=y CONFIG_SPIRAM_SPEED_120M=y CONFIG_LCD_RGB_ISR_IRAM_SAFE=y @@ -95,6 +111,10 @@ framework = arduino build_type = debug board = esp32-c6-devkitc-1 monitor_speed = 115200 +lib_ignore = wifi + bt + spiffs + NetworkClientSecure custom_component_remove = espressif/esp_hosted espressif/esp_wifi_remote espressif/mdns @@ -107,6 +127,10 @@ platform = espressif32 framework = arduino board = esp32-h2-devkitm-1 monitor_speed = 115200 +lib_ignore = + bt + spiffs + NetworkClientSecure custom_component_remove = espressif/esp_hosted espressif/esp_wifi_remote espressif/mdns @@ -119,6 +143,10 @@ platform = espressif32 framework = arduino board = esp32-p4 build_flags = -DLED_BUILTIN=2 +lib_ignore = wifi + bt + spiffs + NetworkClientSecure monitor_speed = 115200 custom_component_remove = espressif/esp_hosted espressif/esp_wifi_remote From 5db706c6e311cf1d6874c139353eac80c45491ca Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Thu, 29 May 2025 18:40:51 +0200 Subject: [PATCH 27/97] Update platformio.ini --- examples/arduino-rmt-blink/platformio.ini | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/examples/arduino-rmt-blink/platformio.ini b/examples/arduino-rmt-blink/platformio.ini index e3c6beacd..cf6d562ff 100644 --- a/examples/arduino-rmt-blink/platformio.ini +++ b/examples/arduino-rmt-blink/platformio.ini @@ -2,6 +2,10 @@ platform = espressif32 framework = arduino board = esp32-s2-saola-1 +lib_ignore = wifi + bt + spiffs + NetworkClientSecure build_flags = -DBUILTIN_RGBLED_PIN=18 -DNR_OF_LEDS=1 @@ -9,6 +13,10 @@ build_flags = -DBUILTIN_RGBLED_PIN=18 platform = espressif32 framework = arduino board = esp32-s3-devkitc-1 +lib_ignore = wifi + bt + spiffs + NetworkClientSecure build_flags = -DBUILTIN_RGBLED_PIN=48 -DNR_OF_LEDS=1 @@ -16,6 +24,10 @@ build_flags = -DBUILTIN_RGBLED_PIN=48 platform = espressif32 framework = arduino board = esp32-c3-devkitm-1 +lib_ignore = wifi + bt + spiffs + NetworkClientSecure build_flags = -DBUILTIN_RGBLED_PIN=8 -DNR_OF_LEDS=1 @@ -23,5 +35,9 @@ build_flags = -DBUILTIN_RGBLED_PIN=8 platform = espressif32 framework = arduino board = esp32-c6-devkitm-1 +lib_ignore = wifi + bt + spiffs + NetworkClientSecure build_flags = -DBUILTIN_RGBLED_PIN=8 -DNR_OF_LEDS=1 From 10d7da238b8d78d1acfd270ecdc9838f0c103def Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Thu, 29 May 2025 18:57:14 +0200 Subject: [PATCH 28/97] Update platformio.ini --- examples/arduino-blink/platformio.ini | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/examples/arduino-blink/platformio.ini b/examples/arduino-blink/platformio.ini index cc820b5a6..8534e061e 100644 --- a/examples/arduino-blink/platformio.ini +++ b/examples/arduino-blink/platformio.ini @@ -13,7 +13,6 @@ framework = arduino board = esp32-solo1 build_flags = -DLED_BUILTIN=2 lib_ignore = wifi - bt spiffs NetworkClientSecure @@ -37,7 +36,6 @@ framework = arduino board = esp32-c2-devkitm-1 monitor_speed = 115200 lib_ignore = wifi - bt spiffs NetworkClientSecure custom_component_remove = espressif/esp_hosted @@ -60,7 +58,6 @@ framework = arduino board = arduino_nano_esp32 monitor_speed = 115200 lib_ignore = wifi - bt spiffs NetworkClientSecure custom_component_remove = espressif/esp_hosted @@ -81,7 +78,7 @@ custom_component_remove = espressif/esp_hosted platform = espressif32 framework = arduino board = esp32s3_120_16_8-qio_opi -lib_ignore = bt +lib_ignore = spiffs NetworkClientSecure custom_sdkconfig = CONFIG_SPIRAM_MODE_OCT=y @@ -112,7 +109,6 @@ build_type = debug board = esp32-c6-devkitc-1 monitor_speed = 115200 lib_ignore = wifi - bt spiffs NetworkClientSecure custom_component_remove = espressif/esp_hosted @@ -128,7 +124,6 @@ framework = arduino board = esp32-h2-devkitm-1 monitor_speed = 115200 lib_ignore = - bt spiffs NetworkClientSecure custom_component_remove = espressif/esp_hosted @@ -144,7 +139,6 @@ framework = arduino board = esp32-p4 build_flags = -DLED_BUILTIN=2 lib_ignore = wifi - bt spiffs NetworkClientSecure monitor_speed = 115200 From 0010a7c9f047f89f728903d73e935ca7da7835ad Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Thu, 29 May 2025 18:58:12 +0200 Subject: [PATCH 29/97] Update platformio.ini --- examples/arduino-rmt-blink/platformio.ini | 4 ---- 1 file changed, 4 deletions(-) diff --git a/examples/arduino-rmt-blink/platformio.ini b/examples/arduino-rmt-blink/platformio.ini index cf6d562ff..c40a21bd1 100644 --- a/examples/arduino-rmt-blink/platformio.ini +++ b/examples/arduino-rmt-blink/platformio.ini @@ -3,7 +3,6 @@ platform = espressif32 framework = arduino board = esp32-s2-saola-1 lib_ignore = wifi - bt spiffs NetworkClientSecure build_flags = -DBUILTIN_RGBLED_PIN=18 @@ -14,7 +13,6 @@ platform = espressif32 framework = arduino board = esp32-s3-devkitc-1 lib_ignore = wifi - bt spiffs NetworkClientSecure build_flags = -DBUILTIN_RGBLED_PIN=48 @@ -25,7 +23,6 @@ platform = espressif32 framework = arduino board = esp32-c3-devkitm-1 lib_ignore = wifi - bt spiffs NetworkClientSecure build_flags = -DBUILTIN_RGBLED_PIN=8 @@ -36,7 +33,6 @@ platform = espressif32 framework = arduino board = esp32-c6-devkitm-1 lib_ignore = wifi - bt spiffs NetworkClientSecure build_flags = -DBUILTIN_RGBLED_PIN=8 From 43122e424b7cb72bf604f28628cee9d23d85cc3a Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Thu, 29 May 2025 20:23:06 +0200 Subject: [PATCH 30/97] refactor platform. --- platform.py | 701 ++++++++++++++++++++++++++++++++++------------------ 1 file changed, 467 insertions(+), 234 deletions(-) diff --git a/platform.py b/platform.py index 9545bbd8c..359ae9cb4 100644 --- a/platform.py +++ b/platform.py @@ -14,19 +14,57 @@ import os import contextlib -import requests import json import subprocess import sys import shutil -from os.path import join +import logging +from typing import Optional, Dict, List, Any from platformio.public import PlatformBase, to_unix_path from platformio.proc import get_pythonexe_path from platformio.project.config import ProjectConfig from platformio.package.manager.tool import ToolPackageManager - +# Constants +RETRY_LIMIT = 3 +SUBPROCESS_TIMEOUT = 300 +MKLITTLEFS_VERSION_320 = "3.2.0" +MKLITTLEFS_VERSION_400 = "4.0.0" +DEFAULT_DEBUG_SPEED = "5000" +DEFAULT_APP_OFFSET = "0x10000" + +# MCUs that support ESP-builtin debug +ESP_BUILTIN_DEBUG_MCUS = frozenset(["esp32c3", "esp32c5", "esp32c6", "esp32s3", "esp32h2", "esp32p4"]) + +# MCU configuration +MCU_TOOLCHAIN_CONFIG = { + "xtensa": { + "mcus": frozenset(["esp32", "esp32s2", "esp32s3"]), + "toolchains": ["toolchain-xtensa-esp-elf"], + "debug_tools": ["tool-xtensa-esp-elf-gdb"] + }, + "riscv": { + "mcus": frozenset(["esp32c2", "esp32c3", "esp32c5", "esp32c6", "esp32h2", "esp32p4"]), + "toolchains": ["toolchain-riscv32-esp"], + "debug_tools": ["tool-riscv32-esp-elf-gdb"] + } +} + +COMMON_IDF_PACKAGES = [ + "tool-cmake", + "tool-ninja", + "tool-scons", + "tool-esp-rom-elfs" +] + +CHECK_PACKAGES = [ + "tool-cppcheck", + "tool-clangtidy", + "tool-pvs-studio" +] + +# System-specific configuration IS_WINDOWS = sys.platform.startswith("win") # Set Platformio env var to use windows_amd64 for all windows architectures # only windows_amd64 native espressif toolchains are available @@ -34,219 +72,406 @@ if IS_WINDOWS: os.environ["PLATFORMIO_SYSTEM_TYPE"] = "windows_amd64" +# Global variables python_exe = get_pythonexe_path() pm = ToolPackageManager() +# Configure logger +logger = logging.getLogger(__name__) + + +class ToolInstallationError(Exception): + """Custom exception for tool installation errors""" + pass + + +def safe_file_operation(operation_func): + """Decorator for safe filesystem operations""" + def wrapper(*args, **kwargs): + try: + return operation_func(*args, **kwargs) + except (OSError, IOError, FileNotFoundError) as e: + logger.error(f"Filesystem error in {operation_func.__name__}: {e}") + return False + except Exception as e: + logger.error(f"Unexpected error in {operation_func.__name__}: {e}") + raise # Re-raise unexpected exceptions + return wrapper + + +@safe_file_operation +def safe_remove_directory(path: str) -> bool: + """Safely remove directories""" + if os.path.exists(path) and os.path.isdir(path): + shutil.rmtree(path) + logger.debug(f"Directory removed: {path}") + return True + + +@safe_file_operation +def safe_copy_file(src: str, dst: str) -> bool: + """Safely copy files""" + os.makedirs(os.path.dirname(dst), exist_ok=True) + shutil.copyfile(src, dst) + logger.debug(f"File copied: {src} -> {dst}") + return True + + class Espressif32Platform(PlatformBase): - def configure_default_packages(self, variables, targets): - if not variables.get("board"): - return super().configure_default_packages(variables, targets) + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._packages_dir = None + self._tools_cache = {} + self._mcu_config_cache = {} + + @property + def packages_dir(self) -> str: + """Cached packages directory""" + if self._packages_dir is None: + self._packages_dir = ProjectConfig.get_instance().get("platformio", "packages_dir") + return self._packages_dir + + def _get_tool_paths(self, tool_name: str) -> Dict[str, str]: + """Central path calculation for tools""" + if tool_name not in self._tools_cache: + tool_path = os.path.join(self.packages_dir, tool_name) + self._tools_cache[tool_name] = { + 'tool_path': tool_path, + 'package_path': os.path.join(tool_path, "package.json"), + 'tools_json_path': os.path.join(tool_path, "tools.json"), + 'piopm_path': os.path.join(tool_path, ".piopm"), + 'idf_tools_path': os.path.join(self.packages_dir, "tl-install", "tools", "idf_tools.py") + } + return self._tools_cache[tool_name] + + def _check_tool_status(self, tool_name: str) -> Dict[str, bool]: + """Check tool status""" + paths = self._get_tool_paths(tool_name) + return { + 'has_idf_tools': os.path.exists(paths['idf_tools_path']), + 'has_tools_json': os.path.exists(paths['tools_json_path']), + 'has_piopm': os.path.exists(paths['piopm_path']), + 'tool_exists': os.path.exists(paths['tool_path']) + } - board_config = self.board_config(variables.get("board")) - mcu = variables.get("board_build.mcu", board_config.get("build.mcu", "esp32")) - board_sdkconfig = variables.get("board_espidf.custom_sdkconfig", board_config.get("espidf.custom_sdkconfig", "")) - frameworks = variables.get("pioframework", []) - - def install_tool(TOOL, retry_count=0): - self.packages[TOOL]["optional"] = False - TOOL_PATH = os.path.join(ProjectConfig.get_instance().get("platformio", "packages_dir"), TOOL) - TOOL_PACKAGE_PATH = os.path.join(TOOL_PATH, "package.json") - TOOLS_PATH_DEFAULT = os.path.join(os.path.expanduser("~"), ".platformio") - IDF_TOOLS = os.path.join(ProjectConfig.get_instance().get("platformio", "packages_dir"), "tl-install", "tools", "idf_tools.py") - TOOLS_JSON_PATH = os.path.join(TOOL_PATH, "tools.json") - TOOLS_PIO_PATH = os.path.join(TOOL_PATH, ".piopm") - IDF_TOOLS_CMD = ( - python_exe, - IDF_TOOLS, - "--quiet", - "--non-interactive", - "--tools-json", - TOOLS_JSON_PATH, - "install" + def _run_idf_tools_install(self, tools_json_path: str, idf_tools_path: str) -> bool: + """Execute idf_tools.py install""" + cmd = [ + python_exe, + idf_tools_path, + "--quiet", + "--non-interactive", + "--tools-json", + tools_json_path, + "install" + ] + + try: + result = subprocess.run( + cmd, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + timeout=SUBPROCESS_TIMEOUT ) - tl_flag = bool(os.path.exists(IDF_TOOLS)) - json_flag = bool(os.path.exists(TOOLS_JSON_PATH)) - pio_flag = bool(os.path.exists(TOOLS_PIO_PATH)) - if tl_flag and json_flag: - with open(os.devnull, 'w') as devnull, \ - contextlib.redirect_stdout(devnull), \ - contextlib.redirect_stderr(devnull): - rc = subprocess.run( - IDF_TOOLS_CMD, - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL - ).returncode - if rc != 0: - sys.stderr.write("Error: Couldn't execute 'idf_tools.py install'\n") - else: - tl_path = "file://" + join(TOOLS_PATH_DEFAULT, "tools", TOOL) - try: - shutil.copyfile(TOOL_PACKAGE_PATH, join(TOOLS_PATH_DEFAULT, "tools", TOOL, "package.json")) - except FileNotFoundError as e: - sys.stderr.write(f"Error copying tool package file: {e}\n") - if os.path.exists(TOOL_PATH) and os.path.isdir(TOOL_PATH): - try: - shutil.rmtree(TOOL_PATH) - except Exception as e: - print(f"Error while removing the tool folder: {e}") - pm.install(tl_path) - # tool is already installed, just activate it - if tl_flag and pio_flag and not json_flag: - with open(TOOL_PACKAGE_PATH, "r") as file: - package_data = json.load(file) - # check installed tool version against listed in platforms.json - if "package-version" in self.packages[TOOL] \ - and "version" in package_data \ - and self.packages[TOOL]["package-version"] == package_data["version"]: - self.packages[TOOL]["version"] = TOOL_PATH - self.packages[TOOL]["optional"] = False - elif "package-version" not in self.packages[TOOL]: - # No version check needed, just use the installed tool - self.packages[TOOL]["version"] = TOOL_PATH - self.packages[TOOL]["optional"] = False - elif "version" not in package_data: - print(f"Warning: Cannot determine installed version for {TOOL}. Reinstalling...") - else: # Installed version does not match required version, deinstall existing and install needed - if os.path.exists(TOOL_PATH) and os.path.isdir(TOOL_PATH): - try: - shutil.rmtree(TOOL_PATH) - except Exception as e: - print(f"Error while removing the tool folder: {e}") - if retry_count >= 3: # Limit to 3 retries - print(f"Failed to install {TOOL} after multiple attempts. Please check your network connection and try again manually.") - return - print(f"Wrong version for {TOOL}. Installing needed version...") - install_tool(TOOL, retry_count + 1) + if result.returncode != 0: + logger.error("idf_tools.py installation failed") + return False - return + logger.debug("idf_tools.py executed successfully") + return True - # Installer only needed for setup, deactivate when installed - if bool(os.path.exists(os.path.join(ProjectConfig.get_instance().get("platformio", "packages_dir"), "tl-install", "tools", "idf_tools.py"))): - self.packages["tl-install"]["optional"] = True + except subprocess.TimeoutExpired: + logger.error(f"Timeout in idf_tools.py after {SUBPROCESS_TIMEOUT}s") + return False + except Exception as e: + logger.error(f"Error in idf_tools.py: {e}") + return False + + def _check_tool_version(self, tool_name: str) -> bool: + """Check tool version""" + paths = self._get_tool_paths(tool_name) - if "arduino" in frameworks: - self.packages["framework-arduinoespressif32"]["optional"] = False - self.packages["framework-arduinoespressif32-libs"]["optional"] = False - # use branch master - URL = "https://raw.githubusercontent.com/espressif/arduino-esp32/master/package/package_esp32_index.template.json" - packjdata = requests.get(URL).json() - dyn_lib_url = packjdata['packages'][0]['tools'][0]['systems'][0]['url'] - self.packages["framework-arduinoespressif32-libs"]["version"] = dyn_lib_url + try: + with open(paths['package_path'], 'r') as f: + package_data = json.load(f) - if variables.get("custom_sdkconfig") is not None or len(str(board_sdkconfig)) > 3: + required_version = self.packages.get(tool_name, {}).get("package-version") + installed_version = package_data.get("version") + + if not required_version: + logger.debug(f"No version check required for {tool_name}") + return True + + if not installed_version: + logger.warning(f"Installed version for {tool_name} unknown") + return False + + version_match = required_version == installed_version + if not version_match: + logger.info(f"Version mismatch for {tool_name}: {installed_version} != {required_version}") + + return version_match + + except (json.JSONDecodeError, FileNotFoundError) as e: + logger.error(f"Error reading package data for {tool_name}: {e}") + return False + + def install_tool(self, tool_name: str, retry_count: int = 0) -> bool: + """Optimized tool installation""" + if retry_count >= RETRY_LIMIT: + logger.error(f"Installation of {tool_name} failed after {RETRY_LIMIT} attempts") + return False + + self.packages[tool_name]["optional"] = False + paths = self._get_tool_paths(tool_name) + status = self._check_tool_status(tool_name) + + # Case 1: New installation with idf_tools + if status['has_idf_tools'] and status['has_tools_json']: + return self._install_with_idf_tools(tool_name, paths) + + # Case 2: Tool already installed, version check + if status['has_idf_tools'] and status['has_piopm'] and not status['has_tools_json']: + return self._handle_existing_tool(tool_name, paths, retry_count) + + logger.debug(f"Tool {tool_name} already configured") + return True + + def _install_with_idf_tools(self, tool_name: str, paths: Dict[str, str]) -> bool: + """Installation with idf_tools.py""" + if not self._run_idf_tools_install(paths['tools_json_path'], paths['idf_tools_path']): + return False + + # Copy tool files + tools_path_default = os.path.join(os.path.expanduser("~"), ".platformio") + target_package_path = os.path.join(tools_path_default, "tools", tool_name, "package.json") + + if not safe_copy_file(paths['package_path'], target_package_path): + return False + + safe_remove_directory(paths['tool_path']) + + tl_path = f"file://{os.path.join(tools_path_default, 'tools', tool_name)}" + pm.install(tl_path) + + logger.info(f"Tool {tool_name} successfully installed") + return True + + def _handle_existing_tool(self, tool_name: str, paths: Dict[str, str], retry_count: int) -> bool: + """Handle already installed tools""" + if self._check_tool_version(tool_name): + # Version matches, use tool + self.packages[tool_name]["version"] = paths['tool_path'] + self.packages[tool_name]["optional"] = False + logger.debug(f"Tool {tool_name} found with correct version") + return True + else: + # Wrong version, reinstall + logger.info(f"Reinstalling {tool_name} due to version mismatch") + safe_remove_directory(paths['tool_path']) + return self.install_tool(tool_name, retry_count + 1) + + def _configure_arduino_framework(self, frameworks: List[str]) -> None: + """Configure Arduino framework""" + if "arduino" not in frameworks: + return + + self.packages["framework-arduinoespressif32"]["optional"] = False + self.packages["framework-arduinoespressif32-libs"]["optional"] = False + # use branch master + board_sdkconfig = variables.get("board_espidf.custom_sdkconfig", + URL = "https://raw.githubusercontent.com/espressif/arduino-esp32/master/package/package_esp32_index.template.json" + packjdata = requests.get(URL).json() + dyn_lib_url = packjdata['packages'][0]['tools'][0]['systems'][0]['url'] + self.packages["framework-arduinoespressif32-libs"]["version"] = dyn_lib_url + + def _configure_espidf_framework(self, frameworks: List[str], variables: Dict, board_config: Dict, mcu: str) -> None: + """Configure ESP-IDF framework""" + custom_sdkconfig = variables.get("custom_sdkconfig") + board_sdkconfig = variables.get("board_espidf.custom_sdkconfig", + board_config.get("espidf.custom_sdkconfig", "")) + + if custom_sdkconfig is not None or len(str(board_sdkconfig)) > 3: frameworks.append("espidf") self.packages["framework-espidf"]["optional"] = False if mcu == "esp32c2": self.packages["framework-arduino-c2-skeleton-lib"]["optional"] = False - MCU_TOOLCHAIN_MAPPING = { - # Xtensa based and FSM toolchain - ("esp32", "esp32s2", "esp32s3"): { - "toolchains": ["toolchain-xtensa-esp-elf"], - "ulp_toolchain": ["toolchain-esp32ulp"] + (["toolchain-riscv32-esp"] if mcu != "esp32" else []), - "debug_tools": ["tool-xtensa-esp-elf-gdb"] - }, - # RISC-V based toolchain - ("esp32c2", "esp32c3", "esp32c5", "esp32c6", "esp32h2", "esp32p4"): { - "toolchains": ["toolchain-riscv32-esp"], - "ulp_toolchain": None, - "debug_tools": ["tool-riscv32-esp-elf-gdb"] - } + def _get_mcu_config(self, mcu: str) -> Optional[Dict]: + """MCU configuration with optimized search""" + if mcu in self._mcu_config_cache: + return self._mcu_config_cache[mcu] + + for _, config in MCU_TOOLCHAIN_CONFIG.items(): + if mcu in config["mcus"]: + # Dynamically add ULP toolchain + result = config.copy() + result["ulp_toolchain"] = ["toolchain-esp32ulp"] + if mcu != "esp32": + result["ulp_toolchain"].append("toolchain-riscv32-esp") + self._mcu_config_cache[mcu] = result + return result + return None + + def _needs_debug_tools(self, variables: Dict, targets: List[str]) -> bool: + """Check if debug tools are needed""" + return bool( + variables.get("build_type") or + "debug" in targets or + variables.get("upload_protocol") + ) + + def _configure_mcu_toolchains(self, mcu: str, variables: Dict, targets: List[str]) -> None: + """Optimized MCU toolchain configuration""" + mcu_config = self._get_mcu_config(mcu) + if not mcu_config: + logger.warning(f"Unknown MCU: {mcu}") + return + + # Install base toolchains + for toolchain in mcu_config["toolchains"]: + self.install_tool(toolchain) + + # ULP toolchain if ULP directory exists + if mcu_config.get("ulp_toolchain") and os.path.isdir("ulp"): + for toolchain in mcu_config["ulp_toolchain"]: + self.install_tool(toolchain) + + # Debug tools when needed + if self._needs_debug_tools(variables, targets): + for debug_tool in mcu_config["debug_tools"]: + self.install_tool(debug_tool) + self.install_tool("tool-openocd-esp32") + + def _configure_installer(self) -> None: + """Configure installer""" + installer_path = os.path.join(self.packages_dir, "tl-install", "tools", "idf_tools.py") + if os.path.exists(installer_path): + self.packages["tl-install"]["optional"] = True + + def _install_common_idf_packages(self) -> None: + """Install common IDF packages""" + for package in COMMON_IDF_PACKAGES: + self.install_tool(package) + + def _configure_check_tools(self, variables: Dict) -> None: + """Configure check tools""" + check_tools = variables.get("check_tool", []) + if not check_tools: + return + + for package in CHECK_PACKAGES: + if any(tool in package for tool in check_tools): + self.install_tool(package) + + def _ensure_mklittlefs_version(self) -> None: + """Ensure correct mklittlefs version""" + piopm_path = os.path.join(self.packages_dir, "tool-mklittlefs", ".piopm") + + if os.path.exists(piopm_path): + try: + with open(piopm_path, 'r') as f: + package_data = json.load(f) + if package_data.get('version') != MKLITTLEFS_VERSION_320: + os.remove(piopm_path) + logger.info("Outdated mklittlefs version removed") + except (json.JSONDecodeError, KeyError) as e: + logger.error(f"Error reading mklittlefs package data: {e}") + + def _setup_mklittlefs_for_download(self) -> None: + """Setup mklittlefs for download functionality""" + mklittlefs_dir = os.path.join(self.packages_dir, "tool-mklittlefs") + mklittlefs400_dir = os.path.join(self.packages_dir, "tool-mklittlefs-4.0.0") + + # Ensure mklittlefs 3.2.0 is installed + if not os.path.exists(mklittlefs_dir): + self.install_tool("tool-mklittlefs") + if os.path.exists(os.path.join(mklittlefs_dir, "tools.json")): + self.install_tool("tool-mklittlefs") + + # Install mklittlefs 4.0.0 + if not os.path.exists(mklittlefs400_dir): + self.install_tool("tool-mklittlefs-4.0.0") + if os.path.exists(os.path.join(mklittlefs400_dir, "tools.json")): + self.install_tool("tool-mklittlefs-4.0.0") + + # Copy mklittlefs 4.0.0 over 3.2.0 + if os.path.exists(mklittlefs400_dir): + package_src = os.path.join(mklittlefs_dir, "package.json") + package_dst = os.path.join(mklittlefs400_dir, "package.json") + safe_copy_file(package_src, package_dst) + shutil.copytree(mklittlefs400_dir, mklittlefs_dir, dirs_exist_ok=True) + self.packages.pop("tool-mkfatfs", None) + + def _handle_littlefs_tool(self, for_download: bool) -> None: + """Special handling for LittleFS tools""" + if for_download: + self._setup_mklittlefs_for_download() + else: + self._ensure_mklittlefs_version() + self.install_tool("tool-mklittlefs") + + def _install_filesystem_tool(self, filesystem: str, for_download: bool = False) -> None: + """Install filesystem tools""" + tool_mapping = { + "default": lambda: self._handle_littlefs_tool(for_download), + "fatfs": lambda: self.install_tool("tool-mkfatfs"), + "spiffs": lambda: self.install_tool("tool-mkspiffs") } - # Iterate through MCU mappings - for supported_mcus, toolchain_data in MCU_TOOLCHAIN_MAPPING.items(): - if mcu in supported_mcus: - # Set mandatory toolchains - for toolchain in toolchain_data["toolchains"]: - install_tool(toolchain) - # Set ULP toolchain if applicable - ulp_toolchain = toolchain_data.get("ulp_toolchain") - if ulp_toolchain and os.path.isdir("ulp"): - for toolchain in ulp_toolchain: - install_tool(toolchain) - # Install debug tools if conditions match - if (variables.get("build_type") or "debug" in "".join(targets)) or variables.get("upload_protocol"): - for debug_tool in toolchain_data["debug_tools"]: - install_tool(debug_tool) - install_tool("tool-openocd-esp32") - break # Exit loop once MCU is matched - - # Common packages for IDF and mixed Arduino+IDF projects - COMMON_IDF_PACKAGES = [ - "tool-cmake", - "tool-ninja", - "tool-scons", - "tool-esp-rom-elfs" - ] - if "espidf" in frameworks: - for package in COMMON_IDF_PACKAGES: - install_tool(package) - - CHECK_PACKAGES = [ - "tool-cppcheck", - "tool-clangtidy", - "tool-pvs-studio" - ] - # Install check tool listed in pio entry "check_tool" - if variables.get("check_tool") is not None: - for package in CHECK_PACKAGES: - for check_tool in variables.get("check_tool", ""): - if check_tool in package: - install_tool(package) - - if "buildfs" or "uploadfs" in targets: - filesystem = variables.get("board_build.filesystem", "littlefs") - if filesystem == "littlefs": - # ensure use of mklittlefs 3.2.0 - piopm_path = os.path.join(ProjectConfig.get_instance().get("platformio", "packages_dir"), "tool-mklittlefs", ".piopm") - if os.path.exists(piopm_path): - with open(piopm_path, "r") as file: - package_data = json.load(file) - if package_data['version'] != "3.2.0": - os.remove(piopm_path) - install_tool("tool-mklittlefs") - elif filesystem == "fatfs": - install_tool("tool-mkfatfs") - else: - install_tool("tool-mkspiffs") - if "downloadfs" in targets: - filesystem = variables.get("board_build.filesystem", "littlefs") - if filesystem == "littlefs": - # Use Tasmota mklittlefs v4.0.0 to unpack, older version is incompatible - # make sure mklittlefs 3.2.0 is installed - mklittlefs_dir = os.path.join(ProjectConfig.get_instance().get("platformio", "packages_dir"), "tool-mklittlefs") - if not os.path.exists(mklittlefs_dir): - install_tool("tool-mklittlefs") - if os.path.exists(os.path.join(mklittlefs_dir, "tools.json")): - install_tool("tool-mklittlefs") - mklittlefs400_dir = os.path.join(ProjectConfig.get_instance().get("platformio", "packages_dir"), "tool-mklittlefs-4.0.0") - if not os.path.exists(mklittlefs400_dir): - # install mklittlefs 4.0.0 - install_tool("tool-mklittlefs-4.0.0") - if os.path.exists(os.path.join(mklittlefs400_dir, "tools.json")): - install_tool("tool-mklittlefs-4.0.0") - # use mklittlefs 4.0.0 instead of 3.2.0 by copying over - if os.path.exists(mklittlefs400_dir): - shutil.copyfile( - os.path.join(mklittlefs_dir, "package.json"), - os.path.join(mklittlefs400_dir, "package.json"), - ) - shutil.copytree(mklittlefs400_dir, mklittlefs_dir, dirs_exist_ok=True) - del self.packages["tool-mkfatfs"] - elif filesystem == "fatfs": - install_tool("tool-mkfatfs") - - # Currently only Arduino Nano ESP32 uses the dfuutil tool as uploader + handler = tool_mapping.get(filesystem, tool_mapping["default"]) + handler() + + # Currently only Arduino Nano ESP32 uses the dfuutil tool as uploader + def _handle_dfuutil_tool(self, for_download: bool) -> None: + """Install dfuutil tool for Arduino Nano Board""" if variables.get("board") == "arduino_nano_esp32": - install_tool("tool-dfuutil-arduino") - else: - del self.packages["tool-dfuutil-arduino"] + self.install_tool("tool-dfuutil-arduino") + + def _configure_filesystem_tools(self, variables: Dict, targets: List[str]) -> None: + """Optimized filesystem tool configuration""" + filesystem = variables.get("board_build.filesystem", "littlefs") + + if any(target in targets for target in ["buildfs", "uploadfs"]): + self._install_filesystem_tool(filesystem, for_download=False) + + if "downloadfs" in targets: + self._install_filesystem_tool(filesystem, for_download=True) + + def configure_default_packages(self, variables: Dict, targets: List[str]) -> Any: + """Optimized main configuration method""" + if not variables.get("board"): + return super().configure_default_packages(variables, targets) + + # Base configuration + board_config = self.board_config(variables.get("board")) + mcu = variables.get("board_build.mcu", board_config.get("build.mcu", "esp32")) + frameworks = list(variables.get("pioframework", [])) # Create copy + + try: + # Configuration steps + self._configure_installer() + self._configure_arduino_framework(frameworks) + self._configure_espidf_framework(frameworks, variables, board_config, mcu) + self._configure_mcu_toolchains(mcu, variables, targets) + + if "espidf" in frameworks: + self._install_common_idf_packages() + + self._configure_check_tools(variables) + self._configure_filesystem_tools(variables, targets) + + logger.info("Package configuration completed successfully") + + except Exception as e: + logger.error(f"Error in package configuration: {type(e).__name__}: {e}") + # Don't re-raise to maintain compatibility return super().configure_default_packages(variables, targets) def get_boards(self, id_=None): + """Get board configuration""" result = super().get_boards(id_) if not result: return result @@ -258,13 +483,14 @@ def get_boards(self, id_=None): return result def _add_dynamic_options(self, board): - # upload protocols + """Add dynamic board options""" + # Upload protocols if not board.get("upload.protocols", []): board.manifest["upload"]["protocols"] = ["esptool", "espota"] if not board.get("upload.protocol", ""): board.manifest["upload"]["protocol"] = "esptool" - # debug tools + # Debug tools debug = board.manifest.get("debug", {}) non_debug_protocols = ["esptool", "espota"] supported_debug_tools = [ @@ -278,17 +504,21 @@ def _add_dynamic_options(self, board): "olimex-arm-usb-ocd-h", "olimex-arm-usb-ocd", "olimex-jtag-tiny", - "tumpa", + "tumpa" ] - # A special case for the Kaluga board that has a separate interface config + # Special configuration for Kaluga board if board.id == "esp32-s2-kaluga-1": supported_debug_tools.append("ftdi") - if board.get("build.mcu", "") in ("esp32c3", "esp32c5", "esp32c6", "esp32s3", "esp32h2", "esp32p4"): + + # ESP-builtin for certain MCUs + mcu = board.get("build.mcu", "") + if mcu in ESP_BUILTIN_DEBUG_MCUS: supported_debug_tools.append("esp-builtin") upload_protocol = board.manifest.get("upload", {}).get("protocol") upload_protocols = board.manifest.get("upload", {}).get("protocols", []) + if debug: upload_protocols.extend(supported_debug_tools) if upload_protocol and upload_protocol not in upload_protocols: @@ -298,37 +528,13 @@ def _add_dynamic_options(self, board): if "tools" not in debug: debug["tools"] = {} + # Debug tool configuration for link in upload_protocols: if link in non_debug_protocols or link in debug["tools"]: continue - if link in ("jlink", "cmsis-dap"): - openocd_interface = link - elif link in ("esp-prog", "ftdi"): - if board.id == "esp32-s2-kaluga-1": - openocd_interface = "ftdi/esp32s2_kaluga_v1" - else: - openocd_interface = "ftdi/esp32_devkitj_v1" - elif link == "esp-bridge": - openocd_interface = "esp_usb_bridge" - elif link == "esp-builtin": - openocd_interface = "esp_usb_jtag" - else: - openocd_interface = "ftdi/" + link - - server_args = [ - "-s", - "$PACKAGE_DIR/share/openocd/scripts", - "-f", - "interface/%s.cfg" % openocd_interface, - "-f", - "%s/%s" - % ( - ("target", debug.get("openocd_target")) - if "openocd_target" in debug - else ("board", debug.get("openocd_board")) - ), - ] + openocd_interface = self._get_openocd_interface(link, board) + server_args = self._get_debug_server_args(openocd_interface, debug) debug["tools"][link] = { "server": { @@ -360,13 +566,44 @@ def _add_dynamic_options(self, board): board.manifest["debug"] = debug return board + def _get_openocd_interface(self, link: str, board) -> str: + """Determine OpenOCD interface for debug link""" + if link in ("jlink", "cmsis-dap"): + return link + elif link in ("esp-prog", "ftdi"): + if board.id == "esp32-s2-kaluga-1": + return "ftdi/esp32s2_kaluga_v1" + else: + return "ftdi/esp32_devkitj_v1" + elif link == "esp-bridge": + return "esp_usb_bridge" + elif link == "esp-builtin": + return "esp_usb_jtag" + else: + return f"ftdi/{link}" + + def _get_debug_server_args(self, openocd_interface: str, debug: Dict) -> List[str]: + """Generate debug server arguments""" + if 'openocd_target' in debug: + config_type = 'target' + config_name = debug.get('openocd_target') + else: + config_type = 'board' + config_name = debug.get('openocd_board') + return [ + "-s", "$PACKAGE_DIR/share/openocd/scripts", + "-f", f"interface/{openocd_interface}.cfg", + "-f", f"{config_type}/{config_name}.cfg" + ] + def configure_debug_session(self, debug_config): + """Configure debug session""" build_extra_data = debug_config.build_data.get("extra", {}) flash_images = build_extra_data.get("flash_images", []) if "openocd" in (debug_config.server or {}).get("executable", ""): debug_config.server["arguments"].extend( - ["-c", "adapter speed %s" % (debug_config.speed or "5000")] + ["-c", f"adapter speed {debug_config.speed or DEFAULT_DEBUG_SPEED}"] ) ignore_conds = [ @@ -379,16 +616,12 @@ def configure_debug_session(self, debug_config): return load_cmds = [ - 'monitor program_esp "{{{path}}}" {offset} verify'.format( - path=to_unix_path(item["path"]), offset=item["offset"] - ) + f'monitor program_esp "{to_unix_path(item["path"])}" {item["offset"]} verify' for item in flash_images ] load_cmds.append( - 'monitor program_esp "{%s.bin}" %s verify' - % ( - to_unix_path(debug_config.build_data["prog_path"][:-4]), - build_extra_data.get("application_offset", "0x10000"), - ) + f'monitor program_esp "{to_unix_path(debug_config.build_data["prog_path"][:-4])}.bin" ' + f'{build_extra_data.get("application_offset", DEFAULT_APP_OFFSET)} verify' ) debug_config.load_cmds = load_cmds + From 4caa80524d87119b490274198ed3b12c7b4098bb Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Thu, 29 May 2025 20:26:08 +0200 Subject: [PATCH 31/97] remove copy&paste error --- platform.py | 1 - 1 file changed, 1 deletion(-) diff --git a/platform.py b/platform.py index 359ae9cb4..1b5b1dce6 100644 --- a/platform.py +++ b/platform.py @@ -280,7 +280,6 @@ def _configure_arduino_framework(self, frameworks: List[str]) -> None: self.packages["framework-arduinoespressif32"]["optional"] = False self.packages["framework-arduinoespressif32-libs"]["optional"] = False # use branch master - board_sdkconfig = variables.get("board_espidf.custom_sdkconfig", URL = "https://raw.githubusercontent.com/espressif/arduino-esp32/master/package/package_esp32_index.template.json" packjdata = requests.get(URL).json() dyn_lib_url = packjdata['packages'][0]['tools'][0]['systems'][0]['url'] From f37d892b673987c5516ef0e3d7e2b54ec1ae5196 Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Thu, 29 May 2025 20:33:51 +0200 Subject: [PATCH 32/97] add missing import requests --- platform.py | 1 + 1 file changed, 1 insertion(+) diff --git a/platform.py b/platform.py index 1b5b1dce6..22abd39de 100644 --- a/platform.py +++ b/platform.py @@ -15,6 +15,7 @@ import os import contextlib import json +import requests import subprocess import sys import shutil From eea799f15372df24f3f8699d775ac3d5b99d53da Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Thu, 29 May 2025 20:53:59 +0200 Subject: [PATCH 33/97] Update platformio.ini --- examples/arduino-blink/platformio.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/arduino-blink/platformio.ini b/examples/arduino-blink/platformio.ini index 8534e061e..00c055bd2 100644 --- a/examples/arduino-blink/platformio.ini +++ b/examples/arduino-blink/platformio.ini @@ -15,7 +15,7 @@ build_flags = -DLED_BUILTIN=2 lib_ignore = wifi spiffs NetworkClientSecure - + custom_component_remove = espressif/esp_hosted espressif/esp_wifi_remote From 27dfe295b13fe4e257b50a98e3c8135e8865e000 Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Thu, 29 May 2025 20:57:26 +0200 Subject: [PATCH 34/97] Update component_manager.py --- builder/frameworks/component_manager.py | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/builder/frameworks/component_manager.py b/builder/frameworks/component_manager.py index 66b006b5b..6bf4f627d 100644 --- a/builder/frameworks/component_manager.py +++ b/builder/frameworks/component_manager.py @@ -131,12 +131,8 @@ def _has_bt_ble_dependencies(self) -> bool: lib_deps_str = ' '.join(str(dep) for dep in lib_deps).upper() bt_ble_keywords = ['BLE', 'BT', 'NIMBLE', 'BLUETOOTH'] - - for keyword in bt_ble_keywords: - if keyword in lib_deps_str: - return True - - return False + + return any(keyword in lib_deps_str for keyword in bt_ble_keywords) except Exception: return False @@ -157,12 +153,8 @@ def _is_bt_related_library(self, lib_name: str) -> bool: 'ESP_BLE', 'ESP_BT' ] - - for bt_name in bt_related_names: - if bt_name in lib_name_upper: - return True - - return False + + return any(bt_name in lib_name_upper for bt_name in bt_related_names) def _get_arduino_core_libraries(self) -> Dict[str, str]: """Get all Arduino core libraries and their corresponding include paths.""" From d7f29a2d94aa35d98959ff36c04e1e4ee55aaa42 Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Thu, 29 May 2025 20:58:53 +0200 Subject: [PATCH 35/97] Update espidf.py --- builder/frameworks/espidf.py | 1 - 1 file changed, 1 deletion(-) diff --git a/builder/frameworks/espidf.py b/builder/frameworks/espidf.py index 2fbd1d0e1..da526020b 100644 --- a/builder/frameworks/espidf.py +++ b/builder/frameworks/espidf.py @@ -135,7 +135,6 @@ def _get_installed_standard_pip_packages(): flag_custom_sdkonfig = False flag_custom_component_add = False flag_custom_component_remove = False -removed_components = set() IDF5 = ( platform.get_package_version("framework-espidf") From 022ed65462ac995c4a9423d0fdbd80ee07b81288 Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Thu, 29 May 2025 21:10:40 +0200 Subject: [PATCH 36/97] fix potential path traversal vulnerability --- builder/frameworks/espidf.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/builder/frameworks/espidf.py b/builder/frameworks/espidf.py index da526020b..3ec070e96 100644 --- a/builder/frameworks/espidf.py +++ b/builder/frameworks/espidf.py @@ -219,18 +219,22 @@ def load_custom_sdkconfig_file(): response = requests.get(file_entry.split(" ")[0]) if response.ok: return response.content.decode('utf-8') - except Exception as e: + except requests.RequestException as e: print(f"Error downloading {file_entry}: {e}") + except UnicodeDecodeError as e: + print(f"Error decoding response from {file_entry}: {e}") return "" # Handle local files if "file://" in file_entry: - file_path = join(PROJECT_DIR, file_entry.lstrip("file://").split(os.path.sep)[-1]) + file_ref = file_entry[7:] if file_entry.startswith("file://") else file_entry + filename = os.path.basename(file_ref) + file_path = join(PROJECT_DIR, filename) if os.path.exists(file_path): try: with open(file_path, 'r') as f: return f.read() - except Exception as e: + except IOError as e: print(f"Error reading file {file_path}: {e}") return "" else: From d5407a668f2f4d5e4a2b9ae8367b7c0030fb2317 Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Thu, 29 May 2025 21:25:06 +0200 Subject: [PATCH 37/97] comment C2 Lib copy should be enough in espidf.py --- builder/frameworks/arduino.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/builder/frameworks/arduino.py b/builder/frameworks/arduino.py index a728ef055..e47dbd8d9 100644 --- a/builder/frameworks/arduino.py +++ b/builder/frameworks/arduino.py @@ -488,11 +488,11 @@ def check_reinstall_frwrk(): return False def call_compile_libs(): - if mcu == "esp32c2": - ARDUINO_FRMWRK_C2_LIB_DIR = join(platform.get_package_dir("framework-arduinoespressif32-libs"),mcu) - if not os.path.exists(ARDUINO_FRMWRK_C2_LIB_DIR): - ARDUINO_C2_DIR = join(platform.get_package_dir("framework-arduino-c2-skeleton-lib"),mcu) - shutil.copytree(ARDUINO_C2_DIR, ARDUINO_FRMWRK_C2_LIB_DIR, dirs_exist_ok=True) +# if mcu == "esp32c2": +# ARDUINO_FRMWRK_C2_LIB_DIR = join(platform.get_package_dir("framework-arduinoespressif32-libs"),mcu) +# if not os.path.exists(ARDUINO_FRMWRK_C2_LIB_DIR): +# ARDUINO_C2_DIR = join(platform.get_package_dir("framework-arduino-c2-skeleton-lib"),mcu) +# shutil.copytree(ARDUINO_C2_DIR, ARDUINO_FRMWRK_C2_LIB_DIR, dirs_exist_ok=True) print(f"*** Compile Arduino IDF libs for {pioenv} ***") SConscript("espidf.py") From 90291e74179b473d173feae3269c4c57db8bb400 Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Thu, 29 May 2025 21:33:03 +0200 Subject: [PATCH 38/97] remove unnecessary c2 lib copy done in espidf.py --- builder/frameworks/arduino.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/builder/frameworks/arduino.py b/builder/frameworks/arduino.py index e47dbd8d9..373e033e0 100644 --- a/builder/frameworks/arduino.py +++ b/builder/frameworks/arduino.py @@ -488,11 +488,6 @@ def check_reinstall_frwrk(): return False def call_compile_libs(): -# if mcu == "esp32c2": -# ARDUINO_FRMWRK_C2_LIB_DIR = join(platform.get_package_dir("framework-arduinoespressif32-libs"),mcu) -# if not os.path.exists(ARDUINO_FRMWRK_C2_LIB_DIR): -# ARDUINO_C2_DIR = join(platform.get_package_dir("framework-arduino-c2-skeleton-lib"),mcu) -# shutil.copytree(ARDUINO_C2_DIR, ARDUINO_FRMWRK_C2_LIB_DIR, dirs_exist_ok=True) print(f"*** Compile Arduino IDF libs for {pioenv} ***") SConscript("espidf.py") From 1740f77de0db593909a2bed47a5bae379cb221a5 Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Thu, 29 May 2025 22:23:12 +0200 Subject: [PATCH 39/97] fix sdk_dir --- builder/frameworks/arduino.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/builder/frameworks/arduino.py b/builder/frameworks/arduino.py index 373e033e0..b026a192f 100644 --- a/builder/frameworks/arduino.py +++ b/builder/frameworks/arduino.py @@ -116,7 +116,7 @@ def framework_lib_dir(self): def sdk_dir(self): if self._sdk_dir is None: self._sdk_dir = fs.to_unix_path( - join(self.framework_dir, "tools", "esp32-arduino-libs", self.mcu, "include") + join(self.framework_lib_dir, self.mcu, "include") ) return self._sdk_dir From dae7f84f5cf5649d93388ee50df7b227747f4e92 Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Fri, 30 May 2025 10:58:58 +0200 Subject: [PATCH 40/97] refactor components add and remove --- builder/frameworks/component_manager.py | 60 ++++++++++++++----------- 1 file changed, 34 insertions(+), 26 deletions(-) diff --git a/builder/frameworks/component_manager.py b/builder/frameworks/component_manager.py index 6bf4f627d..fddc23ccf 100644 --- a/builder/frameworks/component_manager.py +++ b/builder/frameworks/component_manager.py @@ -26,35 +26,43 @@ def __init__(self, env): def handle_component_settings(self, add_components: bool = False, remove_components: bool = False) -> None: """Handle adding and removing IDF components based on project configuration.""" - if not (add_components or remove_components): - return - + # Create backup before first component removal - if remove_components and not self.removed_components: + if remove_components and not self.removed_components or add_components and not self.add_components: self._backup_pioarduino_build_py() + + # Check if env and GetProjectOption are available + if hasattr(self, 'env') or hasattr(self.env, 'GetProjectOption'): + component_yml_path = self._get_or_create_component_yml() + component_data = self._load_component_yml(component_yml_path) + + if remove_components: + try: + remove_option = self.env.GetProjectOption("custom_component_remove", None) + if remove_option: + components_to_remove = remove_option.splitlines() + self._remove_components(component_data, components_to_remove) + except Exception as e: + # Optional: Logging for debugging + # print(f"Error removing components: {e}") + pass + + if add_components: + try: + add_option = self.env.GetProjectOption("custom_component_add", None) + if add_option: + components_to_add = add_option.splitlines() + self._add_components(component_data, components_to_add) + except Exception as e: + # Optional: Logging for debugging + # print(f"Error adding components: {e}") + pass + + self._save_component_yml(component_yml_path, component_data) - component_yml_path = self._get_or_create_component_yml() - component_data = self._load_component_yml(component_yml_path) - - if remove_components: - try: - components_to_remove = self.env.GetProjectOption("custom_component_remove").splitlines() - self._remove_components(component_data, components_to_remove) - except: - pass - - if add_components: - try: - components_to_add = self.env.GetProjectOption("custom_component_add").splitlines() - self._add_components(component_data, components_to_add) - except: - pass - - self._save_component_yml(component_yml_path, component_data) - - # Clean up removed components - if self.removed_components: - self._cleanup_removed_components() + # Clean up removed components + if self.removed_components: + self._cleanup_removed_components() self.handle_lib_ignore() From ca701446456cd4935e871f65553c4f7a0e483d5c Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Fri, 30 May 2025 11:01:56 +0200 Subject: [PATCH 41/97] always call ComponentManager() --- builder/frameworks/arduino.py | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/builder/frameworks/arduino.py b/builder/frameworks/arduino.py index b026a192f..a6aca8a08 100644 --- a/builder/frameworks/arduino.py +++ b/builder/frameworks/arduino.py @@ -688,18 +688,13 @@ def get_frameworks_in_current_env(): if ("arduino" in pioframework and "espidf" not in pioframework and arduino_lib_compile_flag in ("Inactive", "True")): - - if flag_custom_component_remove or flag_lib_ignore: - flag_remove = True - from component_manager import ComponentManager - component_manager = ComponentManager(env) - component_manager.handle_component_settings( - add_components=flag_custom_component_add, - remove_components=flag_remove - ) - silent_action = env.Action(component_manager.restore_pioarduino_build_py) - silent_action.strfunction = lambda target, source, env: '' # hack to silence scons command output - env.AddPostAction("checkprogsize", silent_action) + # try to remove not needed include path if an lib_ignore entry exists + from component_manager import ComponentManager + component_manager = ComponentManager(env) + component_manager.handle_component_settings() + silent_action = env.Action(component_manager.restore_pioarduino_build_py) + silent_action.strfunction = lambda target, source, env: '' # hack to silence scons command output + env.AddPostAction("checkprogsize", silent_action) if IS_WINDOWS: # Smart include path optimization based on total path length From 2122614f79a698057b93259ddd62dbb583a34170 Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Fri, 30 May 2025 11:17:15 +0200 Subject: [PATCH 42/97] add logging --- builder/frameworks/component_manager.py | 258 ++++++++++++++++++++---- 1 file changed, 222 insertions(+), 36 deletions(-) diff --git a/builder/frameworks/component_manager.py b/builder/frameworks/component_manager.py index fddc23ccf..5233cb23e 100644 --- a/builder/frameworks/component_manager.py +++ b/builder/frameworks/component_manager.py @@ -9,9 +9,15 @@ class ComponentManager: - """Manages IDF components for ESP32 Arduino framework builds.""" + """Manages IDF components for ESP32 Arduino framework builds with logging support.""" def __init__(self, env): + """ + Initialize the ComponentManager. + + Args: + env: PlatformIO environment object + """ self.env = env self.platform = env.PioPlatform() self.config = env.GetProjectConfig() @@ -21,18 +27,38 @@ def __init__(self, env): self.removed_components: Set[str] = set() self.ignored_libs: Set[str] = set() + # Simple logging attributes + self.component_changes: List[str] = [] + self.arduino_framework_dir = self.platform.get_package_dir("framework-arduinoespressif32") self.arduino_libs_mcu = join(self.platform.get_package_dir("framework-arduinoespressif32-libs"), self.mcu) + def _log_change(self, message: str) -> None: + """ + Simple logging without timestamp. + + Args: + message: Log message to record + """ + self.component_changes.append(message) + print(f"[ComponentManager] {message}") + def handle_component_settings(self, add_components: bool = False, remove_components: bool = False) -> None: - """Handle adding and removing IDF components based on project configuration.""" + """ + Handle adding and removing IDF components based on project configuration. + + Args: + add_components: Whether to process component additions + remove_components: Whether to process component removals + """ # Create backup before first component removal - if remove_components and not self.removed_components or add_components and not self.add_components: + if remove_components and not self.removed_components or add_components: self._backup_pioarduino_build_py() + self._log_change("Created backup of build file") # Check if env and GetProjectOption are available - if hasattr(self, 'env') or hasattr(self.env, 'GetProjectOption'): + if hasattr(self, 'env') and hasattr(self.env, 'GetProjectOption'): component_yml_path = self._get_or_create_component_yml() component_data = self._load_component_yml(component_yml_path) @@ -43,9 +69,7 @@ def handle_component_settings(self, add_components: bool = False, remove_compone components_to_remove = remove_option.splitlines() self._remove_components(component_data, components_to_remove) except Exception as e: - # Optional: Logging for debugging - # print(f"Error removing components: {e}") - pass + self._log_change(f"Error removing components: {str(e)}") if add_components: try: @@ -54,9 +78,7 @@ def handle_component_settings(self, add_components: bool = False, remove_compone components_to_add = add_option.splitlines() self._add_components(component_data, components_to_add) except Exception as e: - # Optional: Logging for debugging - # print(f"Error adding components: {e}") - pass + self._log_change(f"Error adding components: {str(e)}") self._save_component_yml(component_yml_path, component_data) @@ -65,6 +87,10 @@ def handle_component_settings(self, add_components: bool = False, remove_compone self._cleanup_removed_components() self.handle_lib_ignore() + + # Print summary + if self.component_changes: + self._log_change(f"Session completed with {len(self.component_changes)} changes") def handle_lib_ignore(self) -> None: """Handle lib_ignore entries from platformio.ini and remove corresponding includes.""" @@ -78,10 +104,15 @@ def handle_lib_ignore(self) -> None: if lib_ignore_entries: self.ignored_libs.update(lib_ignore_entries) self._remove_ignored_lib_includes() - print(f"Removed include paths for {len(lib_ignore_entries)} ignored libraries") + self._log_change(f"Processed {len(lib_ignore_entries)} ignored libraries") def _get_lib_ignore_entries(self) -> List[str]: - """Get lib_ignore entries from current environment configuration only.""" + """ + Get lib_ignore entries from current environment configuration only. + + Returns: + List of library names to ignore + """ try: # Get lib_ignore from current environment only lib_ignore = self.env.GetProjectOption("lib_ignore", []) @@ -125,7 +156,12 @@ def _get_lib_ignore_entries(self) -> List[str]: return [] def _has_bt_ble_dependencies(self) -> bool: - """Check if lib_deps contains any BT/BLE related dependencies.""" + """ + Check if lib_deps contains any BT/BLE related dependencies. + + Returns: + True if BT/BLE dependencies are found + """ try: # Get lib_deps from current environment lib_deps = self.env.GetProjectOption("lib_deps", []) @@ -146,7 +182,15 @@ def _has_bt_ble_dependencies(self) -> bool: return False def _is_bt_related_library(self, lib_name: str) -> bool: - """Check if a library name is related to Bluetooth/BLE functionality.""" + """ + Check if a library name is related to Bluetooth/BLE functionality. + + Args: + lib_name: Library name to check + + Returns: + True if library is BT/BLE related + """ lib_name_upper = lib_name.upper() bt_related_names = [ @@ -165,7 +209,12 @@ def _is_bt_related_library(self, lib_name: str) -> bool: return any(bt_name in lib_name_upper for bt_name in bt_related_names) def _get_arduino_core_libraries(self) -> Dict[str, str]: - """Get all Arduino core libraries and their corresponding include paths.""" + """ + Get all Arduino core libraries and their corresponding include paths. + + Returns: + Dictionary mapping library names to include paths + """ libraries_mapping = {} # Path to Arduino Core Libraries @@ -189,7 +238,15 @@ def _get_arduino_core_libraries(self) -> Dict[str, str]: return libraries_mapping def _get_library_name_from_properties(self, lib_dir: str) -> Optional[str]: - """Extract library name from library.properties file.""" + """ + Extract library name from library.properties file. + + Args: + lib_dir: Library directory path + + Returns: + Library name or None if not found + """ prop_path = join(lib_dir, "library.properties") if not os.path.isfile(prop_path): return None @@ -206,7 +263,16 @@ def _get_library_name_from_properties(self, lib_dir: str) -> Optional[str]: return None def _map_library_to_include_path(self, lib_name: str, dir_name: str) -> str: - """Map library name to corresponding include path.""" + """ + Map library name to corresponding include path. + + Args: + lib_name: Library name + dir_name: Directory name + + Returns: + Mapped include path + """ lib_name_lower = lib_name.lower().replace(' ', '').replace('-', '_') dir_name_lower = dir_name.lower() @@ -303,7 +369,15 @@ def _map_library_to_include_path(self, lib_name: str, dir_name: str) -> str: return dir_name_lower def _convert_lib_name_to_include(self, lib_name: str) -> str: - """Convert library name to potential include directory name.""" + """ + Convert library name to potential include directory name. + + Args: + lib_name: Library name to convert + + Returns: + Converted include directory name + """ # Load Arduino Core Libraries on first call if not hasattr(self, '_arduino_libraries_cache'): self._arduino_libraries_cache = self._get_arduino_core_libraries() @@ -346,14 +420,17 @@ def _convert_lib_name_to_include(self, lib_name: str) -> str: return cleaned_name def _remove_ignored_lib_includes(self) -> None: - """Remove include entries for ignored libraries from pioarduino-build.py.""" + """Remove include entries for ignored libraries from pioarduino-build.py with simple logging.""" build_py_path = join(self.arduino_libs_mcu, "pioarduino-build.py") if not os.path.exists(build_py_path): + self._log_change("Build file not found") return # Check if BT/BLE dependencies exist in lib_deps bt_ble_protected = self._has_bt_ble_dependencies() + if bt_ble_protected: + self._log_change("BT/BLE protection enabled") try: with open(build_py_path, 'r') as f: @@ -366,12 +443,12 @@ def _remove_ignored_lib_includes(self) -> None: for lib_name in self.ignored_libs: # Skip BT-related libraries if BT/BLE dependencies are present if bt_ble_protected and self._is_bt_related_library(lib_name): - print(f"Skipping removal of BT-related library '{lib_name}' due to BT/BLE dependency in lib_deps") + self._log_change(f"Protected BT library: {lib_name}") continue # Hard protection for DSP components if lib_name.lower() in ['dsp', 'esp_dsp', 'dsps', 'fft2r', 'dsps_fft2r']: - print(f"Hard-protected DSP component '{lib_name}' from removal") + self._log_change(f"Protected DSP component: {lib_name}") continue # Multiple patterns to catch different include formats @@ -386,11 +463,16 @@ def _remove_ignored_lib_includes(self) -> None: rf'\s*"[^"]*/{re.escape(lib_name)}/[^"]*",?\n' ] + removed_count = 0 for pattern in patterns: matches = re.findall(pattern, content) if matches: content = re.sub(pattern, '', content) - total_removed += len(matches) + removed_count += len(matches) + + if removed_count > 0: + self._log_change(f"Ignored library: {lib_name} ({removed_count} entries)") + total_removed += removed_count # Clean up empty lines and trailing commas content = re.sub(r'\n\s*\n', '\n', content) @@ -400,12 +482,22 @@ def _remove_ignored_lib_includes(self) -> None: if self._validate_changes(original_content, content) and content != original_content: with open(build_py_path, 'w') as f: f.write(content) + self._log_change(f"Updated build file ({total_removed} total removals)") - except Exception: - pass + except Exception as e: + self._log_change(f"Error processing libraries: {str(e)}") def _validate_changes(self, original_content: str, new_content: str) -> bool: - """Validate that the changes are reasonable.""" + """ + Validate that the changes are reasonable. + + Args: + original_content: Original file content + new_content: Modified file content + + Returns: + True if changes are valid + """ original_lines = len(original_content.splitlines()) new_lines = len(new_content.splitlines()) removed_lines = original_lines - new_lines @@ -414,7 +506,12 @@ def _validate_changes(self, original_content: str, new_content: str) -> bool: return not (removed_lines > original_lines * 0.5 or removed_lines < 0) def _get_or_create_component_yml(self) -> str: - """Get path to idf_component.yml, creating it if necessary.""" + """ + Get path to idf_component.yml, creating it if necessary. + + Returns: + Path to component YAML file + """ # Try Arduino framework first framework_yml = join(self.arduino_framework_dir, "idf_component.yml") if os.path.exists(framework_yml): @@ -432,13 +529,23 @@ def _get_or_create_component_yml(self) -> str: return project_yml def _create_backup(self, file_path: str) -> None: - """Create backup of a file.""" + """ + Create backup of a file. + + Args: + file_path: Path to file to backup + """ backup_path = f"{file_path}.orig" if not os.path.exists(backup_path): shutil.copy(file_path, backup_path) def _create_default_component_yml(self, file_path: str) -> None: - """Create a default idf_component.yml file.""" + """ + Create a default idf_component.yml file. + + Args: + file_path: Path where to create the file + """ default_content = { "dependencies": { "idf": ">=5.1" @@ -449,7 +556,15 @@ def _create_default_component_yml(self, file_path: str) -> None: yaml.dump(default_content, f) def _load_component_yml(self, file_path: str) -> Dict[str, Any]: - """Load and parse idf_component.yml file.""" + """ + Load and parse idf_component.yml file. + + Args: + file_path: Path to YAML file + + Returns: + Parsed YAML data + """ try: with open(file_path, "r") as f: return yaml.load(f, Loader=SafeLoader) or {"dependencies": {}} @@ -457,7 +572,13 @@ def _load_component_yml(self, file_path: str) -> Dict[str, Any]: return {"dependencies": {}} def _save_component_yml(self, file_path: str, data: Dict[str, Any]) -> None: - """Save component data to YAML file.""" + """ + Save component data to YAML file. + + Args: + file_path: Path to YAML file + data: Data to save + """ try: with open(file_path, "w") as f: yaml.dump(data, f) @@ -465,7 +586,13 @@ def _save_component_yml(self, file_path: str, data: Dict[str, Any]) -> None: pass def _remove_components(self, component_data: Dict[str, Any], components_to_remove: list) -> None: - """Remove specified components from the configuration.""" + """ + Remove specified components from the configuration with simple logging. + + Args: + component_data: Component configuration data + components_to_remove: List of components to remove + """ dependencies = component_data.setdefault("dependencies", {}) for component in components_to_remove: @@ -474,14 +601,23 @@ def _remove_components(self, component_data: Dict[str, Any], components_to_remov continue if component in dependencies: + self._log_change(f"Removed component: {component}") del dependencies[component] # Track for cleanup filesystem_name = self._convert_component_name_to_filesystem(component) self.removed_components.add(filesystem_name) + else: + self._log_change(f"Component not found: {component}") def _add_components(self, component_data: Dict[str, Any], components_to_add: list) -> None: - """Add specified components to the configuration.""" + """ + Add specified components to the configuration with simple logging. + + Args: + component_data: Component configuration data + components_to_add: List of components to add + """ dependencies = component_data.setdefault("dependencies", {}) for component in components_to_add: @@ -493,16 +629,35 @@ def _add_components(self, component_data: Dict[str, Any], components_to_add: lis if component_name not in dependencies: dependencies[component_name] = {"version": version} + self._log_change(f"Added component: {component_name} ({version})") + else: + self._log_change(f"Component already exists: {component_name}") def _parse_component_entry(self, entry: str) -> tuple[str, str]: - """Parse component entry into name and version.""" + """ + Parse component entry into name and version. + + Args: + entry: Component entry string + + Returns: + Tuple of (component_name, version) + """ if "@" in entry: name, version = entry.split("@", 1) return (name.strip(), version.strip()) return (entry.strip(), "*") def _convert_component_name_to_filesystem(self, component_name: str) -> str: - """Convert component name from registry format to filesystem format.""" + """ + Convert component name from registry format to filesystem format. + + Args: + component_name: Component name to convert + + Returns: + Filesystem-safe component name + """ return component_name.replace("/", "__") def _backup_pioarduino_build_py(self) -> None: @@ -524,7 +679,12 @@ def _cleanup_removed_components(self) -> None: self._remove_cpppath_entries() def _remove_include_directory(self, component: str) -> None: - """Remove include directory for a component.""" + """ + Remove include directory for a component. + + Args: + component: Component name + """ include_path = join(self.arduino_libs_mcu, "include", component) if os.path.exists(include_path): @@ -562,10 +722,36 @@ def _remove_cpppath_entries(self) -> None: pass def restore_pioarduino_build_py(self, source=None, target=None, env=None) -> None: - """Restore the original pioarduino-build.py from backup.""" + """ + Restore the original pioarduino-build.py from backup. + + Args: + source: Build source (unused) + target: Build target (unused) + env: Environment (unused) + """ build_py_path = join(self.arduino_libs_mcu, "pioarduino-build.py") backup_path = join(self.arduino_libs_mcu, f"pioarduino-build.py.{self.mcu}") if os.path.exists(backup_path): shutil.copy2(backup_path, build_py_path) os.remove(backup_path) + + def get_changes_summary(self) -> List[str]: + """ + Get simple list of all changes made. + + Returns: + List of change messages + """ + return self.component_changes.copy() + + def print_changes_summary(self) -> None: + """Print a simple summary of all changes.""" + if self.component_changes: + print("\n=== Component Manager Changes ===") + for change in self.component_changes: + print(f" {change}") + print("=" * 35) + else: + print("[ComponentManager] No changes made") From ee04c7dae3d54a5a48fab95e16fe1e0f36bb06f5 Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Fri, 30 May 2025 11:25:32 +0200 Subject: [PATCH 43/97] revert not intended logic change --- builder/frameworks/component_manager.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/builder/frameworks/component_manager.py b/builder/frameworks/component_manager.py index 5233cb23e..38114b9fe 100644 --- a/builder/frameworks/component_manager.py +++ b/builder/frameworks/component_manager.py @@ -53,12 +53,12 @@ def handle_component_settings(self, add_components: bool = False, remove_compone """ # Create backup before first component removal - if remove_components and not self.removed_components or add_components: + if remove_components and not self.removed_components or add_components and not self.add_components: self._backup_pioarduino_build_py() self._log_change("Created backup of build file") # Check if env and GetProjectOption are available - if hasattr(self, 'env') and hasattr(self.env, 'GetProjectOption'): + if hasattr(self, 'env') or hasattr(self.env, 'GetProjectOption'): component_yml_path = self._get_or_create_component_yml() component_data = self._load_component_yml(component_yml_path) From 358d1be97ac0adf0cdd8aa6c66b31efd82a0248c Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Fri, 30 May 2025 11:31:33 +0200 Subject: [PATCH 44/97] Update component_manager.py --- builder/frameworks/component_manager.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/builder/frameworks/component_manager.py b/builder/frameworks/component_manager.py index 38114b9fe..dc6b59223 100644 --- a/builder/frameworks/component_manager.py +++ b/builder/frameworks/component_manager.py @@ -52,8 +52,8 @@ def handle_component_settings(self, add_components: bool = False, remove_compone remove_components: Whether to process component removals """ - # Create backup before first component removal - if remove_components and not self.removed_components or add_components and not self.add_components: + # Create backup before first component removal and on every add of a component + if remove_components and not self.removed_components or add_components: self._backup_pioarduino_build_py() self._log_change("Created backup of build file") From efac91ced224b123b70c64a1e8c40a7bcf23f201 Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Fri, 30 May 2025 11:49:45 +0200 Subject: [PATCH 45/97] refactor backup of components yml and pioarduino-build.py --- builder/frameworks/component_manager.py | 180 +++++++++++++++++++----- 1 file changed, 146 insertions(+), 34 deletions(-) diff --git a/builder/frameworks/component_manager.py b/builder/frameworks/component_manager.py index dc6b59223..12adf3922 100644 --- a/builder/frameworks/component_manager.py +++ b/builder/frameworks/component_manager.py @@ -30,6 +30,10 @@ def __init__(self, env): # Simple logging attributes self.component_changes: List[str] = [] + # MCU-specific backup tracking + self.backup_created_per_mcu = {} # Track backups per MCU + self.yaml_backup_created = False + self.arduino_framework_dir = self.platform.get_package_dir("framework-arduinoespressif32") self.arduino_libs_mcu = join(self.platform.get_package_dir("framework-arduinoespressif32-libs"), self.mcu) @@ -52,15 +56,16 @@ def handle_component_settings(self, add_components: bool = False, remove_compone remove_components: Whether to process component removals """ - # Create backup before first component removal and on every add of a component - if remove_components and not self.removed_components or add_components: - self._backup_pioarduino_build_py() - self._log_change("Created backup of build file") + # Create MCU-specific backup before first component removal and on every add of a component + if remove_components and not self.backup_created_per_mcu.get(self.mcu, False) or add_components: + if self._backup_pioarduino_build_py(): + self._log_change(f"Created MCU backup for {self.mcu}") # Check if env and GetProjectOption are available - if hasattr(self, 'env') or hasattr(self.env, 'GetProjectOption'): + if hasattr(self, 'env') and hasattr(self.env, 'GetProjectOption'): component_yml_path = self._get_or_create_component_yml() component_data = self._load_component_yml(component_yml_path) + original_data = component_data.copy() if remove_components: try: @@ -80,7 +85,9 @@ def handle_component_settings(self, add_components: bool = False, remove_compone except Exception as e: self._log_change(f"Error adding components: {str(e)}") - self._save_component_yml(component_yml_path, component_data) + # Only save if changes were made + if component_data != original_data: + self._save_component_yml(component_yml_path, component_data) # Clean up removed components if self.removed_components: @@ -94,10 +101,6 @@ def handle_component_settings(self, add_components: bool = False, remove_compone def handle_lib_ignore(self) -> None: """Handle lib_ignore entries from platformio.ini and remove corresponding includes.""" - # Create backup before processing lib_ignore - if not self.ignored_libs: - self._backup_pioarduino_build_py() - # Get lib_ignore entries from current environment only lib_ignore_entries = self._get_lib_ignore_entries() @@ -434,9 +437,9 @@ def _remove_ignored_lib_includes(self) -> None: try: with open(build_py_path, 'r') as f: - content = f.read() + original_content = f.read() - original_content = content + modified_content = original_content total_removed = 0 # Remove CPPPATH entries for each ignored library @@ -465,9 +468,9 @@ def _remove_ignored_lib_includes(self) -> None: removed_count = 0 for pattern in patterns: - matches = re.findall(pattern, content) + matches = re.findall(pattern, modified_content) if matches: - content = re.sub(pattern, '', content) + modified_content = re.sub(pattern, '', modified_content) removed_count += len(matches) if removed_count > 0: @@ -475,14 +478,20 @@ def _remove_ignored_lib_includes(self) -> None: total_removed += removed_count # Clean up empty lines and trailing commas - content = re.sub(r'\n\s*\n', '\n', content) - content = re.sub(r',\s*\n\s*\]', '\n]', content) + modified_content = re.sub(r'\n\s*\n', '\n', modified_content) + modified_content = re.sub(r',\s*\n\s*\]', '\n]', modified_content) - # Validate and write changes - if self._validate_changes(original_content, content) and content != original_content: + # Only write if content actually changed + if self._validate_changes(original_content, modified_content) and modified_content != original_content: + # Ensure MCU-specific backup exists before modification + if not self.backup_created_per_mcu.get(self.mcu, False): + self._backup_pioarduino_build_py() + with open(build_py_path, 'w') as f: - f.write(content) + f.write(modified_content) self._log_change(f"Updated build file ({total_removed} total removals)") + else: + self._log_change("No library changes needed") except Exception as e: self._log_change(f"Error processing libraries: {str(e)}") @@ -528,16 +537,28 @@ def _get_or_create_component_yml(self) -> str: self._create_default_component_yml(project_yml) return project_yml - def _create_backup(self, file_path: str) -> None: + def _create_backup(self, file_path: str) -> bool: """ - Create backup of a file. + Create single backup of a file if it doesn't exist. Args: file_path: Path to file to backup + + Returns: + True if backup was created or already exists """ backup_path = f"{file_path}.orig" - if not os.path.exists(backup_path): - shutil.copy(file_path, backup_path) + + if os.path.exists(file_path) and not os.path.exists(backup_path): + try: + shutil.copy2(file_path, backup_path) + self._log_change(f"Single backup created: {backup_path}") + return True + except Exception as e: + self._log_change(f"Backup failed: {str(e)}") + return False + + return os.path.exists(backup_path) def _create_default_component_yml(self, file_path: str) -> None: """ @@ -573,17 +594,36 @@ def _load_component_yml(self, file_path: str) -> Dict[str, Any]: def _save_component_yml(self, file_path: str, data: Dict[str, Any]) -> None: """ - Save component data to YAML file. + Save component data to YAML file only if changed. Args: file_path: Path to YAML file data: Data to save """ try: + # Check if content would actually change + if os.path.exists(file_path): + with open(file_path, "r") as f: + existing_data = yaml.load(f, Loader=SafeLoader) or {} + + # Compare data structures + if existing_data == data: + self._log_change(f"YAML unchanged, skipping write: {file_path}") + return + + # Create backup before first modification (only once) + if not self.yaml_backup_created: + self._create_backup(file_path) + self.yaml_backup_created = True + + # Write only if changed with open(file_path, "w") as f: - yaml.dump(data, f) - except Exception: - pass + yaml.dump(data, f, default_flow_style=False, sort_keys=True) + + self._log_change(f"YAML updated: {file_path}") + + except Exception as e: + self._log_change(f"Error saving YAML: {str(e)}") def _remove_components(self, component_data: Dict[str, Any], components_to_remove: list) -> None: """ @@ -660,16 +700,61 @@ def _convert_component_name_to_filesystem(self, component_name: str) -> str: """ return component_name.replace("/", "__") - def _backup_pioarduino_build_py(self) -> None: - """Create backup of the original pioarduino-build.py.""" + def _backup_pioarduino_build_py(self) -> bool: + """Create MCU-specific backup of pioarduino-build.py only once per MCU.""" if "arduino" not in self.env.subst("$PIOFRAMEWORK"): - return + return False + + # Check if backup already created for this MCU in this session + if self.backup_created_per_mcu.get(self.mcu, False): + return True build_py_path = join(self.arduino_libs_mcu, "pioarduino-build.py") backup_path = join(self.arduino_libs_mcu, f"pioarduino-build.py.{self.mcu}") + # Only create backup if source exists and MCU-specific backup doesn't exist if os.path.exists(build_py_path) and not os.path.exists(backup_path): - shutil.copy2(build_py_path, backup_path) + try: + shutil.copy2(build_py_path, backup_path) + self.backup_created_per_mcu[self.mcu] = True + self._log_change(f"Created MCU-specific backup for {self.mcu}: {backup_path}") + return True + except Exception as e: + self._log_change(f"MCU backup creation failed for {self.mcu}: {str(e)}") + return False + elif os.path.exists(backup_path): + self.backup_created_per_mcu[self.mcu] = True + self._log_change(f"MCU backup already exists for {self.mcu}") + return True + + return False + + def _validate_mcu_backup(self) -> bool: + """ + Validate that MCU-specific backup exists and is valid. + + Returns: + True if backup is valid + """ + backup_path = join(self.arduino_libs_mcu, f"pioarduino-build.py.{self.mcu}") + + if not os.path.exists(backup_path): + return False + + try: + # Basic validation - check if file is readable and not empty + with open(backup_path, 'r') as f: + content = f.read() + if len(content) < 100: # Minimum expected size + self._log_change(f"MCU backup for {self.mcu} appears corrupted") + return False + + self._log_change(f"MCU backup for {self.mcu} validated") + return True + + except Exception as e: + self._log_change(f"MCU backup validation failed for {self.mcu}: {str(e)}") + return False def _cleanup_removed_components(self) -> None: """Clean up removed components and restore original build file.""" @@ -723,7 +808,7 @@ def _remove_cpppath_entries(self) -> None: def restore_pioarduino_build_py(self, source=None, target=None, env=None) -> None: """ - Restore the original pioarduino-build.py from backup. + Restore the MCU-specific pioarduino-build.py from backup. Args: source: Build source (unused) @@ -734,8 +819,35 @@ def restore_pioarduino_build_py(self, source=None, target=None, env=None) -> Non backup_path = join(self.arduino_libs_mcu, f"pioarduino-build.py.{self.mcu}") if os.path.exists(backup_path): - shutil.copy2(backup_path, build_py_path) - os.remove(backup_path) + try: + shutil.copy2(backup_path, build_py_path) + self._log_change(f"Restored MCU-specific build file for {self.mcu}") + + # Optional: Remove backup after restore if desired + # os.remove(backup_path) + # self._log_change(f"Removed MCU backup for {self.mcu}") + + except Exception as e: + self._log_change(f"Failed to restore MCU backup for {self.mcu}: {str(e)}") + else: + self._log_change(f"No MCU backup found for {self.mcu}") + + def get_mcu_backup_status(self) -> Dict[str, bool]: + """ + Get status of MCU-specific backups. + + Returns: + Dictionary with MCU types as keys and backup existence as values + """ + backup_dir = self.arduino_libs_mcu + mcu_types = ["esp32", "esp32s2", "esp32s3", "esp32c3", "esp32c6", "esp32h2"] + + status = {} + for mcu in mcu_types: + backup_path = join(backup_dir, f"pioarduino-build.py.{mcu}") + status[mcu] = os.path.exists(backup_path) + + return status def get_changes_summary(self) -> List[str]: """ From 1249333fde7dfcf78588c3a597b34a151084be08 Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Fri, 30 May 2025 12:41:48 +0200 Subject: [PATCH 46/97] revert: backup of refactor yml and pioarduino-build.py --- builder/frameworks/component_manager.py | 178 +++++------------------- 1 file changed, 33 insertions(+), 145 deletions(-) diff --git a/builder/frameworks/component_manager.py b/builder/frameworks/component_manager.py index 12adf3922..d6c248a6f 100644 --- a/builder/frameworks/component_manager.py +++ b/builder/frameworks/component_manager.py @@ -30,10 +30,6 @@ def __init__(self, env): # Simple logging attributes self.component_changes: List[str] = [] - # MCU-specific backup tracking - self.backup_created_per_mcu = {} # Track backups per MCU - self.yaml_backup_created = False - self.arduino_framework_dir = self.platform.get_package_dir("framework-arduinoespressif32") self.arduino_libs_mcu = join(self.platform.get_package_dir("framework-arduinoespressif32-libs"), self.mcu) @@ -56,16 +52,15 @@ def handle_component_settings(self, add_components: bool = False, remove_compone remove_components: Whether to process component removals """ - # Create MCU-specific backup before first component removal and on every add of a component - if remove_components and not self.backup_created_per_mcu.get(self.mcu, False) or add_components: - if self._backup_pioarduino_build_py(): - self._log_change(f"Created MCU backup for {self.mcu}") + # Create backup before first component removal and on every add of a component + if remove_components and not self.removed_components or add_components: + self._backup_pioarduino_build_py() + self._log_change("Created backup of build file") # Check if env and GetProjectOption are available if hasattr(self, 'env') and hasattr(self.env, 'GetProjectOption'): component_yml_path = self._get_or_create_component_yml() component_data = self._load_component_yml(component_yml_path) - original_data = component_data.copy() if remove_components: try: @@ -85,9 +80,7 @@ def handle_component_settings(self, add_components: bool = False, remove_compone except Exception as e: self._log_change(f"Error adding components: {str(e)}") - # Only save if changes were made - if component_data != original_data: - self._save_component_yml(component_yml_path, component_data) + self._save_component_yml(component_yml_path, component_data) # Clean up removed components if self.removed_components: @@ -101,6 +94,10 @@ def handle_component_settings(self, add_components: bool = False, remove_compone def handle_lib_ignore(self) -> None: """Handle lib_ignore entries from platformio.ini and remove corresponding includes.""" + # Create backup before processing lib_ignore + if not self.ignored_libs: + self._backup_pioarduino_build_py() + # Get lib_ignore entries from current environment only lib_ignore_entries = self._get_lib_ignore_entries() @@ -437,9 +434,9 @@ def _remove_ignored_lib_includes(self) -> None: try: with open(build_py_path, 'r') as f: - original_content = f.read() + content = f.read() - modified_content = original_content + original_content = content total_removed = 0 # Remove CPPPATH entries for each ignored library @@ -468,9 +465,9 @@ def _remove_ignored_lib_includes(self) -> None: removed_count = 0 for pattern in patterns: - matches = re.findall(pattern, modified_content) + matches = re.findall(pattern, content) if matches: - modified_content = re.sub(pattern, '', modified_content) + content = re.sub(pattern, '', content) removed_count += len(matches) if removed_count > 0: @@ -478,20 +475,14 @@ def _remove_ignored_lib_includes(self) -> None: total_removed += removed_count # Clean up empty lines and trailing commas - modified_content = re.sub(r'\n\s*\n', '\n', modified_content) - modified_content = re.sub(r',\s*\n\s*\]', '\n]', modified_content) + content = re.sub(r'\n\s*\n', '\n', content) + content = re.sub(r',\s*\n\s*\]', '\n]', content) - # Only write if content actually changed - if self._validate_changes(original_content, modified_content) and modified_content != original_content: - # Ensure MCU-specific backup exists before modification - if not self.backup_created_per_mcu.get(self.mcu, False): - self._backup_pioarduino_build_py() - + # Validate and write changes + if self._validate_changes(original_content, content) and content != original_content: with open(build_py_path, 'w') as f: - f.write(modified_content) + f.write(content) self._log_change(f"Updated build file ({total_removed} total removals)") - else: - self._log_change("No library changes needed") except Exception as e: self._log_change(f"Error processing libraries: {str(e)}") @@ -537,28 +528,16 @@ def _get_or_create_component_yml(self) -> str: self._create_default_component_yml(project_yml) return project_yml - def _create_backup(self, file_path: str) -> bool: + def _create_backup(self, file_path: str) -> None: """ - Create single backup of a file if it doesn't exist. + Create backup of a file. Args: file_path: Path to file to backup - - Returns: - True if backup was created or already exists """ backup_path = f"{file_path}.orig" - - if os.path.exists(file_path) and not os.path.exists(backup_path): - try: - shutil.copy2(file_path, backup_path) - self._log_change(f"Single backup created: {backup_path}") - return True - except Exception as e: - self._log_change(f"Backup failed: {str(e)}") - return False - - return os.path.exists(backup_path) + if not os.path.exists(backup_path): + shutil.copy(file_path, backup_path) def _create_default_component_yml(self, file_path: str) -> None: """ @@ -594,36 +573,17 @@ def _load_component_yml(self, file_path: str) -> Dict[str, Any]: def _save_component_yml(self, file_path: str, data: Dict[str, Any]) -> None: """ - Save component data to YAML file only if changed. + Save component data to YAML file. Args: file_path: Path to YAML file data: Data to save """ try: - # Check if content would actually change - if os.path.exists(file_path): - with open(file_path, "r") as f: - existing_data = yaml.load(f, Loader=SafeLoader) or {} - - # Compare data structures - if existing_data == data: - self._log_change(f"YAML unchanged, skipping write: {file_path}") - return - - # Create backup before first modification (only once) - if not self.yaml_backup_created: - self._create_backup(file_path) - self.yaml_backup_created = True - - # Write only if changed with open(file_path, "w") as f: - yaml.dump(data, f, default_flow_style=False, sort_keys=True) - - self._log_change(f"YAML updated: {file_path}") - - except Exception as e: - self._log_change(f"Error saving YAML: {str(e)}") + yaml.dump(data, f) + except Exception: + pass def _remove_components(self, component_data: Dict[str, Any], components_to_remove: list) -> None: """ @@ -700,61 +660,16 @@ def _convert_component_name_to_filesystem(self, component_name: str) -> str: """ return component_name.replace("/", "__") - def _backup_pioarduino_build_py(self) -> bool: - """Create MCU-specific backup of pioarduino-build.py only once per MCU.""" + def _backup_pioarduino_build_py(self) -> None: + """Create backup of the original pioarduino-build.py.""" if "arduino" not in self.env.subst("$PIOFRAMEWORK"): - return False - - # Check if backup already created for this MCU in this session - if self.backup_created_per_mcu.get(self.mcu, False): - return True + return build_py_path = join(self.arduino_libs_mcu, "pioarduino-build.py") backup_path = join(self.arduino_libs_mcu, f"pioarduino-build.py.{self.mcu}") - # Only create backup if source exists and MCU-specific backup doesn't exist if os.path.exists(build_py_path) and not os.path.exists(backup_path): - try: - shutil.copy2(build_py_path, backup_path) - self.backup_created_per_mcu[self.mcu] = True - self._log_change(f"Created MCU-specific backup for {self.mcu}: {backup_path}") - return True - except Exception as e: - self._log_change(f"MCU backup creation failed for {self.mcu}: {str(e)}") - return False - elif os.path.exists(backup_path): - self.backup_created_per_mcu[self.mcu] = True - self._log_change(f"MCU backup already exists for {self.mcu}") - return True - - return False - - def _validate_mcu_backup(self) -> bool: - """ - Validate that MCU-specific backup exists and is valid. - - Returns: - True if backup is valid - """ - backup_path = join(self.arduino_libs_mcu, f"pioarduino-build.py.{self.mcu}") - - if not os.path.exists(backup_path): - return False - - try: - # Basic validation - check if file is readable and not empty - with open(backup_path, 'r') as f: - content = f.read() - if len(content) < 100: # Minimum expected size - self._log_change(f"MCU backup for {self.mcu} appears corrupted") - return False - - self._log_change(f"MCU backup for {self.mcu} validated") - return True - - except Exception as e: - self._log_change(f"MCU backup validation failed for {self.mcu}: {str(e)}") - return False + shutil.copy2(build_py_path, backup_path) def _cleanup_removed_components(self) -> None: """Clean up removed components and restore original build file.""" @@ -808,7 +723,7 @@ def _remove_cpppath_entries(self) -> None: def restore_pioarduino_build_py(self, source=None, target=None, env=None) -> None: """ - Restore the MCU-specific pioarduino-build.py from backup. + Restore the original pioarduino-build.py from backup. Args: source: Build source (unused) @@ -819,35 +734,8 @@ def restore_pioarduino_build_py(self, source=None, target=None, env=None) -> Non backup_path = join(self.arduino_libs_mcu, f"pioarduino-build.py.{self.mcu}") if os.path.exists(backup_path): - try: - shutil.copy2(backup_path, build_py_path) - self._log_change(f"Restored MCU-specific build file for {self.mcu}") - - # Optional: Remove backup after restore if desired - # os.remove(backup_path) - # self._log_change(f"Removed MCU backup for {self.mcu}") - - except Exception as e: - self._log_change(f"Failed to restore MCU backup for {self.mcu}: {str(e)}") - else: - self._log_change(f"No MCU backup found for {self.mcu}") - - def get_mcu_backup_status(self) -> Dict[str, bool]: - """ - Get status of MCU-specific backups. - - Returns: - Dictionary with MCU types as keys and backup existence as values - """ - backup_dir = self.arduino_libs_mcu - mcu_types = ["esp32", "esp32s2", "esp32s3", "esp32c3", "esp32c6", "esp32h2"] - - status = {} - for mcu in mcu_types: - backup_path = join(backup_dir, f"pioarduino-build.py.{mcu}") - status[mcu] = os.path.exists(backup_path) - - return status + shutil.copy2(backup_path, build_py_path) + os.remove(backup_path) def get_changes_summary(self) -> List[str]: """ From bd947d88be498cd5acffcb3bae629112e59b5685 Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Fri, 30 May 2025 13:22:02 +0200 Subject: [PATCH 47/97] make PATH_LENGTH_THRESHOLD configurable --- builder/frameworks/arduino.py | 265 +++++++++++++++++++++++++++++++--- 1 file changed, 246 insertions(+), 19 deletions(-) diff --git a/builder/frameworks/arduino.py b/builder/frameworks/arduino.py index a6aca8a08..34cc1a1d1 100644 --- a/builder/frameworks/arduino.py +++ b/builder/frameworks/arduino.py @@ -43,11 +43,6 @@ IS_WINDOWS = sys.platform.startswith("win") -# Include path length threshold for path shortening, only valid and needed for Windows -# Windows has a path length limit of ~260 characters per path, but the total command line -# length is also limited to ~32000 characters -INCLUDE_PATH_LENGTH_THRESHOLD = 31500 # Total character count threshold - python_deps = { "wheel": ">=0.35.1", "rich-click": ">=1.8.6", @@ -90,6 +85,219 @@ def setup_logging(): 'long_path_warning_shown': False } +def get_platform_default_threshold(mcu): + """ + Platform-specific bleeding edge default values for INCLUDE_PATH_LENGTH_THRESHOLD + These values push the limits for maximum performance and minimal path shortening + + Args: + mcu: MCU type (esp32, esp32s2, esp32s3, etc.) + + Returns: + int: Platform-specific bleeding edge default threshold + """ + # Bleeding edge values - pushing Windows command line limits + # Windows CMD has ~32768 character limit, we use aggressive values close to this + platform_defaults = { + "esp32": 45000, # Standard ESP32 + "esp32s2": 43000, # ESP32-S2 + "esp32s3": 48000, # ESP32-S3 + "esp32c3": 41000, # ESP32-C3 + "esp32c2": 38000, # ESP32-C2 + "esp32c6": 44000, # ESP32-C6 + "esp32h2": 40000, # ESP32-H2 + "esp32p4": 50000, # ESP32-P4 + } + + default_value = platform_defaults.get(mcu, 45000) # Aggressive fallback + + # Debug output only in verbose mode + if logging.getLogger().isEnabledFor(logging.DEBUG): + logging.debug(f"Bleeding edge platform default threshold for {mcu}: {default_value}") + + return default_value + +def validate_threshold(threshold, mcu): + """ + Validates threshold value with bleeding edge limits + Uses aggressive boundaries for maximum performance + + Args: + threshold: Threshold value to validate + mcu: MCU type for context-specific validation + + Returns: + int: Validated threshold value + """ + # Bleeding edge absolute limits - pushing boundaries + min_threshold = 15000 # Minimum reasonable value for complex projects + max_threshold = 65000 # Maximum aggressive value (beyond Windows CMD limit for testing) + + # MCU-specific bleeding edge adjustments - all values are aggressive + mcu_adjustments = { + "esp32c2": {"min": 30000, "max": 40000}, + "esp32c3": {"min": 30000, "max": 45000}, + "esp32": {"min": 30000, "max": 50000}, + "esp32s2": {"min": 30000, "max": 50000}, + "esp32s3": {"min": 30000, "max": 50000}, + "esp32p4": {"min": 30000, "max": 55000}, + "esp32c6": {"min": 30000, "max": 50000}, + "esp32h2": {"min": 30000, "max": 40000}, + } + + # Apply MCU-specific bleeding edge limits + if mcu in mcu_adjustments: + min_threshold = max(min_threshold, mcu_adjustments[mcu]["min"]) + max_threshold = min(max_threshold, mcu_adjustments[mcu]["max"]) + + original_threshold = threshold + + if threshold < min_threshold: + print(f"*** Warning: Include path threshold {threshold} too conservative for {mcu}, using bleeding edge minimum {min_threshold} ***") + threshold = min_threshold + elif threshold > max_threshold: + print(f"*** Warning: Include path threshold {threshold} exceeds bleeding edge maximum for {mcu}, using {max_threshold} ***") + threshold = max_threshold + + # Warning for conservative values (opposite of original - warn if too low) + platform_default = get_platform_default_threshold(mcu) + if threshold < platform_default * 0.7: # More than 30% below bleeding edge default + print(f"*** Info: Include path threshold {threshold} is conservative compared to bleeding edge default {platform_default} for {mcu} ***") + print(f"*** Consider using higher values for maximum performance ***") + + if original_threshold != threshold: + logging.warning(f"Threshold adjusted from {original_threshold} to bleeding edge value {threshold} for {mcu}") + + return threshold + +def get_include_path_threshold(env, config, current_env_section): + """ + Determines Windows INCLUDE_PATH_LENGTH_THRESHOLD from various sources + with priority order and bleeding edge validation + + Priority order: + 1. Environment variable PLATFORMIO_INCLUDE_PATH_THRESHOLD + 2. Environment-specific setting in platformio.ini + 3. Global setting in [env] section + 4. Setting in [platformio] section + 5. MCU-specific bleeding edge default value + + Args: + env: PlatformIO Environment + config: Project Configuration + current_env_section: Current environment section + + Returns: + int: Validated bleeding edge threshold value + """ + mcu = env.BoardConfig().get("build.mcu", "esp32") + default_threshold = get_platform_default_threshold(mcu) + setting_name = "custom_include_path_length_threshold" + + try: + # 1. Check environment variable (highest priority) + env_var = os.environ.get("PLATFORMIO_INCLUDE_PATH_THRESHOLD") + if env_var: + try: + threshold = int(env_var) + threshold = validate_threshold(threshold, mcu) + print(f"*** Using environment variable bleeding edge include path threshold: {threshold} (MCU: {mcu}) ***") + return threshold + except ValueError: + print(f"*** Warning: Invalid environment variable PLATFORMIO_INCLUDE_PATH_THRESHOLD='{env_var}', ignoring ***") + + # 2. Check environment-specific setting + if config.has_option(current_env_section, setting_name): + threshold = config.getint(current_env_section, setting_name) + threshold = validate_threshold(threshold, mcu) + print(f"*** Using environment-specific bleeding edge include path threshold: {threshold} (MCU: {mcu}) ***") + return threshold + + # 3. Check global setting in [env] section + if config.has_option("env", setting_name): + threshold = config.getint("env", setting_name) + threshold = validate_threshold(threshold, mcu) + print(f"*** Using global [env] bleeding edge include path threshold: {threshold} (MCU: {mcu}) ***") + return threshold + + # 4. Check setting in [platformio] section + if config.has_option("platformio", setting_name): + threshold = config.getint("platformio", setting_name) + threshold = validate_threshold(threshold, mcu) + print(f"*** Using [platformio] section bleeding edge include path threshold: {threshold} (MCU: {mcu}) ***") + return threshold + + # 5. Use MCU-specific bleeding edge default value + threshold = validate_threshold(default_threshold, mcu) + if env.get("VERBOSE"): + print(f"*** Using platform-specific bleeding edge default include path threshold: {threshold} (MCU: {mcu}) ***") + + return threshold + + except (ValueError, TypeError) as e: + print(f"*** Warning: Invalid include path threshold value, using bleeding edge platform default {default_threshold} for {mcu}: {e} ***") + return validate_threshold(default_threshold, mcu) + +def get_threshold_info(env, config, current_env_section): + """ + Helper function for debug information about bleeding edge threshold configuration + + Args: + env: PlatformIO Environment + config: Project Configuration + current_env_section: Current environment section + + Returns: + dict: Information about threshold configuration + """ + mcu = env.BoardConfig().get("build.mcu", "esp32") + setting_name = "custom_include_path_length_threshold" + + info = { + "mcu": mcu, + "platform_default": get_platform_default_threshold(mcu), + "env_variable": os.environ.get("PLATFORMIO_INCLUDE_PATH_THRESHOLD"), + "env_specific": None, + "global_env": None, + "platformio_section": None, + "final_threshold": None, + "source": "bleeding_edge_platform_default", + "is_bleeding_edge": True + } + + # Collect all possible sources + if config.has_option(current_env_section, setting_name): + try: + info["env_specific"] = config.getint(current_env_section, setting_name) + except ValueError: + pass + + if config.has_option("env", setting_name): + try: + info["global_env"] = config.getint("env", setting_name) + except ValueError: + pass + + if config.has_option("platformio", setting_name): + try: + info["platformio_section"] = config.getint("platformio", setting_name) + except ValueError: + pass + + # Determine final threshold and source + info["final_threshold"] = get_include_path_threshold(env, config, current_env_section) + + # Determine source + if info["env_variable"]: + info["source"] = "environment_variable" + elif info["env_specific"] is not None: + info["source"] = "env_specific" + elif info["global_env"] is not None: + info["source"] = "global_env" + elif info["platformio_section"] is not None: + info["source"] = "platformio_section" + + return info # Cache class for frequently used paths class PathCache: @@ -545,7 +753,7 @@ def debug_framework_paths(env, include_count, total_length): print(f"*** FRAMEWORK_SDK_DIR: {FRAMEWORK_SDK_DIR} ***") print(f"*** SDK exists: {exists(FRAMEWORK_SDK_DIR)} ***") print(f"*** Include count: {include_count} ***") - print(f"*** Total path length: {total_length} (threshold: {INCLUDE_PATH_LENGTH_THRESHOLD}) ***") + print(f"*** Total path length: {total_length} ***") includes = env.get("CPPPATH", []) framework_count = 0 @@ -619,31 +827,50 @@ def apply_include_shortening(env, node, includes, total_length): ) def smart_include_length_shorten(env, node): - """Include path shortening based on total path length threshold""" + """ + Include path shortening based on bleeding edge configurable threshold with enhanced MCU support + Uses aggressive thresholds for maximum performance + """ if IS_INTEGRATION_DUMP: - # Don't shorten include paths for IDE integrations return node if not IS_WINDOWS: return env.Object(node) - # Check long path support once + # Get dynamically configurable bleeding edge threshold + include_path_threshold = get_include_path_threshold(env, config, current_env_section) + check_and_warn_long_path_support() includes = env.get("CPPPATH", []) include_count = len(includes) total_path_length = calculate_include_path_length(includes) - # Debug output in verbose mode - debug_framework_paths(env, include_count, total_path_length) - - # Apply shortening only if total path length exceeds threshold - # This is more accurate than just counting includes, as it considers - # the actual command line length impact - if total_path_length <= INCLUDE_PATH_LENGTH_THRESHOLD: - return env.Object(node) # Normal compilation + # Debug information in verbose mode + if env.get("VERBOSE"): + debug_framework_paths(env, include_count, total_path_length) + + # Extended debug information about bleeding edge threshold configuration + threshold_info = get_threshold_info(env, config, current_env_section) + print(f"*** Bleeding Edge Threshold Configuration Debug ***") + print(f"*** MCU: {threshold_info['mcu']} ***") + print(f"*** Bleeding Edge Platform Default: {threshold_info['platform_default']} ***") + print(f"*** Final Bleeding Edge Threshold: {threshold_info['final_threshold']} ***") + print(f"*** Source: {threshold_info['source']} ***") + print(f"*** Performance Mode: Maximum Aggressive ***") + if threshold_info['env_variable']: + print(f"*** Env Variable: {threshold_info['env_variable']} ***") + if threshold_info['env_specific']: + print(f"*** Env Specific: {threshold_info['env_specific']} ***") + if threshold_info['global_env']: + print(f"*** Global Env: {threshold_info['global_env']} ***") + if threshold_info['platformio_section']: + print(f"*** PlatformIO Section: {threshold_info['platformio_section']} ***") + + # Use the configurable and validated bleeding edge threshold + if total_path_length <= include_path_threshold: + return env.Object(node) - # Apply include path shortening return apply_include_shortening(env, node, includes, total_path_length) def get_frameworks_in_current_env(): @@ -697,7 +924,7 @@ def get_frameworks_in_current_env(): env.AddPostAction("checkprogsize", silent_action) if IS_WINDOWS: - # Smart include path optimization based on total path length + # Smart include path optimization based on bleeding edge configurable threshold env.AddBuildMiddleware(smart_include_length_shorten) build_script_path = join(FRAMEWORK_DIR, "tools", "pioarduino-build.py") From 493f2983af3fbe9a814e14fcd1b13ec38fb39da3 Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Fri, 30 May 2025 14:19:54 +0200 Subject: [PATCH 48/97] fix dfu-util install --- platform.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platform.py b/platform.py index 22abd39de..3c0ee7b9e 100644 --- a/platform.py +++ b/platform.py @@ -424,7 +424,7 @@ def _install_filesystem_tool(self, filesystem: str, for_download: bool = False) handler() # Currently only Arduino Nano ESP32 uses the dfuutil tool as uploader - def _handle_dfuutil_tool(self, for_download: bool) -> None: + def _handle_dfuutil_tool(self, variables: Dict, for_download: bool = False) -> None: """Install dfuutil tool for Arduino Nano Board""" if variables.get("board") == "arduino_nano_esp32": self.install_tool("tool-dfuutil-arduino") From da5225da854fe5f5be7ec00bbf5c39f87ad08cf6 Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Fri, 30 May 2025 14:23:32 +0200 Subject: [PATCH 49/97] move import sys --- builder/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/builder/main.py b/builder/main.py index 8a21d8334..07f15ff3e 100644 --- a/builder/main.py +++ b/builder/main.py @@ -14,6 +14,7 @@ import os import re +import sys import locale from os.path import isfile, join @@ -380,7 +381,6 @@ def firmware_metrics(target, source, env): try: import subprocess - import sys import shlex cmd = [env.subst("$PYTHONEXE"), "-m", "esp_idf_size", "--ng"] From 1b4c460ce44df9e8c7a2f06ec63279210afa72d5 Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Fri, 30 May 2025 14:40:50 +0200 Subject: [PATCH 50/97] create function create_silent_action() --- builder/frameworks/espidf.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/builder/frameworks/espidf.py b/builder/frameworks/espidf.py index 3ec070e96..22352baf8 100644 --- a/builder/frameworks/espidf.py +++ b/builder/frameworks/espidf.py @@ -153,6 +153,11 @@ def _get_installed_standard_pip_packages(): assert os.path.isdir(FRAMEWORK_DIR) assert os.path.isdir(TOOLCHAIN_DIR) +def create_silent_action(action_func): + """Create a silent SCons action that suppresses output""" + silent_action = env.Action(action_func) + silent_action.strfunction = lambda target, source, env: '' + return silent_action if "arduino" in env.subst("$PIOFRAMEWORK"): ARDUINO_FRAMEWORK_DIR = platform.get_package_dir("framework-arduinoespressif32") @@ -2196,8 +2201,7 @@ def idf_lib_copy(source, target, env): from component_manager import ComponentManager component_manager = ComponentManager(env) component_manager.restore_pioarduino_build_py() - silent_action = env.Action(idf_lib_copy) - silent_action.strfunction = lambda target, source, env: '' # hack to silence scons command output + silent_action = create_silent_action(idf_lib_copy) env.AddPostAction("checkprogsize", silent_action) if "espidf" in env.subst("$PIOFRAMEWORK") and (flag_custom_component_add == True or flag_custom_component_remove == True): @@ -2220,8 +2224,7 @@ def idf_custom_component(source, target, env): from component_manager import ComponentManager component_manager = ComponentManager(env) component_manager.restore_pioarduino_build_py() - silent_action = env.Action(idf_custom_component) - silent_action.strfunction = lambda target, source, env: '' # hack to silence scons command output + silent_action = create_silent_action(idf_custom_component) env.AddPostAction("checkprogsize", silent_action) # # Process OTA partition and image From 5adc2c540ea7abfb925777a8a3e2600c19224cef Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Fri, 30 May 2025 16:24:56 +0200 Subject: [PATCH 51/97] more compatible --- builder/frameworks/component_manager.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/builder/frameworks/component_manager.py b/builder/frameworks/component_manager.py index d6c248a6f..2ba3e881e 100644 --- a/builder/frameworks/component_manager.py +++ b/builder/frameworks/component_manager.py @@ -5,7 +5,7 @@ import yaml from yaml import SafeLoader from os.path import join -from typing import Set, Optional, Dict, Any, List +from typing import Set, Optional, Dict, Any, List, Tuple class ComponentManager: @@ -633,7 +633,7 @@ def _add_components(self, component_data: Dict[str, Any], components_to_add: lis else: self._log_change(f"Component already exists: {component_name}") - def _parse_component_entry(self, entry: str) -> tuple[str, str]: + def _parse_component_entry(self, entry: str) -> Tuple[str, str]: """ Parse component entry into name and version. @@ -721,13 +721,13 @@ def _remove_cpppath_entries(self) -> None: except Exception: pass - def restore_pioarduino_build_py(self, source=None, target=None, env=None) -> None: + def restore_pioarduino_build_py(self, target=None, source=None, env=None) -> None: """ Restore the original pioarduino-build.py from backup. Args: + target: Build target (unused) source: Build source (unused) - target: Build target (unused) env: Environment (unused) """ build_py_path = join(self.arduino_libs_mcu, "pioarduino-build.py") From bb67bf8be6c49b6fd3ff0d1281a394f6a5b15cd5 Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Fri, 30 May 2025 16:36:38 +0200 Subject: [PATCH 52/97] minor changes --- builder/frameworks/arduino.py | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/builder/frameworks/arduino.py b/builder/frameworks/arduino.py index 34cc1a1d1..ee2dadc5c 100644 --- a/builder/frameworks/arduino.py +++ b/builder/frameworks/arduino.py @@ -163,7 +163,7 @@ def validate_threshold(threshold, mcu): platform_default = get_platform_default_threshold(mcu) if threshold < platform_default * 0.7: # More than 30% below bleeding edge default print(f"*** Info: Include path threshold {threshold} is conservative compared to bleeding edge default {platform_default} for {mcu} ***") - print(f"*** Consider using higher values for maximum performance ***") + print("*** Consider using higher values for maximum performance ***") if original_threshold != threshold: logging.warning(f"Threshold adjusted from {original_threshold} to bleeding edge value {threshold} for {mcu}") @@ -267,22 +267,16 @@ def get_threshold_info(env, config, current_env_section): # Collect all possible sources if config.has_option(current_env_section, setting_name): - try: + with suppress(ValueError): info["env_specific"] = config.getint(current_env_section, setting_name) - except ValueError: - pass if config.has_option("env", setting_name): - try: + with suppress(ValueError): info["global_env"] = config.getint("env", setting_name) - except ValueError: - pass - + if config.has_option("platformio", setting_name): - try: + with suppress(ValueError): info["platformio_section"] = config.getint("platformio", setting_name) - except ValueError: - pass # Determine final threshold and source info["final_threshold"] = get_include_path_threshold(env, config, current_env_section) @@ -805,7 +799,8 @@ def apply_include_shortening(env, node, includes, total_length): with _PATH_SHORTENING_LOCK: if not _PATH_SHORTENING_MESSAGES['shortening_applied']: if shortened_includes: - new_total_length = original_length - saved_chars + len(f"-iprefix{FRAMEWORK_SDK_DIR}") + removed_i_flags = len(shortened_includes) * 2 # Each -I is 2 chars + new_total_length = original_length - saved_chars + len(f"-iprefix{FRAMEWORK_SDK_DIR}") - removed_i_flags print(f"*** Applied include path shortening for {len(shortened_includes)} framework paths ***") print(f"*** Path length reduced from {original_length} to ~{new_total_length} characters ***") print(f"*** Estimated savings: {saved_chars} characters ***") @@ -852,12 +847,12 @@ def smart_include_length_shorten(env, node): # Extended debug information about bleeding edge threshold configuration threshold_info = get_threshold_info(env, config, current_env_section) - print(f"*** Bleeding Edge Threshold Configuration Debug ***") + print("*** Bleeding Edge Threshold Configuration Debug ***") print(f"*** MCU: {threshold_info['mcu']} ***") print(f"*** Bleeding Edge Platform Default: {threshold_info['platform_default']} ***") print(f"*** Final Bleeding Edge Threshold: {threshold_info['final_threshold']} ***") print(f"*** Source: {threshold_info['source']} ***") - print(f"*** Performance Mode: Maximum Aggressive ***") + print("*** Performance Mode: Maximum Aggressive ***") if threshold_info['env_variable']: print(f"*** Env Variable: {threshold_info['env_variable']} ***") if threshold_info['env_specific']: From 3c7e002c71660415b9bb3c33d2c797e60532fa83 Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Fri, 30 May 2025 18:59:57 +0200 Subject: [PATCH 53/97] normalize path -> case-insensitive comparison on Windows --- builder/frameworks/arduino.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/builder/frameworks/arduino.py b/builder/frameworks/arduino.py index ee2dadc5c..4b9693262 100644 --- a/builder/frameworks/arduino.py +++ b/builder/frameworks/arduino.py @@ -469,7 +469,9 @@ def validate_deletion_path(path: Union[str, Path], for critical in filter(None, critical_paths): try: - if path == critical or critical in path.parents: + normalized_path = path.resolve() + normalized_critical = critical.resolve() + if normalized_path == normalized_critical or normalized_critical in normalized_path.parents: logging.error(f"Critical system path detected: {path}") return False except (OSError, ValueError): From 95f9c2e5c920823a1f863c6fc8f6d94e7cc4b2d4 Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Fri, 30 May 2025 19:04:00 +0200 Subject: [PATCH 54/97] rm unused class ToolInstallationError() --- platform.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/platform.py b/platform.py index 3c0ee7b9e..cb77fe405 100644 --- a/platform.py +++ b/platform.py @@ -81,11 +81,6 @@ logger = logging.getLogger(__name__) -class ToolInstallationError(Exception): - """Custom exception for tool installation errors""" - pass - - def safe_file_operation(operation_func): """Decorator for safe filesystem operations""" def wrapper(*args, **kwargs): From 2f8fbcbd22d05f2c7b9e9c9b6ce5c50366066b30 Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Fri, 30 May 2025 19:25:03 +0200 Subject: [PATCH 55/97] correct check for not existing env --- builder/frameworks/arduino.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/builder/frameworks/arduino.py b/builder/frameworks/arduino.py index 4b9693262..e37b34d44 100644 --- a/builder/frameworks/arduino.py +++ b/builder/frameworks/arduino.py @@ -600,8 +600,9 @@ def has_unicore_flags(): # Esp32-solo1 libs settings if flag_custom_sdkconfig and has_unicore_flags(): - if len(str(env.GetProjectOption("build_unflags"))) == 2: # No valid env, needs init - env['BUILD_UNFLAGS'] = {} + build_unflags = env.GetProjectOption("build_unflags") + if not build_unflags: # not existing needs init + env['BUILD_UNFLAGS'] = [] build_unflags = " ".join(env['BUILD_UNFLAGS']) + " -mdisable-hardware-atomics -ustart_app_other_cores" new_build_unflags = build_unflags.split() From 898ffbe3796dfe76b477cbddd6b27a3ba688f0ba Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Fri, 30 May 2025 20:23:50 +0200 Subject: [PATCH 56/97] revert refactor for solo1 detection --- builder/frameworks/arduino.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/builder/frameworks/arduino.py b/builder/frameworks/arduino.py index e37b34d44..5cdbffad9 100644 --- a/builder/frameworks/arduino.py +++ b/builder/frameworks/arduino.py @@ -427,8 +427,7 @@ def validate_platformio_path(path: Union[str, Path]) -> bool: # Must be framework-related framework_indicators = [ "framework-arduinoespressif32", - "framework-arduinoespressif32-libs", - "framework-arduino-c2-skeleton-lib" + "framework-arduinoespressif32-libs" ] if not any(indicator in path_str for indicator in framework_indicators): @@ -578,11 +577,7 @@ def safe_remove_sdkconfig_files(): if len(board_sdkconfig) > 2: flag_custom_sdkconfig = True -extra_flags_raw = board.get("build.extra_flags", []) -if isinstance(extra_flags_raw, list): - extra_flags = " ".join(extra_flags_raw).replace("-D", " ") -else: - extra_flags = str(extra_flags_raw).replace("-D", " ") +extra_flags = (''.join([element for element in board.get("build.extra_flags", "")])).replace("-D", " ") framework_reinstall = False From 5d46279fb0d2a74db1d4f131208649385b512993 Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Fri, 30 May 2025 20:34:06 +0200 Subject: [PATCH 57/97] revert defect refactored code --- builder/frameworks/arduino.py | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/builder/frameworks/arduino.py b/builder/frameworks/arduino.py index 5cdbffad9..a4154687f 100644 --- a/builder/frameworks/arduino.py +++ b/builder/frameworks/arduino.py @@ -71,12 +71,6 @@ def setup_logging(): if os.environ.get('ARDUINO_FRAMEWORK_ENABLE_LOGGING'): setup_logging() -# Constants for better performance -UNICORE_FLAGS = { - "CORE32SOLO1", - "CONFIG_FREERTOS_UNICORE=y" -} - # Thread-safe global flags to prevent message spam _PATH_SHORTENING_LOCK = threading.Lock() _PATH_SHORTENING_MESSAGES = { @@ -588,13 +582,8 @@ def safe_remove_sdkconfig_files(): flag_any_custom_sdkconfig = exists(join(platform.get_package_dir("framework-arduinoespressif32-libs"),"sdkconfig")) -def has_unicore_flags(): - """Check if any UNICORE flags are present in configuration""" - return any(flag in extra_flags or flag in entry_custom_sdkconfig - or flag in board_sdkconfig for flag in UNICORE_FLAGS) - # Esp32-solo1 libs settings -if flag_custom_sdkconfig and has_unicore_flags(): +if flag_custom_sdkconfig == True and ("CORE32SOLO1" in extra_flags or "CONFIG_FREERTOS_UNICORE=y" in entry_custom_sdkconfig or "CONFIG_FREERTOS_UNICORE=y" in board_sdkconfig): build_unflags = env.GetProjectOption("build_unflags") if not build_unflags: # not existing needs init env['BUILD_UNFLAGS'] = [] From 41d239c3ce85978774f5b3d930ee980dd7f2afb3 Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Fri, 30 May 2025 20:58:07 +0200 Subject: [PATCH 58/97] fix wrong AI refactoring --- builder/frameworks/arduino.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/builder/frameworks/arduino.py b/builder/frameworks/arduino.py index a4154687f..f89f18756 100644 --- a/builder/frameworks/arduino.py +++ b/builder/frameworks/arduino.py @@ -542,7 +542,8 @@ def safe_remove_sdkconfig_files(): pioenv = env["PIOENV"] project_dir = env.subst("$PROJECT_DIR") path_cache = PathCache(platform, mcu) -current_env_section = f"env:{pioenv}" +current_env_section = "env:"+env["PIOENV"] + # Board configuration board_sdkconfig = board.get("espidf.custom_sdkconfig", "") From c72f48b00d238ec9cc0d5697ea627225e4b5737a Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Fri, 30 May 2025 21:17:15 +0200 Subject: [PATCH 59/97] Update arduino.py --- builder/frameworks/arduino.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/builder/frameworks/arduino.py b/builder/frameworks/arduino.py index f89f18756..90bda0aa3 100644 --- a/builder/frameworks/arduino.py +++ b/builder/frameworks/arduino.py @@ -535,6 +535,7 @@ def safe_remove_sdkconfig_files(): pm = ToolPackageManager() platform = env.PioPlatform() config = env.GetProjectConfig() +print("******* config", config) board = env.BoardConfig() # Cached values @@ -567,6 +568,7 @@ def safe_remove_sdkconfig_files(): # Custom SDKConfig check if config.has_option(current_env_section, "custom_sdkconfig"): entry_custom_sdkconfig = env.GetProjectOption("custom_sdkconfig") + print("entry custom_sdkconfig", entry_custom_sdkconfig) flag_custom_sdkconfig = True if len(board_sdkconfig) > 2: From 6177cc19e4f35f8f065af7638c19eb70cae881c3 Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Fri, 30 May 2025 21:21:50 +0200 Subject: [PATCH 60/97] Update arduino.py --- builder/frameworks/arduino.py | 1 + 1 file changed, 1 insertion(+) diff --git a/builder/frameworks/arduino.py b/builder/frameworks/arduino.py index 90bda0aa3..a12a3c9a4 100644 --- a/builder/frameworks/arduino.py +++ b/builder/frameworks/arduino.py @@ -39,6 +39,7 @@ from SCons.Script import DefaultEnvironment, SConscript from platformio import fs from platformio.package.version import pepver_to_semver +from platformio.project.config import ProjectConfig from platformio.package.manager.tool import ToolPackageManager IS_WINDOWS = sys.platform.startswith("win") From 523f970abf1af76c4428bc3b4876be05b075a0d2 Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Fri, 30 May 2025 21:29:29 +0200 Subject: [PATCH 61/97] Update arduino.py --- builder/frameworks/arduino.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/builder/frameworks/arduino.py b/builder/frameworks/arduino.py index a12a3c9a4..7b9fafe3a 100644 --- a/builder/frameworks/arduino.py +++ b/builder/frameworks/arduino.py @@ -566,6 +566,10 @@ def safe_remove_sdkconfig_files(): if config.has_option(current_env_section, "custom_component_remove"): flag_custom_component_remove = True +if config.has_option("env:"+env["PIOENV"], "custom_sdkconfig"): + entry_custom_sdkconfig = env.GetProjectOption("custom_sdkconfig") + flag_custom_sdkconfig = True + # Custom SDKConfig check if config.has_option(current_env_section, "custom_sdkconfig"): entry_custom_sdkconfig = env.GetProjectOption("custom_sdkconfig") From a9744d67e137982f7608cf8f69d887902c57365a Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Fri, 30 May 2025 21:41:24 +0200 Subject: [PATCH 62/97] Update arduino.py --- builder/frameworks/arduino.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/builder/frameworks/arduino.py b/builder/frameworks/arduino.py index 7b9fafe3a..abd7fc687 100644 --- a/builder/frameworks/arduino.py +++ b/builder/frameworks/arduino.py @@ -536,19 +536,21 @@ def safe_remove_sdkconfig_files(): pm = ToolPackageManager() platform = env.PioPlatform() config = env.GetProjectConfig() -print("******* config", config) board = env.BoardConfig() +print("******* board", board) + # Cached values mcu = board.get("build.mcu", "esp32") pioenv = env["PIOENV"] project_dir = env.subst("$PROJECT_DIR") path_cache = PathCache(platform, mcu) -current_env_section = "env:"+env["PIOENV"] +current_env_section = "env:"+pioenv # Board configuration board_sdkconfig = board.get("espidf.custom_sdkconfig", "") +print("****** board sdkconfig:", board_sdkconfig) entry_custom_sdkconfig = "\n" flag_custom_sdkconfig = False flag_custom_component_remove = False @@ -567,6 +569,7 @@ def safe_remove_sdkconfig_files(): flag_custom_component_remove = True if config.has_option("env:"+env["PIOENV"], "custom_sdkconfig"): + print("entry custom_sdkconfig", entry_custom_sdkconfig) entry_custom_sdkconfig = env.GetProjectOption("custom_sdkconfig") flag_custom_sdkconfig = True From fcff03a78cdfd88f399981fc1ad5d248fe52143e Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Fri, 30 May 2025 21:50:16 +0200 Subject: [PATCH 63/97] if len(str(board_sdkconfig)) > 2: --- builder/frameworks/arduino.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/builder/frameworks/arduino.py b/builder/frameworks/arduino.py index abd7fc687..cd849f43f 100644 --- a/builder/frameworks/arduino.py +++ b/builder/frameworks/arduino.py @@ -579,7 +579,7 @@ def safe_remove_sdkconfig_files(): print("entry custom_sdkconfig", entry_custom_sdkconfig) flag_custom_sdkconfig = True -if len(board_sdkconfig) > 2: +if len(str(board_sdkconfig)) > 2: flag_custom_sdkconfig = True extra_flags = (''.join([element for element in board.get("build.extra_flags", "")])).replace("-D", " ") From d225def33c6dead8bda915f46523f6c06933b513 Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Fri, 30 May 2025 22:11:07 +0200 Subject: [PATCH 64/97] Update arduino.py --- builder/frameworks/arduino.py | 35 ++++++++++++++++++++--------------- 1 file changed, 20 insertions(+), 15 deletions(-) diff --git a/builder/frameworks/arduino.py b/builder/frameworks/arduino.py index cd849f43f..14029dbf8 100644 --- a/builder/frameworks/arduino.py +++ b/builder/frameworks/arduino.py @@ -39,7 +39,6 @@ from SCons.Script import DefaultEnvironment, SConscript from platformio import fs from platformio.package.version import pepver_to_semver -from platformio.project.config import ProjectConfig from platformio.package.manager.tool import ToolPackageManager IS_WINDOWS = sys.platform.startswith("win") @@ -72,6 +71,12 @@ def setup_logging(): if os.environ.get('ARDUINO_FRAMEWORK_ENABLE_LOGGING'): setup_logging() +# Constants for better performance +UNICORE_FLAGS = { + "CORE32SOLO1", + "CONFIG_FREERTOS_UNICORE=y" +} + # Thread-safe global flags to prevent message spam _PATH_SHORTENING_LOCK = threading.Lock() _PATH_SHORTENING_MESSAGES = { @@ -537,20 +542,16 @@ def safe_remove_sdkconfig_files(): platform = env.PioPlatform() config = env.GetProjectConfig() board = env.BoardConfig() -print("******* board", board) - # Cached values mcu = board.get("build.mcu", "esp32") pioenv = env["PIOENV"] project_dir = env.subst("$PROJECT_DIR") path_cache = PathCache(platform, mcu) -current_env_section = "env:"+pioenv - +current_env_section = f"env:{pioenv}" # Board configuration board_sdkconfig = board.get("espidf.custom_sdkconfig", "") -print("****** board sdkconfig:", board_sdkconfig) entry_custom_sdkconfig = "\n" flag_custom_sdkconfig = False flag_custom_component_remove = False @@ -568,21 +569,20 @@ def safe_remove_sdkconfig_files(): if config.has_option(current_env_section, "custom_component_remove"): flag_custom_component_remove = True -if config.has_option("env:"+env["PIOENV"], "custom_sdkconfig"): - print("entry custom_sdkconfig", entry_custom_sdkconfig) - entry_custom_sdkconfig = env.GetProjectOption("custom_sdkconfig") - flag_custom_sdkconfig = True - # Custom SDKConfig check if config.has_option(current_env_section, "custom_sdkconfig"): entry_custom_sdkconfig = env.GetProjectOption("custom_sdkconfig") - print("entry custom_sdkconfig", entry_custom_sdkconfig) flag_custom_sdkconfig = True -if len(str(board_sdkconfig)) > 2: +if board_sdkconfig: + print(f"board_sdkconfig: {board_sdkconfig}") flag_custom_sdkconfig = True -extra_flags = (''.join([element for element in board.get("build.extra_flags", "")])).replace("-D", " ") +extra_flags_raw = board.get("build.extra_flags", []) +if isinstance(extra_flags_raw, list): + extra_flags = " ".join(extra_flags_raw).replace("-D", " ") +else: + extra_flags = str(extra_flags_raw).replace("-D", " ") framework_reinstall = False @@ -593,8 +593,13 @@ def safe_remove_sdkconfig_files(): flag_any_custom_sdkconfig = exists(join(platform.get_package_dir("framework-arduinoespressif32-libs"),"sdkconfig")) +def has_unicore_flags(): + """Check if any UNICORE flags are present in configuration""" + return any(flag in extra_flags or flag in entry_custom_sdkconfig + or flag in board_sdkconfig for flag in UNICORE_FLAGS) + # Esp32-solo1 libs settings -if flag_custom_sdkconfig == True and ("CORE32SOLO1" in extra_flags or "CONFIG_FREERTOS_UNICORE=y" in entry_custom_sdkconfig or "CONFIG_FREERTOS_UNICORE=y" in board_sdkconfig): +if flag_custom_sdkconfig and has_unicore_flags(): build_unflags = env.GetProjectOption("build_unflags") if not build_unflags: # not existing needs init env['BUILD_UNFLAGS'] = [] From f40322e93d0687ed15e9621df122c1081fef58e0 Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Fri, 30 May 2025 23:02:23 +0200 Subject: [PATCH 65/97] fix install dfu-util --- platform.py | 1 + 1 file changed, 1 insertion(+) diff --git a/platform.py b/platform.py index cb77fe405..04ce9517a 100644 --- a/platform.py +++ b/platform.py @@ -456,6 +456,7 @@ def configure_default_packages(self, variables: Dict, targets: List[str]) -> Any self._configure_check_tools(variables) self._configure_filesystem_tools(variables, targets) + self._handle_dfuutil_tool(variables) logger.info("Package configuration completed successfully") From 99dbcfac955dccc20858b46fa62f0153a6e59d7c Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Fri, 30 May 2025 23:05:59 +0200 Subject: [PATCH 66/97] remove debug print --- builder/frameworks/arduino.py | 1 - 1 file changed, 1 deletion(-) diff --git a/builder/frameworks/arduino.py b/builder/frameworks/arduino.py index 14029dbf8..40078b705 100644 --- a/builder/frameworks/arduino.py +++ b/builder/frameworks/arduino.py @@ -575,7 +575,6 @@ def safe_remove_sdkconfig_files(): flag_custom_sdkconfig = True if board_sdkconfig: - print(f"board_sdkconfig: {board_sdkconfig}") flag_custom_sdkconfig = True extra_flags_raw = board.get("build.extra_flags", []) From 5463beea82d8286d7981123d2e92211b43904f84 Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Sat, 31 May 2025 12:03:30 +0200 Subject: [PATCH 67/97] add timeout when request for framework lib fails --- platform.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/platform.py b/platform.py index 04ce9517a..aa8baeb2a 100644 --- a/platform.py +++ b/platform.py @@ -167,7 +167,8 @@ def _run_idf_tools_install(self, tools_json_path: str, idf_tools_path: str) -> b cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, - timeout=SUBPROCESS_TIMEOUT + timeout=SUBPROCESS_TIMEOUT, + check=False ) if result.returncode != 0: @@ -277,9 +278,14 @@ def _configure_arduino_framework(self, frameworks: List[str]) -> None: self.packages["framework-arduinoespressif32-libs"]["optional"] = False # use branch master URL = "https://raw.githubusercontent.com/espressif/arduino-esp32/master/package/package_esp32_index.template.json" - packjdata = requests.get(URL).json() - dyn_lib_url = packjdata['packages'][0]['tools'][0]['systems'][0]['url'] - self.packages["framework-arduinoespressif32-libs"]["version"] = dyn_lib_url + try: + response = requests.get(URL, timeout=30) + response.raise_for_status() + packjdata = response.json() + dyn_lib_url = packjdata['packages'][0]['tools'][0]['systems'][0]['url'] + self.packages["framework-arduinoespressif32-libs"]["version"] = dyn_lib_url + except (requests.RequestException, KeyError, IndexError) as e: + logger.error(f"Failed to fetch Arduino framework library URL: {e}") def _configure_espidf_framework(self, frameworks: List[str], variables: Dict, board_config: Dict, mcu: str) -> None: """Configure ESP-IDF framework""" From a099f57f2e1a46e7e68c7cccbf40e3ca828d502b Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Sat, 31 May 2025 12:10:37 +0200 Subject: [PATCH 68/97] file operation with UTF-8 --- builder/frameworks/component_manager.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/builder/frameworks/component_manager.py b/builder/frameworks/component_manager.py index 2ba3e881e..4f05bf4af 100644 --- a/builder/frameworks/component_manager.py +++ b/builder/frameworks/component_manager.py @@ -433,7 +433,7 @@ def _remove_ignored_lib_includes(self) -> None: self._log_change("BT/BLE protection enabled") try: - with open(build_py_path, 'r') as f: + with open(build_py_path, 'r', encoding='utf-8') as f: content = f.read() original_content = content @@ -480,7 +480,7 @@ def _remove_ignored_lib_includes(self) -> None: # Validate and write changes if self._validate_changes(original_content, content) and content != original_content: - with open(build_py_path, 'w') as f: + with open(build_py_path, 'w', encoding='utf-8') as f: f.write(content) self._log_change(f"Updated build file ({total_removed} total removals)") @@ -552,7 +552,7 @@ def _create_default_component_yml(self, file_path: str) -> None: } } - with open(file_path, 'w') as f: + with open(file_path, 'w', encoding='utf-8') as f: yaml.dump(default_content, f) def _load_component_yml(self, file_path: str) -> Dict[str, Any]: @@ -566,7 +566,7 @@ def _load_component_yml(self, file_path: str) -> Dict[str, Any]: Parsed YAML data """ try: - with open(file_path, "r") as f: + with open(file_path, "r", encoding='utf-8') as f: return yaml.load(f, Loader=SafeLoader) or {"dependencies": {}} except Exception: return {"dependencies": {}} @@ -580,7 +580,7 @@ def _save_component_yml(self, file_path: str, data: Dict[str, Any]) -> None: data: Data to save """ try: - with open(file_path, "w") as f: + with open(file_path, "w", encoding='utf-8') as f: yaml.dump(data, f) except Exception: pass @@ -698,7 +698,7 @@ def _remove_cpppath_entries(self) -> None: return try: - with open(build_py_path, 'r') as f: + with open(build_py_path, 'r', encoding='utf-8') as f: content = f.read() original_content = content @@ -715,7 +715,7 @@ def _remove_cpppath_entries(self) -> None: content = re.sub(pattern, '', content) if content != original_content: - with open(build_py_path, 'w') as f: + with open(build_py_path, 'w', encoding='utf-8') as f: f.write(content) except Exception: From dbc55c4e15ba7529708654541eed16af40a16bc7 Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Sat, 31 May 2025 12:15:45 +0200 Subject: [PATCH 69/97] log error more precisely --- builder/frameworks/component_manager.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/builder/frameworks/component_manager.py b/builder/frameworks/component_manager.py index 4f05bf4af..d15b16210 100644 --- a/builder/frameworks/component_manager.py +++ b/builder/frameworks/component_manager.py @@ -484,8 +484,10 @@ def _remove_ignored_lib_includes(self) -> None: f.write(content) self._log_change(f"Updated build file ({total_removed} total removals)") - except Exception as e: + except (IOError, OSError) as e: self._log_change(f"Error processing libraries: {str(e)}") + except Exception as e: + self._log_change(f"Unexpected error processing libraries: {str(e)}") def _validate_changes(self, original_content: str, new_content: str) -> bool: """ From 1ff223a1c3ec6ba126f300550550d1647559d5cb Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Sat, 31 May 2025 12:30:09 +0200 Subject: [PATCH 70/97] add path_traversal detection for sdkconfig URL --- builder/frameworks/espidf.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/builder/frameworks/espidf.py b/builder/frameworks/espidf.py index 22352baf8..14c8dbc22 100644 --- a/builder/frameworks/espidf.py +++ b/builder/frameworks/espidf.py @@ -187,6 +187,18 @@ def create_silent_action(action_func): os.path.join(PROJECT_DIR, "sdkconfig.%s" % env.subst("$PIOENV")), )) +def contains_path_traversal(url): + """Check for Path Traversal patterns""" + dangerous_patterns = [ + '../', '..\\', # Standard Path Traversal + '%2e%2e%2f', '%2e%2e%5c', # URL-encoded + '..%2f', '..%5c', # Mixed + '%252e%252e%252f', # Double encoded + ] + + url_lower = url.lower() + return any(pattern in url_lower for pattern in dangerous_patterns) + # # generate modified Arduino IDF sdkconfig, applying settings from "custom_sdkconfig" # @@ -220,6 +232,11 @@ def load_custom_sdkconfig_file(): for file_entry in sdkconfig_entries: # Handle HTTP/HTTPS URLs if "http" in file_entry and "://" in file_entry: + url = file_entry.split(" ")[0] + # Path Traversal protection + if contains_path_traversal(url): + print(f"Path Traversal detected: {url} check your URL path") + else: try: response = requests.get(file_entry.split(" ")[0]) if response.ok: From d086262ac29a50fd0f395f748a86eab33c8ae2f8 Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Sat, 31 May 2025 12:45:59 +0200 Subject: [PATCH 71/97] move imports --- builder/main.py | 77 ++++++++++++++++++++++++------------------------- 1 file changed, 37 insertions(+), 40 deletions(-) diff --git a/builder/main.py b/builder/main.py index 07f15ff3e..c078ef04d 100644 --- a/builder/main.py +++ b/builder/main.py @@ -14,6 +14,8 @@ import os import re +import subprocess +import shlex import sys import locale from os.path import isfile, join @@ -378,48 +380,43 @@ def firmware_metrics(target, source, env): print(f"Error: Map file not found: {map_file}") print("Make sure the project is built first with 'pio run'") return - - try: - import subprocess - import shlex - - cmd = [env.subst("$PYTHONEXE"), "-m", "esp_idf_size", "--ng"] - - # Parameters from platformio.ini - extra_args = env.GetProjectOption("custom_esp_idf_size_args", "") - if extra_args: - cmd.extend(shlex.split(extra_args)) - - # Command Line Parameter, after -- - cli_args = [] - if "--" in sys.argv: - dash_index = sys.argv.index("--") - if dash_index + 1 < len(sys.argv): - cli_args = sys.argv[dash_index + 1:] - cmd.extend(cli_args) - - # Map-file as last argument - cmd.append(map_file) + + cmd = [env.subst("$PYTHONEXE"), "-m", "esp_idf_size", "--ng"] + + # Parameters from platformio.ini + extra_args = env.GetProjectOption("custom_esp_idf_size_args", "") + if extra_args: + cmd.extend(shlex.split(extra_args)) + + # Command Line Parameter, after -- + cli_args = [] + if "--" in sys.argv: + dash_index = sys.argv.index("--") + if dash_index + 1 < len(sys.argv): + cli_args = sys.argv[dash_index + 1:] + + # Map-file as last argument + cmd.append(map_file) - # Debug-Info if wanted - if env.GetProjectOption("custom_esp_idf_size_verbose", False): - print(f"Running command: {' '.join(cmd)}") - - # Call esp-idf-size - result = subprocess.run(cmd, check=False, capture_output=False) - - if result.returncode != 0: - print(f"Warning: esp-idf-size exited with code {result.returncode}") + # Debug-Info if wanted + if env.GetProjectOption("custom_esp_idf_size_verbose", False): + print(f"Running command: {' '.join(cmd)}") + + # Call esp-idf-size + result = subprocess.run(cmd, check=False, capture_output=False) + + if result.returncode != 0: + print(f"Warning: esp-idf-size exited with code {result.returncode}") - except ImportError: - print("Error: esp-idf-size module not found.") - print("Install with: pip install esp-idf-size") - except FileNotFoundError: - print("Error: Python executable not found.") - print("Check your Python installation.") - except Exception as e: - print(f"Error: Failed to run firmware metrics: {e}") - print("Make sure esp-idf-size is installed: pip install esp-idf-size") +except ImportError: + print("Error: esp-idf-size module not found.") + print("Install with: pip install esp-idf-size") +except FileNotFoundError: + print("Error: Python executable not found.") + print("Check your Python installation.") +except Exception as e: + print(f"Error: Failed to run firmware metrics: {e}") + print("Make sure esp-idf-size is installed: pip install esp-idf-size") # # Target: Build executable and linkable firmware or FS image From c105c65946825f4641300d31880de79c39d2fb6c Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Sat, 31 May 2025 12:47:58 +0200 Subject: [PATCH 72/97] Update main.py --- builder/main.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/builder/main.py b/builder/main.py index c078ef04d..38fc6c06b 100644 --- a/builder/main.py +++ b/builder/main.py @@ -408,15 +408,15 @@ def firmware_metrics(target, source, env): if result.returncode != 0: print(f"Warning: esp-idf-size exited with code {result.returncode}") -except ImportError: - print("Error: esp-idf-size module not found.") - print("Install with: pip install esp-idf-size") -except FileNotFoundError: - print("Error: Python executable not found.") - print("Check your Python installation.") -except Exception as e: - print(f"Error: Failed to run firmware metrics: {e}") - print("Make sure esp-idf-size is installed: pip install esp-idf-size") + except ImportError: + print("Error: esp-idf-size module not found.") + print("Install with: pip install esp-idf-size") + except FileNotFoundError: + print("Error: Python executable not found.") + print("Check your Python installation.") + except Exception as e: + print(f"Error: Failed to run firmware metrics: {e}") + print("Make sure esp-idf-size is installed: pip install esp-idf-size") # # Target: Build executable and linkable firmware or FS image From f7e09de8f35079f9055ad9e4481f9a6a467ae7db Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Sat, 31 May 2025 12:57:10 +0200 Subject: [PATCH 73/97] add doc string --- builder/frameworks/component_manager.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/builder/frameworks/component_manager.py b/builder/frameworks/component_manager.py index d15b16210..c4505718d 100644 --- a/builder/frameworks/component_manager.py +++ b/builder/frameworks/component_manager.py @@ -1,4 +1,10 @@ -# component_manager.py +""" +Component manager for ESP32 Arduino framework builds in PlatformIO. + +This module provides the ComponentManager class for handling IDF component +addition/removal, library ignore processing, and build script modifications. +""" + import os import shutil import re From 4d700f09c5c218cc0f05210853c52fcb898e158e Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Sat, 31 May 2025 13:03:15 +0200 Subject: [PATCH 74/97] simplify check for existing env "BUILD_UNFLAGS" --- builder/frameworks/arduino.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/builder/frameworks/arduino.py b/builder/frameworks/arduino.py index 40078b705..281bad2af 100644 --- a/builder/frameworks/arduino.py +++ b/builder/frameworks/arduino.py @@ -599,8 +599,7 @@ def has_unicore_flags(): # Esp32-solo1 libs settings if flag_custom_sdkconfig and has_unicore_flags(): - build_unflags = env.GetProjectOption("build_unflags") - if not build_unflags: # not existing needs init + if not env.get('BUILD_UNFLAGS'): # Initialize if not set env['BUILD_UNFLAGS'] = [] build_unflags = " ".join(env['BUILD_UNFLAGS']) + " -mdisable-hardware-atomics -ustart_app_other_cores" From 36cb4b61cb7ebb93aa71d748bbe1f0332c33db99 Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Sat, 31 May 2025 13:08:08 +0200 Subject: [PATCH 75/97] fix ident --- builder/frameworks/espidf.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/builder/frameworks/espidf.py b/builder/frameworks/espidf.py index 14c8dbc22..fe077750e 100644 --- a/builder/frameworks/espidf.py +++ b/builder/frameworks/espidf.py @@ -233,19 +233,19 @@ def load_custom_sdkconfig_file(): # Handle HTTP/HTTPS URLs if "http" in file_entry and "://" in file_entry: url = file_entry.split(" ")[0] - # Path Traversal protection - if contains_path_traversal(url): - print(f"Path Traversal detected: {url} check your URL path") - else: - try: - response = requests.get(file_entry.split(" ")[0]) - if response.ok: - return response.content.decode('utf-8') - except requests.RequestException as e: - print(f"Error downloading {file_entry}: {e}") - except UnicodeDecodeError as e: - print(f"Error decoding response from {file_entry}: {e}") - return "" + # Path Traversal protection + if contains_path_traversal(url): + print(f"Path Traversal detected: {url} check your URL path") + else: + try: + response = requests.get(file_entry.split(" ")[0]) + if response.ok: + return response.content.decode('utf-8') + except requests.RequestException as e: + print(f"Error downloading {file_entry}: {e}") + except UnicodeDecodeError as e: + print(f"Error decoding response from {file_entry}: {e}") + return "" # Handle local files if "file://" in file_entry: From 3d6f616db7d5c4ddca40f8968f238a776fdb46c0 Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Sat, 31 May 2025 13:11:51 +0200 Subject: [PATCH 76/97] fix import error exception --- builder/main.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/builder/main.py b/builder/main.py index 38fc6c06b..1a3dab95d 100644 --- a/builder/main.py +++ b/builder/main.py @@ -407,10 +407,7 @@ def firmware_metrics(target, source, env): if result.returncode != 0: print(f"Warning: esp-idf-size exited with code {result.returncode}") - - except ImportError: - print("Error: esp-idf-size module not found.") - print("Install with: pip install esp-idf-size") + except FileNotFoundError: print("Error: Python executable not found.") print("Check your Python installation.") From 37779c8131f7d3d947605d502a36146be6eb7119 Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Sat, 31 May 2025 13:20:28 +0200 Subject: [PATCH 77/97] remove left overs from move import --- builder/main.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/builder/main.py b/builder/main.py index 1a3dab95d..29de2614b 100644 --- a/builder/main.py +++ b/builder/main.py @@ -408,13 +408,6 @@ def firmware_metrics(target, source, env): if result.returncode != 0: print(f"Warning: esp-idf-size exited with code {result.returncode}") - except FileNotFoundError: - print("Error: Python executable not found.") - print("Check your Python installation.") - except Exception as e: - print(f"Error: Failed to run firmware metrics: {e}") - print("Make sure esp-idf-size is installed: pip install esp-idf-size") - # # Target: Build executable and linkable firmware or FS image # From 7b180e61be3fa5c27060e02db58548b73bef4e94 Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Sat, 31 May 2025 13:34:00 +0200 Subject: [PATCH 78/97] refactor in smaller (sub) classes --- builder/frameworks/component_manager.py | 654 ++++++++++++++---------- 1 file changed, 398 insertions(+), 256 deletions(-) diff --git a/builder/frameworks/component_manager.py b/builder/frameworks/component_manager.py index c4505718d..a1c602ba3 100644 --- a/builder/frameworks/component_manager.py +++ b/builder/frameworks/component_manager.py @@ -14,12 +14,12 @@ from typing import Set, Optional, Dict, Any, List, Tuple -class ComponentManager: - """Manages IDF components for ESP32 Arduino framework builds with logging support.""" +class ComponentManagerConfig: + """Handles configuration and environment setup for component management.""" def __init__(self, env): """ - Initialize the ComponentManager. + Initialize the configuration manager. Args: env: PlatformIO environment object @@ -30,18 +30,20 @@ def __init__(self, env): self.board = env.BoardConfig() self.mcu = self.board.get("build.mcu", "esp32").lower() self.project_src_dir = env.subst("$PROJECT_SRC_DIR") - self.removed_components: Set[str] = set() - self.ignored_libs: Set[str] = set() - - # Simple logging attributes - self.component_changes: List[str] = [] - self.arduino_framework_dir = self.platform.get_package_dir("framework-arduinoespressif32") self.arduino_libs_mcu = join(self.platform.get_package_dir("framework-arduinoespressif32-libs"), self.mcu) + + +class ComponentLogger: + """Simple logging functionality for component operations.""" - def _log_change(self, message: str) -> None: + def __init__(self): + """Initialize the logger.""" + self.component_changes: List[str] = [] + + def log_change(self, message: str) -> None: """ - Simple logging without timestamp. + Log a change message. Args: message: Log message to record @@ -49,6 +51,41 @@ def _log_change(self, message: str) -> None: self.component_changes.append(message) print(f"[ComponentManager] {message}") + def get_changes_summary(self) -> List[str]: + """ + Get list of all changes made. + + Returns: + List of change messages + """ + return self.component_changes.copy() + + def print_changes_summary(self) -> None: + """Print summary of all changes.""" + if self.component_changes: + print("\n=== Component Manager Changes ===") + for change in self.component_changes: + print(f" {change}") + print("=" * 35) + else: + print("[ComponentManager] No changes made") + + +class ComponentHandler: + """Handles IDF component addition and removal operations.""" + + def __init__(self, config: ComponentManagerConfig, logger: ComponentLogger): + """ + Initialize the component handler. + + Args: + config: Configuration manager instance + logger: Logger instance + """ + self.config = config + self.logger = logger + self.removed_components: Set[str] = set() + def handle_component_settings(self, add_components: bool = False, remove_components: bool = False) -> None: """ Handle adding and removing IDF components based on project configuration. @@ -57,46 +94,290 @@ def handle_component_settings(self, add_components: bool = False, remove_compone add_components: Whether to process component additions remove_components: Whether to process component removals """ - # Create backup before first component removal and on every add of a component if remove_components and not self.removed_components or add_components: self._backup_pioarduino_build_py() - self._log_change("Created backup of build file") - + self.logger.log_change("Created backup of build file") + # Check if env and GetProjectOption are available - if hasattr(self, 'env') and hasattr(self.env, 'GetProjectOption'): + if hasattr(self.config, 'env') and hasattr(self.config.env, 'GetProjectOption'): component_yml_path = self._get_or_create_component_yml() component_data = self._load_component_yml(component_yml_path) - + if remove_components: - try: - remove_option = self.env.GetProjectOption("custom_component_remove", None) - if remove_option: - components_to_remove = remove_option.splitlines() - self._remove_components(component_data, components_to_remove) - except Exception as e: - self._log_change(f"Error removing components: {str(e)}") - + self._process_component_removals(component_data) + if add_components: - try: - add_option = self.env.GetProjectOption("custom_component_add", None) - if add_option: - components_to_add = add_option.splitlines() - self._add_components(component_data, components_to_add) - except Exception as e: - self._log_change(f"Error adding components: {str(e)}") - + self._process_component_additions(component_data) + self._save_component_yml(component_yml_path, component_data) - + # Clean up removed components if self.removed_components: self._cleanup_removed_components() + + def _process_component_removals(self, component_data: Dict[str, Any]) -> None: + """ + Process component removal requests. + + Args: + component_data: Component configuration data + """ + try: + remove_option = self.config.env.GetProjectOption("custom_component_remove", None) + if remove_option: + components_to_remove = remove_option.splitlines() + self._remove_components(component_data, components_to_remove) + except Exception as e: + self.logger.log_change(f"Error removing components: {str(e)}") + + def _process_component_additions(self, component_data: Dict[str, Any]) -> None: + """ + Process component addition requests. + + Args: + component_data: Component configuration data + """ + try: + add_option = self.config.env.GetProjectOption("custom_component_add", None) + if add_option: + components_to_add = add_option.splitlines() + self._add_components(component_data, components_to_add) + except Exception as e: + self.logger.log_change(f"Error adding components: {str(e)}") + + def _get_or_create_component_yml(self) -> str: + """ + Get path to idf_component.yml, creating it if necessary. + + Returns: + Path to component YAML file + """ + # Try Arduino framework first + framework_yml = join(self.config.arduino_framework_dir, "idf_component.yml") + if os.path.exists(framework_yml): + self._create_backup(framework_yml) + return framework_yml + + # Try project source directory + project_yml = join(self.config.project_src_dir, "idf_component.yml") + if os.path.exists(project_yml): + self._create_backup(project_yml) + return project_yml + + # Create new file in project source + self._create_default_component_yml(project_yml) + return project_yml + + def _create_backup(self, file_path: str) -> None: + """ + Create backup of a file. + + Args: + file_path: Path to file to backup + """ + backup_path = f"{file_path}.orig" + if not os.path.exists(backup_path): + shutil.copy(file_path, backup_path) + + def _create_default_component_yml(self, file_path: str) -> None: + """ + Create a default idf_component.yml file. + + Args: + file_path: Path where to create the file + """ + default_content = { + "dependencies": { + "idf": ">=5.1" + } + } + + with open(file_path, 'w', encoding='utf-8') as f: + yaml.dump(default_content, f) + + def _load_component_yml(self, file_path: str) -> Dict[str, Any]: + """ + Load and parse idf_component.yml file. + + Args: + file_path: Path to YAML file + + Returns: + Parsed YAML data + """ + try: + with open(file_path, "r", encoding='utf-8') as f: + return yaml.load(f, Loader=SafeLoader) or {"dependencies": {}} + except Exception: + return {"dependencies": {}} + + def _save_component_yml(self, file_path: str, data: Dict[str, Any]) -> None: + """ + Save component data to YAML file. + + Args: + file_path: Path to YAML file + data: Data to save + """ + try: + with open(file_path, "w", encoding='utf-8') as f: + yaml.dump(data, f) + except Exception: + pass + + def _remove_components(self, component_data: Dict[str, Any], components_to_remove: list) -> None: + """ + Remove specified components from the configuration. + + Args: + component_data: Component configuration data + components_to_remove: List of components to remove + """ + dependencies = component_data.setdefault("dependencies", {}) + + for component in components_to_remove: + component = component.strip() + if not component: + continue + + if component in dependencies: + self.logger.log_change(f"Removed component: {component}") + del dependencies[component] + + # Track for cleanup + filesystem_name = self._convert_component_name_to_filesystem(component) + self.removed_components.add(filesystem_name) + else: + self.logger.log_change(f"Component not found: {component}") + + def _add_components(self, component_data: Dict[str, Any], components_to_add: list) -> None: + """ + Add specified components to the configuration. + + Args: + component_data: Component configuration data + components_to_add: List of components to add + """ + dependencies = component_data.setdefault("dependencies", {}) + + for component in components_to_add: + component = component.strip() + if len(component) <= 4: # Skip too short entries + continue + + component_name, version = self._parse_component_entry(component) + + if component_name not in dependencies: + dependencies[component_name] = {"version": version} + self.logger.log_change(f"Added component: {component_name} ({version})") + else: + self.logger.log_change(f"Component already exists: {component_name}") + + def _parse_component_entry(self, entry: str) -> Tuple[str, str]: + """ + Parse component entry into name and version. + + Args: + entry: Component entry string + + Returns: + Tuple of (component_name, version) + """ + if "@" in entry: + name, version = entry.split("@", 1) + return (name.strip(), version.strip()) + return (entry.strip(), "*") + + def _convert_component_name_to_filesystem(self, component_name: str) -> str: + """ + Convert component name from registry format to filesystem format. + + Args: + component_name: Component name to convert + + Returns: + Filesystem-safe component name + """ + return component_name.replace("/", "__") + + def _backup_pioarduino_build_py(self) -> None: + """Create backup of the original pioarduino-build.py.""" + if "arduino" not in self.config.env.subst("$PIOFRAMEWORK"): + return + + build_py_path = join(self.config.arduino_libs_mcu, "pioarduino-build.py") + backup_path = join(self.config.arduino_libs_mcu, f"pioarduino-build.py.{self.config.mcu}") + + if os.path.exists(build_py_path) and not os.path.exists(backup_path): + shutil.copy2(build_py_path, backup_path) + + def _cleanup_removed_components(self) -> None: + """Clean up removed components and restore original build file.""" + for component in self.removed_components: + self._remove_include_directory(component) + + self._remove_cpppath_entries() + + def _remove_include_directory(self, component: str) -> None: + """ + Remove include directory for a component. + + Args: + component: Component name + """ + include_path = join(self.config.arduino_libs_mcu, "include", component) + + if os.path.exists(include_path): + shutil.rmtree(include_path) + + def _remove_cpppath_entries(self) -> None: + """Remove CPPPATH entries for removed components from pioarduino-build.py.""" + build_py_path = join(self.config.arduino_libs_mcu, "pioarduino-build.py") + + if not os.path.exists(build_py_path): + return + + try: + with open(build_py_path, 'r', encoding='utf-8') as f: + content = f.read() + + original_content = content + + # Remove CPPPATH entries for each removed component + for component in self.removed_components: + patterns = [ + rf'.*join\([^,]*,\s*"include",\s*"{re.escape(component)}"[^)]*\),?\n', + rf'.*"include/{re.escape(component)}"[^,\n]*,?\n', + rf'.*"[^"]*include[^"]*{re.escape(component)}[^"]*"[^,\n]*,?\n' + ] + + for pattern in patterns: + content = re.sub(pattern, '', content) + + if content != original_content: + with open(build_py_path, 'w', encoding='utf-8') as f: + f.write(content) + + except Exception: + pass + - self.handle_lib_ignore() +class LibraryIgnoreHandler: + """Handles lib_ignore processing and include removal.""" + + def __init__(self, config: ComponentManagerConfig, logger: ComponentLogger): + """ + Initialize the library ignore handler. - # Print summary - if self.component_changes: - self._log_change(f"Session completed with {len(self.component_changes)} changes") + Args: + config: Configuration manager instance + logger: Logger instance + """ + self.config = config + self.logger = logger + self.ignored_libs: Set[str] = set() + self._arduino_libraries_cache = None def handle_lib_ignore(self) -> None: """Handle lib_ignore entries from platformio.ini and remove corresponding includes.""" @@ -110,7 +391,7 @@ def handle_lib_ignore(self) -> None: if lib_ignore_entries: self.ignored_libs.update(lib_ignore_entries) self._remove_ignored_lib_includes() - self._log_change(f"Processed {len(lib_ignore_entries)} ignored libraries") + self.logger.log_change(f"Processed {len(lib_ignore_entries)} ignored libraries") def _get_lib_ignore_entries(self) -> List[str]: """ @@ -121,7 +402,7 @@ def _get_lib_ignore_entries(self) -> List[str]: """ try: # Get lib_ignore from current environment only - lib_ignore = self.env.GetProjectOption("lib_ignore", []) + lib_ignore = self.config.env.GetProjectOption("lib_ignore", []) if isinstance(lib_ignore, str): lib_ignore = [lib_ignore] @@ -170,7 +451,7 @@ def _has_bt_ble_dependencies(self) -> bool: """ try: # Get lib_deps from current environment - lib_deps = self.env.GetProjectOption("lib_deps", []) + lib_deps = self.config.env.GetProjectOption("lib_deps", []) if isinstance(lib_deps, str): lib_deps = [lib_deps] @@ -224,7 +505,7 @@ def _get_arduino_core_libraries(self) -> Dict[str, str]: libraries_mapping = {} # Path to Arduino Core Libraries - arduino_libs_dir = join(self.arduino_framework_dir, "libraries") + arduino_libs_dir = join(self.config.arduino_framework_dir, "libraries") if not os.path.exists(arduino_libs_dir): return libraries_mapping @@ -426,17 +707,17 @@ def _convert_lib_name_to_include(self, lib_name: str) -> str: return cleaned_name def _remove_ignored_lib_includes(self) -> None: - """Remove include entries for ignored libraries from pioarduino-build.py with simple logging.""" - build_py_path = join(self.arduino_libs_mcu, "pioarduino-build.py") + """Remove include entries for ignored libraries from pioarduino-build.py.""" + build_py_path = join(self.config.arduino_libs_mcu, "pioarduino-build.py") if not os.path.exists(build_py_path): - self._log_change("Build file not found") + self.logger.log_change("Build file not found") return # Check if BT/BLE dependencies exist in lib_deps bt_ble_protected = self._has_bt_ble_dependencies() if bt_ble_protected: - self._log_change("BT/BLE protection enabled") + self.logger.log_change("BT/BLE protection enabled") try: with open(build_py_path, 'r', encoding='utf-8') as f: @@ -449,12 +730,12 @@ def _remove_ignored_lib_includes(self) -> None: for lib_name in self.ignored_libs: # Skip BT-related libraries if BT/BLE dependencies are present if bt_ble_protected and self._is_bt_related_library(lib_name): - self._log_change(f"Protected BT library: {lib_name}") + self.logger.log_change(f"Protected BT library: {lib_name}") continue # Hard protection for DSP components if lib_name.lower() in ['dsp', 'esp_dsp', 'dsps', 'fft2r', 'dsps_fft2r']: - self._log_change(f"Protected DSP component: {lib_name}") + self.logger.log_change(f"Protected DSP component: {lib_name}") continue # Multiple patterns to catch different include formats @@ -477,7 +758,7 @@ def _remove_ignored_lib_includes(self) -> None: removed_count += len(matches) if removed_count > 0: - self._log_change(f"Ignored library: {lib_name} ({removed_count} entries)") + self.logger.log_change(f"Ignored library: {lib_name} ({removed_count} entries)") total_removed += removed_count # Clean up empty lines and trailing commas @@ -488,12 +769,12 @@ def _remove_ignored_lib_includes(self) -> None: if self._validate_changes(original_content, content) and content != original_content: with open(build_py_path, 'w', encoding='utf-8') as f: f.write(content) - self._log_change(f"Updated build file ({total_removed} total removals)") + self.logger.log_change(f"Updated build file ({total_removed} total removals)") except (IOError, OSError) as e: - self._log_change(f"Error processing libraries: {str(e)}") + self.logger.log_change(f"Error processing libraries: {str(e)}") except Exception as e: - self._log_change(f"Unexpected error processing libraries: {str(e)}") + self.logger.log_change(f"Unexpected error processing libraries: {str(e)}") def _validate_changes(self, original_content: str, new_content: str) -> bool: """ @@ -513,253 +794,114 @@ def _validate_changes(self, original_content: str, new_content: str) -> bool: # Don't allow removing more than 50% of the file or negative changes return not (removed_lines > original_lines * 0.5 or removed_lines < 0) - def _get_or_create_component_yml(self) -> str: - """ - Get path to idf_component.yml, creating it if necessary. - - Returns: - Path to component YAML file - """ - # Try Arduino framework first - framework_yml = join(self.arduino_framework_dir, "idf_component.yml") - if os.path.exists(framework_yml): - self._create_backup(framework_yml) - return framework_yml - - # Try project source directory - project_yml = join(self.project_src_dir, "idf_component.yml") - if os.path.exists(project_yml): - self._create_backup(project_yml) - return project_yml - - # Create new file in project source - self._create_default_component_yml(project_yml) - return project_yml - - def _create_backup(self, file_path: str) -> None: - """ - Create backup of a file. - - Args: - file_path: Path to file to backup - """ - backup_path = f"{file_path}.orig" - if not os.path.exists(backup_path): - shutil.copy(file_path, backup_path) - - def _create_default_component_yml(self, file_path: str) -> None: - """ - Create a default idf_component.yml file. + def _backup_pioarduino_build_py(self) -> None: + """Create backup of the original pioarduino-build.py.""" + if "arduino" not in self.config.env.subst("$PIOFRAMEWORK"): + return - Args: - file_path: Path where to create the file - """ - default_content = { - "dependencies": { - "idf": ">=5.1" - } - } + build_py_path = join(self.config.arduino_libs_mcu, "pioarduino-build.py") + backup_path = join(self.config.arduino_libs_mcu, f"pioarduino-build.py.{self.config.mcu}") - with open(file_path, 'w', encoding='utf-8') as f: - yaml.dump(default_content, f) - - def _load_component_yml(self, file_path: str) -> Dict[str, Any]: - """ - Load and parse idf_component.yml file. - - Args: - file_path: Path to YAML file - - Returns: - Parsed YAML data - """ - try: - with open(file_path, "r", encoding='utf-8') as f: - return yaml.load(f, Loader=SafeLoader) or {"dependencies": {}} - except Exception: - return {"dependencies": {}} + if os.path.exists(build_py_path) and not os.path.exists(backup_path): + shutil.copy2(build_py_path, backup_path) + + +class BackupManager: + """Handles backup and restore operations for build files.""" - def _save_component_yml(self, file_path: str, data: Dict[str, Any]) -> None: + def __init__(self, config: ComponentManagerConfig): """ - Save component data to YAML file. + Initialize the backup manager. Args: - file_path: Path to YAML file - data: Data to save + config: Configuration manager instance """ - try: - with open(file_path, "w", encoding='utf-8') as f: - yaml.dump(data, f) - except Exception: - pass + self.config = config - def _remove_components(self, component_data: Dict[str, Any], components_to_remove: list) -> None: - """ - Remove specified components from the configuration with simple logging. + def backup_pioarduino_build_py(self) -> None: + """Create backup of the original pioarduino-build.py.""" + if "arduino" not in self.config.env.subst("$PIOFRAMEWORK"): + return - Args: - component_data: Component configuration data - components_to_remove: List of components to remove - """ - dependencies = component_data.setdefault("dependencies", {}) + build_py_path = join(self.config.arduino_libs_mcu, "pioarduino-build.py") + backup_path = join(self.config.arduino_libs_mcu, f"pioarduino-build.py.{self.config.mcu}") - for component in components_to_remove: - component = component.strip() - if not component: - continue - - if component in dependencies: - self._log_change(f"Removed component: {component}") - del dependencies[component] - - # Track for cleanup - filesystem_name = self._convert_component_name_to_filesystem(component) - self.removed_components.add(filesystem_name) - else: - self._log_change(f"Component not found: {component}") + if os.path.exists(build_py_path) and not os.path.exists(backup_path): + shutil.copy2(build_py_path, backup_path) - def _add_components(self, component_data: Dict[str, Any], components_to_add: list) -> None: + def restore_pioarduino_build_py(self, target=None, source=None, env=None) -> None: """ - Add specified components to the configuration with simple logging. + Restore the original pioarduino-build.py from backup. Args: - component_data: Component configuration data - components_to_add: List of components to add - """ - dependencies = component_data.setdefault("dependencies", {}) - - for component in components_to_add: - component = component.strip() - if len(component) <= 4: # Skip too short entries - continue - - component_name, version = self._parse_component_entry(component) - - if component_name not in dependencies: - dependencies[component_name] = {"version": version} - self._log_change(f"Added component: {component_name} ({version})") - else: - self._log_change(f"Component already exists: {component_name}") - - def _parse_component_entry(self, entry: str) -> Tuple[str, str]: + target: Build target (unused) + source: Build source (unused) + env: Environment (unused) """ - Parse component entry into name and version. + build_py_path = join(self.config.arduino_libs_mcu, "pioarduino-build.py") + backup_path = join(self.config.arduino_libs_mcu, f"pioarduino-build.py.{self.config.mcu}") - Args: - entry: Component entry string - - Returns: - Tuple of (component_name, version) - """ - if "@" in entry: - name, version = entry.split("@", 1) - return (name.strip(), version.strip()) - return (entry.strip(), "*") + if os.path.exists(backup_path): + shutil.copy2(backup_path, build_py_path) + os.remove(backup_path) + + +class ComponentManager: + """Main component manager that orchestrates all operations.""" - def _convert_component_name_to_filesystem(self, component_name: str) -> str: + def __init__(self, env): """ - Convert component name from registry format to filesystem format. + Initialize the ComponentManager with composition pattern. Args: - component_name: Component name to convert - - Returns: - Filesystem-safe component name + env: PlatformIO environment object """ - return component_name.replace("/", "__") - - def _backup_pioarduino_build_py(self) -> None: - """Create backup of the original pioarduino-build.py.""" - if "arduino" not in self.env.subst("$PIOFRAMEWORK"): - return - - build_py_path = join(self.arduino_libs_mcu, "pioarduino-build.py") - backup_path = join(self.arduino_libs_mcu, f"pioarduino-build.py.{self.mcu}") - - if os.path.exists(build_py_path) and not os.path.exists(backup_path): - shutil.copy2(build_py_path, backup_path) + self.config = ComponentManagerConfig(env) + self.logger = ComponentLogger() + self.component_handler = ComponentHandler(self.config, self.logger) + self.library_handler = LibraryIgnoreHandler(self.config, self.logger) + self.backup_manager = BackupManager(self.config) - def _cleanup_removed_components(self) -> None: - """Clean up removed components and restore original build file.""" - for component in self.removed_components: - self._remove_include_directory(component) - - self._remove_cpppath_entries() - - def _remove_include_directory(self, component: str) -> None: + def handle_component_settings(self, add_components: bool = False, remove_components: bool = False) -> None: """ - Remove include directory for a component. + Handle component operations by delegating to specialized handlers. Args: - component: Component name + add_components: Whether to process component additions + remove_components: Whether to process component removals """ - include_path = join(self.arduino_libs_mcu, "include", component) + self.component_handler.handle_component_settings(add_components, remove_components) + self.library_handler.handle_lib_ignore() - if os.path.exists(include_path): - shutil.rmtree(include_path) + # Print summary + changes = self.logger.get_changes_summary() + if changes: + self.logger.log_change(f"Session completed with {len(changes)} changes") - def _remove_cpppath_entries(self) -> None: - """Remove CPPPATH entries for removed components from pioarduino-build.py.""" - build_py_path = join(self.arduino_libs_mcu, "pioarduino-build.py") - - if not os.path.exists(build_py_path): - return - - try: - with open(build_py_path, 'r', encoding='utf-8') as f: - content = f.read() - - original_content = content - - # Remove CPPPATH entries for each removed component - for component in self.removed_components: - patterns = [ - rf'.*join\([^,]*,\s*"include",\s*"{re.escape(component)}"[^)]*\),?\n', - rf'.*"include/{re.escape(component)}"[^,\n]*,?\n', - rf'.*"[^"]*include[^"]*{re.escape(component)}[^"]*"[^,\n]*,?\n' - ] - - for pattern in patterns: - content = re.sub(pattern, '', content) - - if content != original_content: - with open(build_py_path, 'w', encoding='utf-8') as f: - f.write(content) - - except Exception: - pass + def handle_lib_ignore(self) -> None: + """Delegate lib_ignore handling to specialized handler.""" + self.library_handler.handle_lib_ignore() def restore_pioarduino_build_py(self, target=None, source=None, env=None) -> None: """ - Restore the original pioarduino-build.py from backup. + Delegate backup restoration to backup manager. Args: target: Build target (unused) source: Build source (unused) env: Environment (unused) """ - build_py_path = join(self.arduino_libs_mcu, "pioarduino-build.py") - backup_path = join(self.arduino_libs_mcu, f"pioarduino-build.py.{self.mcu}") - - if os.path.exists(backup_path): - shutil.copy2(backup_path, build_py_path) - os.remove(backup_path) + self.backup_manager.restore_pioarduino_build_py(target, source, env) def get_changes_summary(self) -> List[str]: """ - Get simple list of all changes made. + Get summary of changes from logger. Returns: List of change messages """ - return self.component_changes.copy() - + return self.logger.get_changes_summary() + def print_changes_summary(self) -> None: - """Print a simple summary of all changes.""" - if self.component_changes: - print("\n=== Component Manager Changes ===") - for change in self.component_changes: - print(f" {change}") - print("=" * 35) - else: - print("[ComponentManager] No changes made") + """Print changes summary via logger.""" + self.logger.print_changes_summary() From c0d2d43447290713d86140c24a1ca017959ac2d8 Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Sat, 31 May 2025 14:06:37 +0200 Subject: [PATCH 79/97] doc strings more detailed --- builder/frameworks/component_manager.py | 408 +++++++++++++++++++----- 1 file changed, 323 insertions(+), 85 deletions(-) diff --git a/builder/frameworks/component_manager.py b/builder/frameworks/component_manager.py index a1c602ba3..d775816c1 100644 --- a/builder/frameworks/component_manager.py +++ b/builder/frameworks/component_manager.py @@ -3,6 +3,9 @@ This module provides the ComponentManager class for handling IDF component addition/removal, library ignore processing, and build script modifications. +It supports managing ESP-IDF components within Arduino framework projects, +allowing developers to add or remove specific components and handle library +dependencies efficiently. """ import os @@ -15,53 +18,94 @@ class ComponentManagerConfig: - """Handles configuration and environment setup for component management.""" + """ + Handles configuration and environment setup for component management. + + This class centralizes all configuration-related operations and provides + a unified interface for accessing PlatformIO environment settings, + board configurations, and framework paths. + """ def __init__(self, env): """ - Initialize the configuration manager. + Initialize the configuration manager with PlatformIO environment. + + Extracts and stores essential configuration parameters from the PlatformIO + environment including platform details, board configuration, MCU type, + and framework paths. This initialization ensures all dependent classes + have consistent access to configuration data. Args: - env: PlatformIO environment object + env: PlatformIO environment object containing project configuration, + board settings, and platform information """ self.env = env self.platform = env.PioPlatform() self.config = env.GetProjectConfig() self.board = env.BoardConfig() + # Extract MCU type from board configuration, defaulting to esp32 self.mcu = self.board.get("build.mcu", "esp32").lower() + # Get project source directory path self.project_src_dir = env.subst("$PROJECT_SRC_DIR") + # Get Arduino framework installation directory self.arduino_framework_dir = self.platform.get_package_dir("framework-arduinoespressif32") + # Get MCU-specific Arduino libraries directory self.arduino_libs_mcu = join(self.platform.get_package_dir("framework-arduinoespressif32-libs"), self.mcu) class ComponentLogger: - """Simple logging functionality for component operations.""" + """ + Simple logging functionality for component operations. + + Provides centralized logging for all component management operations, + tracking changes made during the build process and offering summary + reporting capabilities. + """ def __init__(self): - """Initialize the logger.""" + """ + Initialize the logger with empty change tracking. + + Sets up internal data structures for tracking component changes + and modifications made during the build process. + """ + # List to store all change messages for summary reporting self.component_changes: List[str] = [] def log_change(self, message: str) -> None: """ - Log a change message. + Log a change message with immediate console output. + + Records the change message internally for summary reporting and + immediately prints it to the console with a component manager prefix + for real-time feedback during build operations. Args: - message: Log message to record + message: Descriptive message about the change or operation performed """ self.component_changes.append(message) print(f"[ComponentManager] {message}") def get_changes_summary(self) -> List[str]: """ - Get list of all changes made. + Get a copy of all changes made during the session. + + Returns a defensive copy of the change log to prevent external + modification while allowing access to the complete change history. Returns: - List of change messages + List of change messages in chronological order """ return self.component_changes.copy() def print_changes_summary(self) -> None: - """Print summary of all changes.""" + """ + Print a formatted summary of all changes made. + + Outputs a nicely formatted summary of all component changes if any + were made, or a simple message indicating no changes occurred. + Useful for end-of-build reporting and debugging. + """ if self.component_changes: print("\n=== Component Manager Changes ===") for change in self.component_changes: @@ -72,27 +116,42 @@ def print_changes_summary(self) -> None: class ComponentHandler: - """Handles IDF component addition and removal operations.""" + """ + Handles IDF component addition and removal operations. + + Manages the core functionality for adding and removing ESP-IDF components + from Arduino framework projects, including YAML file manipulation, + component validation, and cleanup operations. + """ def __init__(self, config: ComponentManagerConfig, logger: ComponentLogger): """ - Initialize the component handler. + Initialize the component handler with configuration and logging. + + Sets up the component handler with necessary dependencies for + configuration access and change logging. Initializes tracking + for removed components to enable proper cleanup operations. Args: - config: Configuration manager instance - logger: Logger instance + config: Configuration manager instance providing access to paths and settings + logger: Logger instance for recording component operations """ self.config = config self.logger = logger + # Track removed components for cleanup operations self.removed_components: Set[str] = set() def handle_component_settings(self, add_components: bool = False, remove_components: bool = False) -> None: """ Handle adding and removing IDF components based on project configuration. + Main entry point for component management operations. Processes both + component additions and removals based on project configuration options, + manages backup creation, and handles cleanup of removed components. + Args: - add_components: Whether to process component additions - remove_components: Whether to process component removals + add_components: Whether to process component additions from custom_component_add + remove_components: Whether to process component removals from custom_component_remove """ # Create backup before first component removal and on every add of a component if remove_components and not self.removed_components or add_components: @@ -118,14 +177,19 @@ def handle_component_settings(self, add_components: bool = False, remove_compone def _process_component_removals(self, component_data: Dict[str, Any]) -> None: """ - Process component removal requests. + Process component removal requests from project configuration. + + Reads the custom_component_remove option from platformio.ini and + processes each component for removal from the dependency list. + Handles errors gracefully and logs all operations. Args: - component_data: Component configuration data + component_data: Component configuration data dictionary containing dependencies """ try: remove_option = self.config.env.GetProjectOption("custom_component_remove", None) if remove_option: + # Split multiline option into individual components components_to_remove = remove_option.splitlines() self._remove_components(component_data, components_to_remove) except Exception as e: @@ -133,14 +197,19 @@ def _process_component_removals(self, component_data: Dict[str, Any]) -> None: def _process_component_additions(self, component_data: Dict[str, Any]) -> None: """ - Process component addition requests. + Process component addition requests from project configuration. + + Reads the custom_component_add option from platformio.ini and + processes each component for addition to the dependency list. + Handles errors gracefully and logs all operations. Args: - component_data: Component configuration data + component_data: Component configuration data dictionary containing dependencies """ try: add_option = self.config.env.GetProjectOption("custom_component_add", None) if add_option: + # Split multiline option into individual components components_to_add = add_option.splitlines() self._add_components(component_data, components_to_add) except Exception as e: @@ -150,8 +219,13 @@ def _get_or_create_component_yml(self) -> str: """ Get path to idf_component.yml, creating it if necessary. + Searches for existing idf_component.yml files in the Arduino framework + directory first, then in the project source directory. If no file + exists, creates a new one in the project source directory with + default content. + Returns: - Path to component YAML file + Absolute path to the component YAML file """ # Try Arduino framework first framework_yml = join(self.config.arduino_framework_dir, "idf_component.yml") @@ -171,10 +245,14 @@ def _get_or_create_component_yml(self) -> str: def _create_backup(self, file_path: str) -> None: """ - Create backup of a file. + Create backup of a file with .orig extension. + + Creates a backup copy of the specified file by appending .orig + to the filename. Only creates the backup if it doesn't already + exist to preserve the original state. Args: - file_path: Path to file to backup + file_path: Absolute path to the file to backup """ backup_path = f"{file_path}.orig" if not os.path.exists(backup_path): @@ -182,10 +260,14 @@ def _create_backup(self, file_path: str) -> None: def _create_default_component_yml(self, file_path: str) -> None: """ - Create a default idf_component.yml file. + Create a default idf_component.yml file with basic ESP-IDF dependency. + + Creates a new component YAML file with minimal default content + specifying ESP-IDF version 5.1 or higher as the base dependency. + This ensures compatibility with modern ESP-IDF features. Args: - file_path: Path where to create the file + file_path: Absolute path where to create the new YAML file """ default_content = { "dependencies": { @@ -198,13 +280,17 @@ def _create_default_component_yml(self, file_path: str) -> None: def _load_component_yml(self, file_path: str) -> Dict[str, Any]: """ - Load and parse idf_component.yml file. + Load and parse idf_component.yml file safely. + + Attempts to load and parse the YAML file using SafeLoader for + security. Returns a default structure with empty dependencies + if the file cannot be read or parsed. Args: - file_path: Path to YAML file + file_path: Absolute path to the YAML file to load Returns: - Parsed YAML data + Parsed YAML data as dictionary, or default structure on failure """ try: with open(file_path, "r", encoding='utf-8') as f: @@ -214,11 +300,15 @@ def _load_component_yml(self, file_path: str) -> Dict[str, Any]: def _save_component_yml(self, file_path: str, data: Dict[str, Any]) -> None: """ - Save component data to YAML file. + Save component data to YAML file safely. + + Attempts to write the component data dictionary to the specified + YAML file. Handles errors gracefully by silently failing to + prevent build interruption. Args: - file_path: Path to YAML file - data: Data to save + file_path: Absolute path to the YAML file to write + data: Component data dictionary to serialize """ try: with open(file_path, "w", encoding='utf-8') as f: @@ -230,9 +320,13 @@ def _remove_components(self, component_data: Dict[str, Any], components_to_remov """ Remove specified components from the configuration. + Iterates through the list of components to remove, checking if each + exists in the dependencies and removing it if found. Tracks removed + components for later cleanup operations and logs all actions. + Args: - component_data: Component configuration data - components_to_remove: List of components to remove + component_data: Component configuration data dictionary + components_to_remove: List of component names to remove """ dependencies = component_data.setdefault("dependencies", {}) @@ -245,7 +339,7 @@ def _remove_components(self, component_data: Dict[str, Any], components_to_remov self.logger.log_change(f"Removed component: {component}") del dependencies[component] - # Track for cleanup + # Track for cleanup - convert to filesystem-safe name filesystem_name = self._convert_component_name_to_filesystem(component) self.removed_components.add(filesystem_name) else: @@ -255,9 +349,13 @@ def _add_components(self, component_data: Dict[str, Any], components_to_add: lis """ Add specified components to the configuration. + Processes each component entry, parsing name and version information, + and adds new components to the dependencies. Skips components that + already exist and filters out entries that are too short to be valid. + Args: - component_data: Component configuration data - components_to_add: List of components to add + component_data: Component configuration data dictionary + components_to_add: List of component entries to add (format: name@version or name) """ dependencies = component_data.setdefault("dependencies", {}) @@ -276,13 +374,17 @@ def _add_components(self, component_data: Dict[str, Any], components_to_add: lis def _parse_component_entry(self, entry: str) -> Tuple[str, str]: """ - Parse component entry into name and version. + Parse component entry into name and version components. + + Splits component entries that contain version information (format: name@version) + and returns both parts. If no version is specified, defaults to "*" for + latest version. Args: - entry: Component entry string + entry: Component entry string (e.g., "espressif/esp_timer@1.0.0" or "espressif/esp_timer") Returns: - Tuple of (component_name, version) + Tuple containing (component_name, version) """ if "@" in entry: name, version = entry.split("@", 1) @@ -293,16 +395,25 @@ def _convert_component_name_to_filesystem(self, component_name: str) -> str: """ Convert component name from registry format to filesystem format. + Converts component names from ESP Component Registry format (using forward slashes) + to filesystem-safe format (using double underscores) for directory operations. + Args: - component_name: Component name to convert + component_name: Component name in registry format (e.g., "espressif/esp_timer") Returns: - Filesystem-safe component name + Filesystem-safe component name (e.g., "espressif__esp_timer") """ return component_name.replace("/", "__") def _backup_pioarduino_build_py(self) -> None: - """Create backup of the original pioarduino-build.py.""" + """ + Create backup of the original pioarduino-build.py file. + + Creates a backup of the Arduino framework's build script before + making modifications. Only operates when Arduino framework is active + and creates MCU-specific backup names to avoid conflicts. + """ if "arduino" not in self.config.env.subst("$PIOFRAMEWORK"): return @@ -313,7 +424,13 @@ def _backup_pioarduino_build_py(self) -> None: shutil.copy2(build_py_path, backup_path) def _cleanup_removed_components(self) -> None: - """Clean up removed components and restore original build file.""" + """ + Clean up removed components and restore original build file. + + Performs cleanup operations for all components that were removed, + including removing include directories and cleaning up CPPPATH + entries from the build script. + """ for component in self.removed_components: self._remove_include_directory(component) @@ -321,10 +438,13 @@ def _cleanup_removed_components(self) -> None: def _remove_include_directory(self, component: str) -> None: """ - Remove include directory for a component. + Remove include directory for a specific component. + + Removes the component's include directory from the Arduino framework + libraries to prevent compilation errors and reduce build overhead. Args: - component: Component name + component: Component name in filesystem format """ include_path = join(self.config.arduino_libs_mcu, "include", component) @@ -332,7 +452,13 @@ def _remove_include_directory(self, component: str) -> None: shutil.rmtree(include_path) def _remove_cpppath_entries(self) -> None: - """Remove CPPPATH entries for removed components from pioarduino-build.py.""" + """ + Remove CPPPATH entries for removed components from pioarduino-build.py. + + Scans the Arduino build script and removes include path entries + for all components that were removed from the project. Uses + multiple regex patterns to catch different include path formats. + """ build_py_path = join(self.config.arduino_libs_mcu, "pioarduino-build.py") if not os.path.exists(build_py_path): @@ -364,23 +490,41 @@ def _remove_cpppath_entries(self) -> None: class LibraryIgnoreHandler: - """Handles lib_ignore processing and include removal.""" + """ + Handles lib_ignore processing and include removal. + + Manages the processing of lib_ignore entries from platformio.ini, + converting library names to include paths and removing corresponding + entries from the build script while protecting critical components. + """ def __init__(self, config: ComponentManagerConfig, logger: ComponentLogger): """ Initialize the library ignore handler. + Sets up the handler with configuration and logging dependencies, + initializes tracking for ignored libraries, and prepares caching + for Arduino library mappings. + Args: - config: Configuration manager instance - logger: Logger instance + config: Configuration manager instance for accessing paths and settings + logger: Logger instance for recording library operations """ self.config = config self.logger = logger + # Track ignored libraries for processing self.ignored_libs: Set[str] = set() + # Cache for Arduino library mappings (lazy loaded) self._arduino_libraries_cache = None def handle_lib_ignore(self) -> None: - """Handle lib_ignore entries from platformio.ini and remove corresponding includes.""" + """ + Handle lib_ignore entries from platformio.ini and remove corresponding includes. + + Main entry point for library ignore processing. Creates backup if needed, + processes lib_ignore entries from the current environment, and removes + corresponding include paths from the build script. + """ # Create backup before processing lib_ignore if not self.ignored_libs: self._backup_pioarduino_build_py() @@ -397,8 +541,12 @@ def _get_lib_ignore_entries(self) -> List[str]: """ Get lib_ignore entries from current environment configuration only. + Extracts and processes lib_ignore entries from the platformio.ini + configuration, converting library names to include directory names + and filtering out critical ESP32 components that should never be ignored. + Returns: - List of library names to ignore + List of processed library names ready for include path removal """ try: # Get lib_ignore from current environment only @@ -446,8 +594,12 @@ def _has_bt_ble_dependencies(self) -> bool: """ Check if lib_deps contains any BT/BLE related dependencies. + Scans the lib_deps configuration option for Bluetooth or BLE + related keywords to determine if BT components should be protected + from removal even if they appear in lib_ignore. + Returns: - True if BT/BLE dependencies are found + True if BT/BLE dependencies are found in lib_deps """ try: # Get lib_deps from current environment @@ -472,11 +624,15 @@ def _is_bt_related_library(self, lib_name: str) -> bool: """ Check if a library name is related to Bluetooth/BLE functionality. + Examines library names for Bluetooth and BLE related keywords + to determine if the library should be protected when BT dependencies + are present in the project. + Args: - lib_name: Library name to check + lib_name: Library name to check for BT/BLE relation Returns: - True if library is BT/BLE related + True if library name contains BT/BLE related keywords """ lib_name_upper = lib_name.upper() @@ -499,8 +655,12 @@ def _get_arduino_core_libraries(self) -> Dict[str, str]: """ Get all Arduino core libraries and their corresponding include paths. + Scans the Arduino framework libraries directory to build a mapping + of library names to their corresponding include paths. Reads + library.properties files to get official library names. + Returns: - Dictionary mapping library names to include paths + Dictionary mapping library names to include directory names """ libraries_mapping = {} @@ -528,11 +688,14 @@ def _get_library_name_from_properties(self, lib_dir: str) -> Optional[str]: """ Extract library name from library.properties file. + Reads the library.properties file in the given directory and + extracts the official library name from the 'name=' field. + Args: - lib_dir: Library directory path + lib_dir: Path to library directory containing library.properties Returns: - Library name or None if not found + Official library name or None if not found or readable """ prop_path = join(lib_dir, "library.properties") if not os.path.isfile(prop_path): @@ -553,12 +716,16 @@ def _map_library_to_include_path(self, lib_name: str, dir_name: str) -> str: """ Map library name to corresponding include path. + Converts Arduino library names to their corresponding ESP-IDF + component include paths using an extensive mapping table. + Handles common Arduino libraries and their ESP-IDF equivalents. + Args: - lib_name: Library name - dir_name: Directory name + lib_name: Official library name from library.properties + dir_name: Directory name of the library Returns: - Mapped include path + Corresponding ESP-IDF component include path name """ lib_name_lower = lib_name.lower().replace(' ', '').replace('-', '_') dir_name_lower = dir_name.lower() @@ -659,11 +826,15 @@ def _convert_lib_name_to_include(self, lib_name: str) -> str: """ Convert library name to potential include directory name. + Converts library names from platformio.ini lib_ignore entries + to their corresponding include directory names. Uses Arduino + core library mappings and common naming conventions. + Args: - lib_name: Library name to convert + lib_name: Library name from lib_ignore configuration Returns: - Converted include directory name + Converted include directory name for path removal """ # Load Arduino Core Libraries on first call if not hasattr(self, '_arduino_libraries_cache'): @@ -707,7 +878,14 @@ def _convert_lib_name_to_include(self, lib_name: str) -> str: return cleaned_name def _remove_ignored_lib_includes(self) -> None: - """Remove include entries for ignored libraries from pioarduino-build.py.""" + """ + Remove include entries for ignored libraries from pioarduino-build.py. + + Processes the Arduino build script to remove CPPPATH entries for + all ignored libraries. Implements protection for BT/BLE and DSP + components when dependencies are detected. Uses multiple regex + patterns to catch different include path formats. + """ build_py_path = join(self.config.arduino_libs_mcu, "pioarduino-build.py") if not os.path.exists(build_py_path): @@ -778,14 +956,18 @@ def _remove_ignored_lib_includes(self) -> None: def _validate_changes(self, original_content: str, new_content: str) -> bool: """ - Validate that the changes are reasonable. + Validate that the changes are reasonable and safe. + + Performs sanity checks on the modified content to ensure that + the changes don't remove too much content or create invalid + modifications that could break the build process. Args: - original_content: Original file content - new_content: Modified file content + original_content: Original file content before modifications + new_content: Modified file content after processing Returns: - True if changes are valid + True if changes are within acceptable limits and safe to apply """ original_lines = len(original_content.splitlines()) new_lines = len(new_content.splitlines()) @@ -795,7 +977,13 @@ def _validate_changes(self, original_content: str, new_content: str) -> bool: return not (removed_lines > original_lines * 0.5 or removed_lines < 0) def _backup_pioarduino_build_py(self) -> None: - """Create backup of the original pioarduino-build.py.""" + """ + Create backup of the original pioarduino-build.py file. + + Creates a backup copy of the Arduino build script before making + modifications. Only operates when Arduino framework is active + and uses MCU-specific backup naming to avoid conflicts. + """ if "arduino" not in self.config.env.subst("$PIOFRAMEWORK"): return @@ -807,19 +995,34 @@ def _backup_pioarduino_build_py(self) -> None: class BackupManager: - """Handles backup and restore operations for build files.""" + """ + Handles backup and restore operations for build files. + + Manages the creation and restoration of backup files for the Arduino + framework build scripts, ensuring that original files can be restored + when needed or when builds are cleaned. + """ def __init__(self, config: ComponentManagerConfig): """ - Initialize the backup manager. + Initialize the backup manager with configuration access. + + Sets up the backup manager with access to configuration paths + and settings needed for backup and restore operations. Args: - config: Configuration manager instance + config: Configuration manager instance providing access to paths """ self.config = config def backup_pioarduino_build_py(self) -> None: - """Create backup of the original pioarduino-build.py.""" + """ + Create backup of the original pioarduino-build.py file. + + Creates a backup copy of the Arduino framework's build script + with MCU-specific naming to prevent conflicts between different + ESP32 variants. Only creates backup if it doesn't already exist. + """ if "arduino" not in self.config.env.subst("$PIOFRAMEWORK"): return @@ -833,10 +1036,14 @@ def restore_pioarduino_build_py(self, target=None, source=None, env=None) -> Non """ Restore the original pioarduino-build.py from backup. + Restores the original Arduino build script from the backup copy + and removes the backup file. This is typically called during + clean operations or when resetting the build environment. + Args: - target: Build target (unused) - source: Build source (unused) - env: Environment (unused) + target: Build target (unused, for PlatformIO compatibility) + source: Build source (unused, for PlatformIO compatibility) + env: Environment (unused, for PlatformIO compatibility) """ build_py_path = join(self.config.arduino_libs_mcu, "pioarduino-build.py") backup_path = join(self.config.arduino_libs_mcu, f"pioarduino-build.py.{self.config.mcu}") @@ -847,14 +1054,25 @@ def restore_pioarduino_build_py(self, target=None, source=None, env=None) -> Non class ComponentManager: - """Main component manager that orchestrates all operations.""" + """ + Main component manager that orchestrates all operations. + + Primary interface for component management operations, coordinating + between specialized handlers for components, libraries, and backups. + Uses composition pattern to organize functionality into focused classes. + """ def __init__(self, env): """ Initialize the ComponentManager with composition pattern. + Creates and configures all specialized handler instances using + the composition pattern for better separation of concerns and + maintainability. Each handler focuses on a specific aspect + of component management. + Args: - env: PlatformIO environment object + env: PlatformIO environment object containing project configuration """ self.config = ComponentManagerConfig(env) self.logger = ComponentLogger() @@ -866,9 +1084,13 @@ def handle_component_settings(self, add_components: bool = False, remove_compone """ Handle component operations by delegating to specialized handlers. + Main entry point for component management operations. Coordinates + component addition/removal and library ignore processing, then + provides a summary of all changes made during the session. + Args: - add_components: Whether to process component additions - remove_components: Whether to process component removals + add_components: Whether to process component additions from configuration + remove_components: Whether to process component removals from configuration """ self.component_handler.handle_component_settings(add_components, remove_components) self.library_handler.handle_lib_ignore() @@ -879,17 +1101,25 @@ def handle_component_settings(self, add_components: bool = False, remove_compone self.logger.log_change(f"Session completed with {len(changes)} changes") def handle_lib_ignore(self) -> None: - """Delegate lib_ignore handling to specialized handler.""" + """ + Delegate lib_ignore handling to specialized handler. + + Provides direct access to library ignore processing for cases + where only library handling is needed without component operations. + """ self.library_handler.handle_lib_ignore() def restore_pioarduino_build_py(self, target=None, source=None, env=None) -> None: """ Delegate backup restoration to backup manager. + Provides access to backup restoration functionality, typically + used during clean operations or build environment resets. + Args: - target: Build target (unused) - source: Build source (unused) - env: Environment (unused) + target: Build target (unused, for PlatformIO compatibility) + source: Build source (unused, for PlatformIO compatibility) + env: Environment (unused, for PlatformIO compatibility) """ self.backup_manager.restore_pioarduino_build_py(target, source, env) @@ -897,11 +1127,19 @@ def get_changes_summary(self) -> List[str]: """ Get summary of changes from logger. + Provides access to the complete list of changes made during + the current session for reporting or debugging purposes. + Returns: - List of change messages + List of change messages in chronological order """ return self.logger.get_changes_summary() def print_changes_summary(self) -> None: - """Print changes summary via logger.""" + """ + Print changes summary via logger. + + Outputs a formatted summary of all changes made during the + session, useful for build reporting and debugging. + """ self.logger.print_changes_summary() From 3249fdbdc0e121addffd28f6082d4c0edb1a6f80 Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Sat, 31 May 2025 14:26:41 +0200 Subject: [PATCH 80/97] formatting Python style conform / rm whitespaces --- builder/frameworks/arduino.py | 476 +++++++++++++++++++++------------- 1 file changed, 295 insertions(+), 181 deletions(-) diff --git a/builder/frameworks/arduino.py b/builder/frameworks/arduino.py index 281bad2af..c7ab1aeec 100644 --- a/builder/frameworks/arduino.py +++ b/builder/frameworks/arduino.py @@ -51,22 +51,24 @@ "esp-idf-size": ">=1.6.1" } + def setup_logging(): """Setup logging with optional file output""" handlers = [logging.StreamHandler()] - + # Only add file handler if writable and not disabled log_file = os.environ.get('ARDUINO_FRAMEWORK_LOG_FILE') if log_file: with suppress(OSError, PermissionError): handlers.append(logging.FileHandler(log_file)) - + logging.basicConfig( level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s', handlers=handlers ) + # Only setup logging if enabled via environment variable if os.environ.get('ARDUINO_FRAMEWORK_ENABLE_LOGGING'): setup_logging() @@ -85,19 +87,23 @@ def setup_logging(): 'long_path_warning_shown': False } + def get_platform_default_threshold(mcu): """ - Platform-specific bleeding edge default values for INCLUDE_PATH_LENGTH_THRESHOLD - These values push the limits for maximum performance and minimal path shortening - + Platform-specific bleeding edge default values for + INCLUDE_PATH_LENGTH_THRESHOLD + These values push the limits for maximum performance and minimal path + shortening + Args: mcu: MCU type (esp32, esp32s2, esp32s3, etc.) - + Returns: int: Platform-specific bleeding edge default threshold """ # Bleeding edge values - pushing Windows command line limits - # Windows CMD has ~32768 character limit, we use aggressive values close to this + # Windows CMD has ~32768 character limit, we use aggressive values close + # to this platform_defaults = { "esp32": 45000, # Standard ESP32 "esp32s2": 43000, # ESP32-S2 @@ -108,31 +114,35 @@ def get_platform_default_threshold(mcu): "esp32h2": 40000, # ESP32-H2 "esp32p4": 50000, # ESP32-P4 } - + default_value = platform_defaults.get(mcu, 45000) # Aggressive fallback - + # Debug output only in verbose mode if logging.getLogger().isEnabledFor(logging.DEBUG): - logging.debug(f"Bleeding edge platform default threshold for {mcu}: {default_value}") - + logging.debug( + f"Bleeding edge platform default threshold for {mcu}: " + f"{default_value}") + return default_value + def validate_threshold(threshold, mcu): """ Validates threshold value with bleeding edge limits Uses aggressive boundaries for maximum performance - + Args: threshold: Threshold value to validate mcu: MCU type for context-specific validation - + Returns: int: Validated threshold value """ # Bleeding edge absolute limits - pushing boundaries min_threshold = 15000 # Minimum reasonable value for complex projects - max_threshold = 65000 # Maximum aggressive value (beyond Windows CMD limit for testing) - + # Maximum aggressive value (beyond Windows CMD limit for testing) + max_threshold = 65000 + # MCU-specific bleeding edge adjustments - all values are aggressive mcu_adjustments = { "esp32c2": {"min": 30000, "max": 40000}, @@ -144,56 +154,63 @@ def validate_threshold(threshold, mcu): "esp32c6": {"min": 30000, "max": 50000}, "esp32h2": {"min": 30000, "max": 40000}, } - + # Apply MCU-specific bleeding edge limits if mcu in mcu_adjustments: min_threshold = max(min_threshold, mcu_adjustments[mcu]["min"]) max_threshold = min(max_threshold, mcu_adjustments[mcu]["max"]) - + original_threshold = threshold - + if threshold < min_threshold: - print(f"*** Warning: Include path threshold {threshold} too conservative for {mcu}, using bleeding edge minimum {min_threshold} ***") + print(f"*** Warning: Include path threshold {threshold} too " + f"conservative for {mcu}, using bleeding edge minimum " + f"{min_threshold} ***") threshold = min_threshold elif threshold > max_threshold: - print(f"*** Warning: Include path threshold {threshold} exceeds bleeding edge maximum for {mcu}, using {max_threshold} ***") + print(f"*** Warning: Include path threshold {threshold} exceeds " + f"bleeding edge maximum for {mcu}, using {max_threshold} ***") threshold = max_threshold - + # Warning for conservative values (opposite of original - warn if too low) platform_default = get_platform_default_threshold(mcu) if threshold < platform_default * 0.7: # More than 30% below bleeding edge default - print(f"*** Info: Include path threshold {threshold} is conservative compared to bleeding edge default {platform_default} for {mcu} ***") + print(f"*** Info: Include path threshold {threshold} is conservative " + f"compared to bleeding edge default {platform_default} for " + f"{mcu} ***") print("*** Consider using higher values for maximum performance ***") - + if original_threshold != threshold: - logging.warning(f"Threshold adjusted from {original_threshold} to bleeding edge value {threshold} for {mcu}") - + logging.warning(f"Threshold adjusted from {original_threshold} to " + f"bleeding edge value {threshold} for {mcu}") + return threshold + def get_include_path_threshold(env, config, current_env_section): """ Determines Windows INCLUDE_PATH_LENGTH_THRESHOLD from various sources with priority order and bleeding edge validation - + Priority order: 1. Environment variable PLATFORMIO_INCLUDE_PATH_THRESHOLD 2. Environment-specific setting in platformio.ini 3. Global setting in [env] section 4. Setting in [platformio] section 5. MCU-specific bleeding edge default value - + Args: env: PlatformIO Environment config: Project Configuration current_env_section: Current environment section - + Returns: int: Validated bleeding edge threshold value """ mcu = env.BoardConfig().get("build.mcu", "esp32") default_threshold = get_platform_default_threshold(mcu) setting_name = "custom_include_path_length_threshold" - + try: # 1. Check environment variable (highest priority) env_var = os.environ.get("PLATFORMIO_INCLUDE_PATH_THRESHOLD") @@ -201,58 +218,69 @@ def get_include_path_threshold(env, config, current_env_section): try: threshold = int(env_var) threshold = validate_threshold(threshold, mcu) - print(f"*** Using environment variable bleeding edge include path threshold: {threshold} (MCU: {mcu}) ***") + print(f"*** Using environment variable bleeding edge include " + f"path threshold: {threshold} (MCU: {mcu}) ***") return threshold except ValueError: - print(f"*** Warning: Invalid environment variable PLATFORMIO_INCLUDE_PATH_THRESHOLD='{env_var}', ignoring ***") - + print(f"*** Warning: Invalid environment variable " + f"PLATFORMIO_INCLUDE_PATH_THRESHOLD='{env_var}', " + f"ignoring ***") + # 2. Check environment-specific setting if config.has_option(current_env_section, setting_name): threshold = config.getint(current_env_section, setting_name) threshold = validate_threshold(threshold, mcu) - print(f"*** Using environment-specific bleeding edge include path threshold: {threshold} (MCU: {mcu}) ***") + print(f"*** Using environment-specific bleeding edge include " + f"path threshold: {threshold} (MCU: {mcu}) ***") return threshold - + # 3. Check global setting in [env] section if config.has_option("env", setting_name): threshold = config.getint("env", setting_name) threshold = validate_threshold(threshold, mcu) - print(f"*** Using global [env] bleeding edge include path threshold: {threshold} (MCU: {mcu}) ***") + print(f"*** Using global [env] bleeding edge include path " + f"threshold: {threshold} (MCU: {mcu}) ***") return threshold - + # 4. Check setting in [platformio] section if config.has_option("platformio", setting_name): threshold = config.getint("platformio", setting_name) threshold = validate_threshold(threshold, mcu) - print(f"*** Using [platformio] section bleeding edge include path threshold: {threshold} (MCU: {mcu}) ***") + print(f"*** Using [platformio] section bleeding edge include " + f"path threshold: {threshold} (MCU: {mcu}) ***") return threshold - + # 5. Use MCU-specific bleeding edge default value threshold = validate_threshold(default_threshold, mcu) if env.get("VERBOSE"): - print(f"*** Using platform-specific bleeding edge default include path threshold: {threshold} (MCU: {mcu}) ***") - + print(f"*** Using platform-specific bleeding edge default " + f"include path threshold: {threshold} (MCU: {mcu}) ***") + return threshold - + except (ValueError, TypeError) as e: - print(f"*** Warning: Invalid include path threshold value, using bleeding edge platform default {default_threshold} for {mcu}: {e} ***") + print(f"*** Warning: Invalid include path threshold value, using " + f"bleeding edge platform default {default_threshold} for " + f"{mcu}: {e} ***") return validate_threshold(default_threshold, mcu) + def get_threshold_info(env, config, current_env_section): """ - Helper function for debug information about bleeding edge threshold configuration - + Helper function for debug information about bleeding edge threshold + configuration + Args: env: PlatformIO Environment - config: Project Configuration + config: Project Configuration current_env_section: Current environment section - + Returns: dict: Information about threshold configuration """ mcu = env.BoardConfig().get("build.mcu", "esp32") setting_name = "custom_include_path_length_threshold" - + info = { "mcu": mcu, "platform_default": get_platform_default_threshold(mcu), @@ -264,23 +292,26 @@ def get_threshold_info(env, config, current_env_section): "source": "bleeding_edge_platform_default", "is_bleeding_edge": True } - + # Collect all possible sources if config.has_option(current_env_section, setting_name): with suppress(ValueError): - info["env_specific"] = config.getint(current_env_section, setting_name) - + info["env_specific"] = config.getint(current_env_section, + setting_name) + if config.has_option("env", setting_name): with suppress(ValueError): info["global_env"] = config.getint("env", setting_name) if config.has_option("platformio", setting_name): with suppress(ValueError): - info["platformio_section"] = config.getint("platformio", setting_name) - + info["platformio_section"] = config.getint("platformio", + setting_name) + # Determine final threshold and source - info["final_threshold"] = get_include_path_threshold(env, config, current_env_section) - + info["final_threshold"] = get_include_path_threshold(env, config, + current_env_section) + # Determine source if info["env_variable"]: info["source"] = "environment_variable" @@ -290,9 +321,10 @@ def get_threshold_info(env, config, current_env_section): info["source"] = "global_env" elif info["platformio_section"] is not None: info["source"] = "platformio_section" - + return info + # Cache class for frequently used paths class PathCache: def __init__(self, platform, mcu): @@ -301,20 +333,22 @@ def __init__(self, platform, mcu): self._framework_dir = None self._framework_lib_dir = None self._sdk_dir = None - + @property def framework_dir(self): if self._framework_dir is None: - self._framework_dir = self.platform.get_package_dir("framework-arduinoespressif32") + self._framework_dir = self.platform.get_package_dir( + "framework-arduinoespressif32") return self._framework_dir @property def framework_lib_dir(self): if self._framework_lib_dir is None: - self._framework_lib_dir = self.platform.get_package_dir("framework-arduinoespressif32-libs") + self._framework_lib_dir = self.platform.get_package_dir( + "framework-arduinoespressif32-libs") return self._framework_lib_dir - @property + @property def sdk_dir(self): if self._sdk_dir is None: self._sdk_dir = fs.to_unix_path( @@ -322,12 +356,14 @@ def sdk_dir(self): ) return self._sdk_dir + def check_and_warn_long_path_support(): """Checks Windows long path support and issues warning if disabled""" with _PATH_SHORTENING_LOCK: # Thread-safe access - if not IS_WINDOWS or _PATH_SHORTENING_MESSAGES['long_path_warning_shown']: + if not IS_WINDOWS or _PATH_SHORTENING_MESSAGES[ + 'long_path_warning_shown']: return - + try: import winreg key = winreg.OpenKey( @@ -336,52 +372,58 @@ def check_and_warn_long_path_support(): ) value, _ = winreg.QueryValueEx(key, "LongPathsEnabled") winreg.CloseKey(key) - + if value != 1: print("*** WARNING: Windows Long Path Support is disabled ***") print("*** Enable it for better performance: ***") print("*** 1. Run as Administrator: gpedit.msc ***") - print("*** 2. Navigate to: Computer Configuration > Administrative Templates > System > Filesystem ***") + print("*** 2. Navigate to: Computer Configuration > " + "Administrative Templates > System > Filesystem ***") print("*** 3. Enable 'Enable Win32 long paths' ***") print("*** OR run PowerShell as Admin: ***") - print("*** New-ItemProperty -Path 'HKLM:\\SYSTEM\\CurrentControlSet\\Control\\FileSystem' -Name 'LongPathsEnabled' -Value 1 -PropertyType DWORD -Force ***") + print("*** New-ItemProperty -Path " + "'HKLM:\\SYSTEM\\CurrentControlSet\\Control\\FileSystem' " + "-Name 'LongPathsEnabled' -Value 1 -PropertyType DWORD " + "-Force ***") print("*** Restart required after enabling ***") except Exception: print("*** WARNING: Could not check Long Path Support status ***") - print("*** Consider enabling Windows Long Path Support for better performance ***") - + print("*** Consider enabling Windows Long Path Support for " + "better performance ***") + _PATH_SHORTENING_MESSAGES['long_path_warning_shown'] = True + # Secure deletion functions -def safe_delete_file(file_path: Union[str, Path], - force: bool = False) -> bool: +def safe_delete_file(file_path: Union[str, Path], + force: bool = False) -> bool: """ Secure file deletion - + Args: file_path: Path to file to be deleted force: Forces deletion even for write-protected files - + Returns: bool: True if successfully deleted """ file_path = Path(file_path) - + try: # Check existence if not file_path.exists(): logging.warning(f"File does not exist: {file_path}") return False - + # Remove write protection if necessary if force and not os.access(file_path, os.W_OK): file_path.chmod(0o666) - + # Delete file file_path.unlink() logging.info(f"File deleted: {file_path}") return True - + except PermissionError: logging.error(f"No permission to delete: {file_path}") return False @@ -389,25 +431,27 @@ def safe_delete_file(file_path: Union[str, Path], logging.error(f"Error deleting {file_path}: {e}") return False + def safe_delete_directory(dir_path: Union[str, Path]) -> bool: """ Secure directory deletion """ dir_path = Path(dir_path) - + try: if not dir_path.exists(): logging.warning(f"Directory does not exist: {dir_path}") return False - + shutil.rmtree(dir_path) logging.info(f"Directory deleted: {dir_path}") return True - + except Exception as e: logging.error(f"Error deleting {dir_path}: {e}") return False + def validate_platformio_path(path: Union[str, Path]) -> bool: """ Enhanced validation for PlatformIO package paths @@ -415,46 +459,48 @@ def validate_platformio_path(path: Union[str, Path]) -> bool: try: path = Path(path).resolve() path_str = str(path) - + # Must be within .platformio directory structure if ".platformio" not in path_str: return False - + # Must be a packages directory if "packages" not in path_str: return False - + # Must be framework-related framework_indicators = [ "framework-arduinoespressif32", "framework-arduinoespressif32-libs" ] - + if not any(indicator in path_str for indicator in framework_indicators): return False - + # Must not be a critical system path - critical_paths = ["/usr", "/bin", "/sbin", "/etc", "/boot", "C:\\Windows", "C:\\Program Files"] + critical_paths = ["/usr", "/bin", "/sbin", "/etc", "/boot", + "C:\\Windows", "C:\\Program Files"] return not any(critical in path_str for critical in critical_paths) - + except Exception as e: logging.error(f"Path validation error: {e}") return False -def validate_deletion_path(path: Union[str, Path], - allowed_patterns: List[str]) -> bool: + +def validate_deletion_path(path: Union[str, Path], + allowed_patterns: List[str]) -> bool: """ Validates if a path can be safely deleted - + Args: path: Path to be checked allowed_patterns: Allowed path patterns - + Returns: bool: True if deletion is safe """ path = Path(path).resolve() - + # Check against critical system paths critical_paths = [ Path.home(), @@ -465,77 +511,88 @@ def validate_deletion_path(path: Union[str, Path], Path("/bin"), Path("/sbin") ] - + for critical in filter(None, critical_paths): try: normalized_path = path.resolve() normalized_critical = critical.resolve() - if normalized_path == normalized_critical or normalized_critical in normalized_path.parents: + if (normalized_path == normalized_critical or + normalized_critical in normalized_path.parents): logging.error(f"Critical system path detected: {path}") return False except (OSError, ValueError): # Path comparison failed, reject for safety logging.error(f"Path comparison failed for: {path}") return False - + # Check against allowed patterns path_str = str(path) is_allowed = any(pattern in path_str for pattern in allowed_patterns) - + if not is_allowed: logging.error(f"Path does not match allowed patterns: {path}") logging.error(f"Allowed patterns: {allowed_patterns}") else: logging.info(f"Path validation successful: {path}") - + return is_allowed + def safe_framework_cleanup(): """Secure cleanup of Arduino Framework with enhanced error handling""" success = True - + # Framework directory cleanup if exists(FRAMEWORK_DIR): - logging.info(f"Attempting to validate framework path: {FRAMEWORK_DIR}") - + logging.info(f"Attempting to validate framework path: " + f"{FRAMEWORK_DIR}") + if validate_platformio_path(FRAMEWORK_DIR): - logging.info(f"Framework path validated successfully: {FRAMEWORK_DIR}") - + logging.info(f"Framework path validated successfully: " + f"{FRAMEWORK_DIR}") + if safe_delete_directory(FRAMEWORK_DIR): print("Framework successfully removed") else: print("Error removing framework") success = False else: - logging.error(f"PlatformIO path validation failed: {FRAMEWORK_DIR}") + logging.error(f"PlatformIO path validation failed: " + f"{FRAMEWORK_DIR}") success = False - + # Framework libs directory cleanup if exists(FRAMEWORK_LIB_DIR): - logging.info(f"Attempting to validate framework lib path: {FRAMEWORK_LIB_DIR}") - + logging.info(f"Attempting to validate framework lib path: " + f"{FRAMEWORK_LIB_DIR}") + if validate_platformio_path(FRAMEWORK_LIB_DIR): - logging.info(f"Framework lib path validated successfully: {FRAMEWORK_LIB_DIR}") - + logging.info(f"Framework lib path validated successfully: " + f"{FRAMEWORK_LIB_DIR}") + if safe_delete_directory(FRAMEWORK_LIB_DIR): print("Framework libs successfully removed") else: print("Error removing framework libs") success = False else: - logging.error(f"PlatformIO path validation failed: {FRAMEWORK_LIB_DIR}") + logging.error(f"PlatformIO path validation failed: " + f"{FRAMEWORK_LIB_DIR}") success = False - + return success + def safe_remove_sdkconfig_files(): """Secure removal of SDKConfig files""" - envs = [section.replace("env:", "") for section in config.sections() if section.startswith("env:")] + envs = [section.replace("env:", "") for section in config.sections() + if section.startswith("env:")] for env_name in envs: file_path = join(project_dir, f"sdkconfig.{env_name}") if exists(file_path): safe_delete_file(file_path) + # Initialization env = DefaultEnvironment() pm = ToolPackageManager() @@ -590,22 +647,28 @@ def safe_remove_sdkconfig_files(): SConscript("_embed_files.py", exports="env") -flag_any_custom_sdkconfig = exists(join(platform.get_package_dir("framework-arduinoespressif32-libs"),"sdkconfig")) +flag_any_custom_sdkconfig = exists(join( + platform.get_package_dir("framework-arduinoespressif32-libs"), + "sdkconfig")) + def has_unicore_flags(): """Check if any UNICORE flags are present in configuration""" - return any(flag in extra_flags or flag in entry_custom_sdkconfig + return any(flag in extra_flags or flag in entry_custom_sdkconfig or flag in board_sdkconfig for flag in UNICORE_FLAGS) + # Esp32-solo1 libs settings if flag_custom_sdkconfig and has_unicore_flags(): if not env.get('BUILD_UNFLAGS'): # Initialize if not set env['BUILD_UNFLAGS'] = [] - - build_unflags = " ".join(env['BUILD_UNFLAGS']) + " -mdisable-hardware-atomics -ustart_app_other_cores" + + build_unflags = (" ".join(env['BUILD_UNFLAGS']) + + " -mdisable-hardware-atomics -ustart_app_other_cores") new_build_unflags = build_unflags.split() env.Replace(BUILD_UNFLAGS=new_build_unflags) + def get_packages_to_install(deps, installed_packages): """Generator for packages to install""" for package, spec in deps.items(): @@ -616,27 +679,32 @@ def get_packages_to_install(deps, installed_packages): if not version_spec.match(installed_packages[package]): yield package + def install_python_deps(): def _get_installed_pip_packages(): result = {} try: pip_output = subprocess.check_output([ env.subst("$PYTHONEXE"), - "-m", "pip", "list", "--format=json", "--disable-pip-version-check" + "-m", "pip", "list", "--format=json", + "--disable-pip-version-check" ]) packages = json.loads(pip_output) for p in packages: result[p["name"]] = pepver_to_semver(p["version"]) except Exception: - print("Warning! Couldn't extract the list of installed Python packages.") - + print("Warning! Couldn't extract the list of installed Python " + "packages.") + return result installed_packages = _get_installed_pip_packages() - packages_to_install = list(get_packages_to_install(python_deps, installed_packages)) + packages_to_install = list(get_packages_to_install(python_deps, + installed_packages)) if packages_to_install: - packages_str = " ".join(f'"{p}{python_deps[p]}"' for p in packages_to_install) + packages_str = " ".join(f'"{p}{python_deps[p]}"' + for p in packages_to_install) env.Execute( env.VerboseAction( f'"$PYTHONEXE" -m pip install -U -q -q -q {packages_str}', @@ -644,25 +712,28 @@ def _get_installed_pip_packages(): ) ) + install_python_deps() + def get_MD5_hash(phrase): return hashlib.md5(phrase.encode('utf-8')).hexdigest()[:16] + def matching_custom_sdkconfig(): """Checks if current environment matches existing sdkconfig""" cust_sdk_is_present = False - + if not flag_any_custom_sdkconfig: return True, cust_sdk_is_present - + last_sdkconfig_path = join(project_dir, "sdkconfig.defaults") if not exists(last_sdkconfig_path): return False, cust_sdk_is_present - + if not flag_custom_sdkconfig: return False, cust_sdk_is_present - + try: with open(last_sdkconfig_path) as src: line = src.readline() @@ -677,55 +748,66 @@ def matching_custom_sdkconfig(): return False, cust_sdk_is_present + def check_reinstall_frwrk(): if not flag_custom_sdkconfig and flag_any_custom_sdkconfig: # case custom sdkconfig exists and an env without "custom_sdkconfig" return True - + if flag_custom_sdkconfig: matching_sdkconfig, _ = matching_custom_sdkconfig() if not matching_sdkconfig: # check if current custom sdkconfig is different from existing return True - + return False + def call_compile_libs(): print(f"*** Compile Arduino IDF libs for {pioenv} ***") SConscript("espidf.py") + FRAMEWORK_SDK_DIR = path_cache.sdk_dir IS_INTEGRATION_DUMP = env.IsIntegrationDump() + def is_framework_subfolder(potential_subfolder): """Check if a path is a subfolder of the framework SDK directory""" # carefully check before change this function if not isabs(potential_subfolder): return False - if splitdrive(FRAMEWORK_SDK_DIR)[0] != splitdrive(potential_subfolder)[0]: + if (splitdrive(FRAMEWORK_SDK_DIR)[0] != + splitdrive(potential_subfolder)[0]): return False - return commonpath([FRAMEWORK_SDK_DIR]) == commonpath([FRAMEWORK_SDK_DIR, potential_subfolder]) + return (commonpath([FRAMEWORK_SDK_DIR]) == + commonpath([FRAMEWORK_SDK_DIR, potential_subfolder])) + # Performance optimization with caching def calculate_include_path_length(includes): """Calculate total character count of all include paths with caching""" if not hasattr(calculate_include_path_length, '_cache'): calculate_include_path_length._cache = {} - + cache_key = tuple(includes) if cache_key not in calculate_include_path_length._cache: - calculate_include_path_length._cache[cache_key] = sum(len(str(inc)) for inc in includes) - + calculate_include_path_length._cache[cache_key] = sum( + len(str(inc)) for inc in includes) + return calculate_include_path_length._cache[cache_key] + def analyze_path_distribution(includes): - """Analyze the distribution of include path lengths for optimization insights""" + """Analyze the distribution of include path lengths for optimization + insights""" if not includes: return {} - + lengths = [len(str(inc)) for inc in includes] - framework_lengths = [len(str(inc)) for inc in includes if is_framework_subfolder(inc)] - + framework_lengths = [len(str(inc)) for inc in includes + if is_framework_subfolder(inc)] + return { 'total_paths': len(includes), 'total_length': sum(lengths), @@ -734,14 +816,17 @@ def analyze_path_distribution(includes): 'min_length': min(lengths), 'framework_paths': len(framework_lengths), 'framework_total_length': sum(framework_lengths), - 'framework_avg_length': sum(framework_lengths) / len(framework_lengths) if framework_lengths else 0 + 'framework_avg_length': (sum(framework_lengths) / + len(framework_lengths) + if framework_lengths else 0) } + def debug_framework_paths(env, include_count, total_length): """Debug framework paths to understand the issue (verbose mode only)""" if not env.get("VERBOSE"): return - + print("*** Debug Framework Paths ***") print(f"*** MCU: {mcu} ***") print(f"*** FRAMEWORK_DIR: {FRAMEWORK_DIR} ***") @@ -749,23 +834,28 @@ def debug_framework_paths(env, include_count, total_length): print(f"*** SDK exists: {exists(FRAMEWORK_SDK_DIR)} ***") print(f"*** Include count: {include_count} ***") print(f"*** Total path length: {total_length} ***") - + includes = env.get("CPPPATH", []) framework_count = 0 longest_paths = sorted(includes, key=len, reverse=True)[:5] - + print("*** Longest include paths: ***") for i, inc in enumerate(longest_paths): is_fw = is_framework_subfolder(inc) if is_fw: framework_count += 1 - print(f"*** {i+1}: {inc} (length: {len(str(inc))}) -> Framework: {is_fw} ***") - - print(f"*** Framework includes found: {framework_count}/{len(includes)} ***") - + print(f"*** {i+1}: {inc} (length: {len(str(inc))}) -> " + f"Framework: {is_fw} ***") + + print(f"*** Framework includes found: {framework_count}/" + f"{len(includes)} ***") + # Show path distribution analysis analysis = analyze_path_distribution(includes) - print(f"*** Path Analysis: Avg={analysis.get('average_length', 0):.1f}, Max={analysis.get('max_length', 0)}, Framework Avg={analysis.get('framework_avg_length', 0):.1f} ***") + print(f"*** Path Analysis: Avg={analysis.get('average_length', 0):.1f}, " + f"Max={analysis.get('max_length', 0)}, " + f"Framework Avg={analysis.get('framework_avg_length', 0):.1f} ***") + def apply_include_shortening(env, node, includes, total_length): """Applies include path shortening technique""" @@ -773,20 +863,20 @@ def apply_include_shortening(env, node, includes, total_length): to_unix_path = fs.to_unix_path ccflags = env["CCFLAGS"] asflags = env["ASFLAGS"] - + includes = [to_unix_path(inc) for inc in env_get("CPPPATH", [])] shortened_includes = [] generic_includes = [] - + original_length = total_length saved_chars = 0 - + for inc in includes: if is_framework_subfolder(inc): relative_path = to_unix_path(relpath(inc, FRAMEWORK_SDK_DIR)) shortened_path = "-iwithprefix/" + relative_path shortened_includes.append(shortened_path) - + # Calculate character savings # Original: full path in -I flag # New: -iprefix + shortened relative path @@ -800,21 +890,31 @@ def apply_include_shortening(env, node, includes, total_length): with _PATH_SHORTENING_LOCK: if not _PATH_SHORTENING_MESSAGES['shortening_applied']: if shortened_includes: - removed_i_flags = len(shortened_includes) * 2 # Each -I is 2 chars - new_total_length = original_length - saved_chars + len(f"-iprefix{FRAMEWORK_SDK_DIR}") - removed_i_flags - print(f"*** Applied include path shortening for {len(shortened_includes)} framework paths ***") - print(f"*** Path length reduced from {original_length} to ~{new_total_length} characters ***") + # Each -I is 2 chars + removed_i_flags = len(shortened_includes) * 2 + new_total_length = (original_length - saved_chars + + len(f"-iprefix{FRAMEWORK_SDK_DIR}") - + removed_i_flags) + print(f"*** Applied include path shortening for " + f"{len(shortened_includes)} framework paths ***") + print(f"*** Path length reduced from {original_length} to " + f"~{new_total_length} characters ***") print(f"*** Estimated savings: {saved_chars} characters ***") else: - if not _PATH_SHORTENING_MESSAGES['no_framework_paths_warning']: - print("*** Warning: Path length high but no framework paths found for shortening ***") - print("*** This may indicate an architecture-specific issue ***") - print("*** Run with -v (verbose) for detailed path analysis ***") - _PATH_SHORTENING_MESSAGES['no_framework_paths_warning'] = True + if not _PATH_SHORTENING_MESSAGES[ + 'no_framework_paths_warning']: + print("*** Warning: Path length high but no framework " + "paths found for shortening ***") + print("*** This may indicate an architecture-specific " + "issue ***") + print("*** Run with -v (verbose) for detailed path " + "analysis ***") + _PATH_SHORTENING_MESSAGES[ + 'no_framework_paths_warning'] = True _PATH_SHORTENING_MESSAGES['shortening_applied'] = True common_flags = ["-iprefix", FRAMEWORK_SDK_DIR] + shortened_includes - + return env.Object( node, CPPPATH=generic_includes, @@ -822,36 +922,42 @@ def apply_include_shortening(env, node, includes, total_length): ASFLAGS=asflags + common_flags, ) + def smart_include_length_shorten(env, node): """ - Include path shortening based on bleeding edge configurable threshold with enhanced MCU support + Include path shortening based on bleeding edge configurable threshold + with enhanced MCU support Uses aggressive thresholds for maximum performance """ if IS_INTEGRATION_DUMP: return node - + if not IS_WINDOWS: return env.Object(node) - + # Get dynamically configurable bleeding edge threshold - include_path_threshold = get_include_path_threshold(env, config, current_env_section) - + include_path_threshold = get_include_path_threshold(env, config, + current_env_section) + check_and_warn_long_path_support() - + includes = env.get("CPPPATH", []) include_count = len(includes) total_path_length = calculate_include_path_length(includes) - + # Debug information in verbose mode if env.get("VERBOSE"): debug_framework_paths(env, include_count, total_path_length) - - # Extended debug information about bleeding edge threshold configuration + + # Extended debug information about bleeding edge threshold + # configuration threshold_info = get_threshold_info(env, config, current_env_section) print("*** Bleeding Edge Threshold Configuration Debug ***") print(f"*** MCU: {threshold_info['mcu']} ***") - print(f"*** Bleeding Edge Platform Default: {threshold_info['platform_default']} ***") - print(f"*** Final Bleeding Edge Threshold: {threshold_info['final_threshold']} ***") + print(f"*** Bleeding Edge Platform Default: " + f"{threshold_info['platform_default']} ***") + print(f"*** Final Bleeding Edge Threshold: " + f"{threshold_info['final_threshold']} ***") print(f"*** Source: {threshold_info['source']} ***") print("*** Performance Mode: Maximum Aggressive ***") if threshold_info['env_variable']: @@ -861,40 +967,46 @@ def smart_include_length_shorten(env, node): if threshold_info['global_env']: print(f"*** Global Env: {threshold_info['global_env']} ***") if threshold_info['platformio_section']: - print(f"*** PlatformIO Section: {threshold_info['platformio_section']} ***") - + print(f"*** PlatformIO Section: " + f"{threshold_info['platformio_section']} ***") + # Use the configurable and validated bleeding edge threshold if total_path_length <= include_path_threshold: return env.Object(node) - + return apply_include_shortening(env, node, includes, total_path_length) + def get_frameworks_in_current_env(): """Determines the frameworks of the current environment""" if "framework" in config.options(current_env_section): return config.get(current_env_section, "framework", "") return [] + # Framework check current_env_frameworks = get_frameworks_in_current_env() if "arduino" in current_env_frameworks and "espidf" in current_env_frameworks: # Arduino as component is set, switch off Hybrid compile flag_custom_sdkconfig = False -# Framework reinstallation if required - Enhanced with secure deletion and error handling +# Framework reinstallation if required - Enhanced with secure deletion and +# error handling if check_reinstall_frwrk(): # Secure removal of SDKConfig files safe_remove_sdkconfig_files() - + print("*** Reinstall Arduino framework ***") - + # Secure framework cleanup with enhanced error handling if safe_framework_cleanup(): - arduino_frmwrk_url = str(platform.get_package_spec("framework-arduinoespressif32")).split("uri=", 1)[1][:-1] - arduino_frmwrk_lib_url = str(platform.get_package_spec("framework-arduinoespressif32-libs")).split("uri=",1)[1][:-1] + arduino_frmwrk_url = str(platform.get_package_spec( + "framework-arduinoespressif32")).split("uri=", 1)[1][:-1] + arduino_frmwrk_lib_url = str(platform.get_package_spec( + "framework-arduinoespressif32-libs")).split("uri=", 1)[1][:-1] pm.install(arduino_frmwrk_url) pm.install(arduino_frmwrk_lib_url) - + if flag_custom_sdkconfig: call_compile_libs() flag_custom_sdkconfig = False @@ -909,19 +1021,21 @@ def get_frameworks_in_current_env(): pioframework = env.subst("$PIOFRAMEWORK") arduino_lib_compile_flag = env.subst("$ARDUINO_LIB_COMPILE_FLAG") -if ("arduino" in pioframework and "espidf" not in pioframework and - arduino_lib_compile_flag in ("Inactive", "True")): +if ("arduino" in pioframework and "espidf" not in pioframework and + arduino_lib_compile_flag in ("Inactive", "True")): # try to remove not needed include path if an lib_ignore entry exists from component_manager import ComponentManager component_manager = ComponentManager(env) component_manager.handle_component_settings() silent_action = env.Action(component_manager.restore_pioarduino_build_py) - silent_action.strfunction = lambda target, source, env: '' # hack to silence scons command output + # hack to silence scons command output + silent_action.strfunction = lambda target, source, env: '' env.AddPostAction("checkprogsize", silent_action) if IS_WINDOWS: - # Smart include path optimization based on bleeding edge configurable threshold + # Smart include path optimization based on bleeding edge configurable + # threshold env.AddBuildMiddleware(smart_include_length_shorten) - - build_script_path = join(FRAMEWORK_DIR, "tools", "pioarduino-build.py") + + build_script_path = join(FRAMEWORK_DIR, "tools", "pioarduino-build.py") SConscript(build_script_path) From 3abef6b9197212867143510f942dd3e617a3334c Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Sat, 31 May 2025 14:35:46 +0200 Subject: [PATCH 81/97] updated doc strings and Python style code formatting --- platform.py | 155 ++++++++++++++++++++++++++++++++-------------------- 1 file changed, 96 insertions(+), 59 deletions(-) diff --git a/platform.py b/platform.py index aa8baeb2a..12152880b 100644 --- a/platform.py +++ b/platform.py @@ -36,9 +36,11 @@ DEFAULT_APP_OFFSET = "0x10000" # MCUs that support ESP-builtin debug -ESP_BUILTIN_DEBUG_MCUS = frozenset(["esp32c3", "esp32c5", "esp32c6", "esp32s3", "esp32h2", "esp32p4"]) +ESP_BUILTIN_DEBUG_MCUS = frozenset([ + "esp32c3", "esp32c5", "esp32c6", "esp32s3", "esp32h2", "esp32p4" +]) -# MCU configuration +# MCU configuration mapping MCU_TOOLCHAIN_CONFIG = { "xtensa": { "mcus": frozenset(["esp32", "esp32s2", "esp32s3"]), @@ -46,7 +48,9 @@ "debug_tools": ["tool-xtensa-esp-elf-gdb"] }, "riscv": { - "mcus": frozenset(["esp32c2", "esp32c3", "esp32c5", "esp32c6", "esp32h2", "esp32p4"]), + "mcus": frozenset([ + "esp32c2", "esp32c3", "esp32c5", "esp32c6", "esp32h2", "esp32p4" + ]), "toolchains": ["toolchain-riscv32-esp"], "debug_tools": ["tool-riscv32-esp-elf-gdb"] } @@ -82,7 +86,7 @@ def safe_file_operation(operation_func): - """Decorator for safe filesystem operations""" + """Decorator for safe filesystem operations with error handling.""" def wrapper(*args, **kwargs): try: return operation_func(*args, **kwargs) @@ -97,7 +101,7 @@ def wrapper(*args, **kwargs): @safe_file_operation def safe_remove_directory(path: str) -> bool: - """Safely remove directories""" + """Safely remove directories with error handling.""" if os.path.exists(path) and os.path.isdir(path): shutil.rmtree(path) logger.debug(f"Directory removed: {path}") @@ -106,7 +110,7 @@ def safe_remove_directory(path: str) -> bool: @safe_file_operation def safe_copy_file(src: str, dst: str) -> bool: - """Safely copy files""" + """Safely copy files with error handling.""" os.makedirs(os.path.dirname(dst), exist_ok=True) shutil.copyfile(src, dst) logger.debug(f"File copied: {src} -> {dst}") @@ -114,7 +118,10 @@ def safe_copy_file(src: str, dst: str) -> bool: class Espressif32Platform(PlatformBase): + """ESP32 platform implementation for PlatformIO with optimized toolchain management.""" + def __init__(self, *args, **kwargs): + """Initialize the ESP32 platform with caching mechanisms.""" super().__init__(*args, **kwargs) self._packages_dir = None self._tools_cache = {} @@ -122,13 +129,14 @@ def __init__(self, *args, **kwargs): @property def packages_dir(self) -> str: - """Cached packages directory""" + """Get cached packages directory path.""" if self._packages_dir is None: - self._packages_dir = ProjectConfig.get_instance().get("platformio", "packages_dir") + config = ProjectConfig.get_instance() + self._packages_dir = config.get("platformio", "packages_dir") return self._packages_dir def _get_tool_paths(self, tool_name: str) -> Dict[str, str]: - """Central path calculation for tools""" + """Get centralized path calculation for tools with caching.""" if tool_name not in self._tools_cache: tool_path = os.path.join(self.packages_dir, tool_name) self._tools_cache[tool_name] = { @@ -136,12 +144,14 @@ def _get_tool_paths(self, tool_name: str) -> Dict[str, str]: 'package_path': os.path.join(tool_path, "package.json"), 'tools_json_path': os.path.join(tool_path, "tools.json"), 'piopm_path': os.path.join(tool_path, ".piopm"), - 'idf_tools_path': os.path.join(self.packages_dir, "tl-install", "tools", "idf_tools.py") + 'idf_tools_path': os.path.join( + self.packages_dir, "tl-install", "tools", "idf_tools.py" + ) } return self._tools_cache[tool_name] def _check_tool_status(self, tool_name: str) -> Dict[str, bool]: - """Check tool status""" + """Check the installation status of a tool.""" paths = self._get_tool_paths(tool_name) return { 'has_idf_tools': os.path.exists(paths['idf_tools_path']), @@ -151,7 +161,7 @@ def _check_tool_status(self, tool_name: str) -> Dict[str, bool]: } def _run_idf_tools_install(self, tools_json_path: str, idf_tools_path: str) -> bool: - """Execute idf_tools.py install""" + """Execute idf_tools.py install command with timeout and error handling.""" cmd = [ python_exe, idf_tools_path, @@ -186,7 +196,7 @@ def _run_idf_tools_install(self, tools_json_path: str, idf_tools_path: str) -> b return False def _check_tool_version(self, tool_name: str) -> bool: - """Check tool version""" + """Check if the installed tool version matches the required version.""" paths = self._get_tool_paths(tool_name) try: @@ -206,7 +216,10 @@ def _check_tool_version(self, tool_name: str) -> bool: version_match = required_version == installed_version if not version_match: - logger.info(f"Version mismatch for {tool_name}: {installed_version} != {required_version}") + logger.info( + f"Version mismatch for {tool_name}: " + f"{installed_version} != {required_version}" + ) return version_match @@ -215,9 +228,11 @@ def _check_tool_version(self, tool_name: str) -> bool: return False def install_tool(self, tool_name: str, retry_count: int = 0) -> bool: - """Optimized tool installation""" + """Install a tool with optimized retry mechanism.""" if retry_count >= RETRY_LIMIT: - logger.error(f"Installation of {tool_name} failed after {RETRY_LIMIT} attempts") + logger.error( + f"Installation of {tool_name} failed after {RETRY_LIMIT} attempts" + ) return False self.packages[tool_name]["optional"] = False @@ -229,20 +244,27 @@ def install_tool(self, tool_name: str, retry_count: int = 0) -> bool: return self._install_with_idf_tools(tool_name, paths) # Case 2: Tool already installed, version check - if status['has_idf_tools'] and status['has_piopm'] and not status['has_tools_json']: + if (status['has_idf_tools'] and status['has_piopm'] and + not status['has_tools_json']): return self._handle_existing_tool(tool_name, paths, retry_count) logger.debug(f"Tool {tool_name} already configured") return True def _install_with_idf_tools(self, tool_name: str, paths: Dict[str, str]) -> bool: - """Installation with idf_tools.py""" - if not self._run_idf_tools_install(paths['tools_json_path'], paths['idf_tools_path']): + """Install tool using idf_tools.py installation method.""" + if not self._run_idf_tools_install( + paths['tools_json_path'], paths['idf_tools_path'] + ): return False # Copy tool files - tools_path_default = os.path.join(os.path.expanduser("~"), ".platformio") - target_package_path = os.path.join(tools_path_default, "tools", tool_name, "package.json") + tools_path_default = os.path.join( + os.path.expanduser("~"), ".platformio" + ) + target_package_path = os.path.join( + tools_path_default, "tools", tool_name, "package.json" + ) if not safe_copy_file(paths['package_path'], target_package_path): return False @@ -255,8 +277,10 @@ def _install_with_idf_tools(self, tool_name: str, paths: Dict[str, str]) -> bool logger.info(f"Tool {tool_name} successfully installed") return True - def _handle_existing_tool(self, tool_name: str, paths: Dict[str, str], retry_count: int) -> bool: - """Handle already installed tools""" + def _handle_existing_tool( + self, tool_name: str, paths: Dict[str, str], retry_count: int + ) -> bool: + """Handle already installed tools with version checking.""" if self._check_tool_version(tool_name): # Version matches, use tool self.packages[tool_name]["version"] = paths['tool_path'] @@ -270,16 +294,18 @@ def _handle_existing_tool(self, tool_name: str, paths: Dict[str, str], retry_cou return self.install_tool(tool_name, retry_count + 1) def _configure_arduino_framework(self, frameworks: List[str]) -> None: - """Configure Arduino framework""" + """Configure Arduino framework with dynamic library URL fetching.""" if "arduino" not in frameworks: return self.packages["framework-arduinoespressif32"]["optional"] = False self.packages["framework-arduinoespressif32-libs"]["optional"] = False - # use branch master - URL = "https://raw.githubusercontent.com/espressif/arduino-esp32/master/package/package_esp32_index.template.json" + + # Use branch master + url = ("https://raw.githubusercontent.com/espressif/arduino-esp32/" + "master/package/package_esp32_index.template.json") try: - response = requests.get(URL, timeout=30) + response = requests.get(url, timeout=30) response.raise_for_status() packjdata = response.json() dyn_lib_url = packjdata['packages'][0]['tools'][0]['systems'][0]['url'] @@ -287,11 +313,15 @@ def _configure_arduino_framework(self, frameworks: List[str]) -> None: except (requests.RequestException, KeyError, IndexError) as e: logger.error(f"Failed to fetch Arduino framework library URL: {e}") - def _configure_espidf_framework(self, frameworks: List[str], variables: Dict, board_config: Dict, mcu: str) -> None: - """Configure ESP-IDF framework""" + def _configure_espidf_framework( + self, frameworks: List[str], variables: Dict, board_config: Dict, mcu: str + ) -> None: + """Configure ESP-IDF framework based on custom sdkconfig settings.""" custom_sdkconfig = variables.get("custom_sdkconfig") - board_sdkconfig = variables.get("board_espidf.custom_sdkconfig", - board_config.get("espidf.custom_sdkconfig", "")) + board_sdkconfig = variables.get( + "board_espidf.custom_sdkconfig", + board_config.get("espidf.custom_sdkconfig", "") + ) if custom_sdkconfig is not None or len(str(board_sdkconfig)) > 3: frameworks.append("espidf") @@ -300,7 +330,7 @@ def _configure_espidf_framework(self, frameworks: List[str], variables: Dict, bo self.packages["framework-arduino-c2-skeleton-lib"]["optional"] = False def _get_mcu_config(self, mcu: str) -> Optional[Dict]: - """MCU configuration with optimized search""" + """Get MCU configuration with optimized caching and search.""" if mcu in self._mcu_config_cache: return self._mcu_config_cache[mcu] @@ -316,15 +346,17 @@ def _get_mcu_config(self, mcu: str) -> Optional[Dict]: return None def _needs_debug_tools(self, variables: Dict, targets: List[str]) -> bool: - """Check if debug tools are needed""" + """Check if debug tools are needed based on build configuration.""" return bool( variables.get("build_type") or "debug" in targets or variables.get("upload_protocol") ) - def _configure_mcu_toolchains(self, mcu: str, variables: Dict, targets: List[str]) -> None: - """Optimized MCU toolchain configuration""" + def _configure_mcu_toolchains( + self, mcu: str, variables: Dict, targets: List[str] + ) -> None: + """Configure MCU-specific toolchains with optimized installation.""" mcu_config = self._get_mcu_config(mcu) if not mcu_config: logger.warning(f"Unknown MCU: {mcu}") @@ -346,18 +378,20 @@ def _configure_mcu_toolchains(self, mcu: str, variables: Dict, targets: List[str self.install_tool("tool-openocd-esp32") def _configure_installer(self) -> None: - """Configure installer""" - installer_path = os.path.join(self.packages_dir, "tl-install", "tools", "idf_tools.py") + """Configure the ESP-IDF tools installer.""" + installer_path = os.path.join( + self.packages_dir, "tl-install", "tools", "idf_tools.py" + ) if os.path.exists(installer_path): self.packages["tl-install"]["optional"] = True def _install_common_idf_packages(self) -> None: - """Install common IDF packages""" + """Install common ESP-IDF packages required for all builds.""" for package in COMMON_IDF_PACKAGES: self.install_tool(package) def _configure_check_tools(self, variables: Dict) -> None: - """Configure check tools""" + """Configure static analysis and check tools based on configuration.""" check_tools = variables.get("check_tool", []) if not check_tools: return @@ -367,7 +401,7 @@ def _configure_check_tools(self, variables: Dict) -> None: self.install_tool(package) def _ensure_mklittlefs_version(self) -> None: - """Ensure correct mklittlefs version""" + """Ensure correct mklittlefs version is installed.""" piopm_path = os.path.join(self.packages_dir, "tool-mklittlefs", ".piopm") if os.path.exists(piopm_path): @@ -381,9 +415,11 @@ def _ensure_mklittlefs_version(self) -> None: logger.error(f"Error reading mklittlefs package data: {e}") def _setup_mklittlefs_for_download(self) -> None: - """Setup mklittlefs for download functionality""" + """Setup mklittlefs for download functionality with version 4.0.0.""" mklittlefs_dir = os.path.join(self.packages_dir, "tool-mklittlefs") - mklittlefs400_dir = os.path.join(self.packages_dir, "tool-mklittlefs-4.0.0") + mklittlefs400_dir = os.path.join( + self.packages_dir, "tool-mklittlefs-4.0.0" + ) # Ensure mklittlefs 3.2.0 is installed if not os.path.exists(mklittlefs_dir): @@ -406,7 +442,7 @@ def _setup_mklittlefs_for_download(self) -> None: self.packages.pop("tool-mkfatfs", None) def _handle_littlefs_tool(self, for_download: bool) -> None: - """Special handling for LittleFS tools""" + """Handle LittleFS tool installation with special download configuration.""" if for_download: self._setup_mklittlefs_for_download() else: @@ -414,7 +450,7 @@ def _handle_littlefs_tool(self, for_download: bool) -> None: self.install_tool("tool-mklittlefs") def _install_filesystem_tool(self, filesystem: str, for_download: bool = False) -> None: - """Install filesystem tools""" + """Install filesystem-specific tools based on the filesystem type.""" tool_mapping = { "default": lambda: self._handle_littlefs_tool(for_download), "fatfs": lambda: self.install_tool("tool-mkfatfs"), @@ -424,14 +460,14 @@ def _install_filesystem_tool(self, filesystem: str, for_download: bool = False) handler = tool_mapping.get(filesystem, tool_mapping["default"]) handler() - # Currently only Arduino Nano ESP32 uses the dfuutil tool as uploader def _handle_dfuutil_tool(self, variables: Dict, for_download: bool = False) -> None: - """Install dfuutil tool for Arduino Nano Board""" + """Install dfuutil tool for Arduino Nano ESP32 board.""" + # Currently only Arduino Nano ESP32 uses the dfuutil tool as uploader if variables.get("board") == "arduino_nano_esp32": self.install_tool("tool-dfuutil-arduino") def _configure_filesystem_tools(self, variables: Dict, targets: List[str]) -> None: - """Optimized filesystem tool configuration""" + """Configure filesystem tools based on build targets and filesystem type.""" filesystem = variables.get("board_build.filesystem", "littlefs") if any(target in targets for target in ["buildfs", "uploadfs"]): @@ -441,7 +477,7 @@ def _configure_filesystem_tools(self, variables: Dict, targets: List[str]) -> No self._install_filesystem_tool(filesystem, for_download=True) def configure_default_packages(self, variables: Dict, targets: List[str]) -> Any: - """Optimized main configuration method""" + """Main configuration method with optimized package management.""" if not variables.get("board"): return super().configure_default_packages(variables, targets) @@ -473,7 +509,7 @@ def configure_default_packages(self, variables: Dict, targets: List[str]) -> Any return super().configure_default_packages(variables, targets) def get_boards(self, id_=None): - """Get board configuration""" + """Get board configuration with dynamic options.""" result = super().get_boards(id_) if not result: return result @@ -485,7 +521,7 @@ def get_boards(self, id_=None): return result def _add_dynamic_options(self, board): - """Add dynamic board options""" + """Add dynamic board options for upload protocols and debug tools.""" # Upload protocols if not board.get("upload.protocols", []): board.manifest["upload"]["protocols"] = ["esptool", "espota"] @@ -569,7 +605,7 @@ def _add_dynamic_options(self, board): return board def _get_openocd_interface(self, link: str, board) -> str: - """Determine OpenOCD interface for debug link""" + """Determine OpenOCD interface configuration for debug link.""" if link in ("jlink", "cmsis-dap"): return link elif link in ("esp-prog", "ftdi"): @@ -585,7 +621,7 @@ def _get_openocd_interface(self, link: str, board) -> str: return f"ftdi/{link}" def _get_debug_server_args(self, openocd_interface: str, debug: Dict) -> List[str]: - """Generate debug server arguments""" + """Generate debug server arguments for OpenOCD configuration.""" if 'openocd_target' in debug: config_type = 'target' config_name = debug.get('openocd_target') @@ -599,14 +635,14 @@ def _get_debug_server_args(self, openocd_interface: str, debug: Dict) -> List[st ] def configure_debug_session(self, debug_config): - """Configure debug session""" + """Configure debug session with flash image loading.""" build_extra_data = debug_config.build_data.get("extra", {}) flash_images = build_extra_data.get("flash_images", []) if "openocd" in (debug_config.server or {}).get("executable", ""): - debug_config.server["arguments"].extend( - ["-c", f"adapter speed {debug_config.speed or DEFAULT_DEBUG_SPEED}"] - ) + debug_config.server["arguments"].extend([ + "-c", f"adapter speed {debug_config.speed or DEFAULT_DEBUG_SPEED}" + ]) ignore_conds = [ debug_config.load_cmds != ["load"], @@ -618,12 +654,13 @@ def configure_debug_session(self, debug_config): return load_cmds = [ - f'monitor program_esp "{to_unix_path(item["path"])}" {item["offset"]} verify' + f'monitor program_esp "{to_unix_path(item["path"])}" ' + f'{item["offset"]} verify' for item in flash_images ] load_cmds.append( - f'monitor program_esp "{to_unix_path(debug_config.build_data["prog_path"][:-4])}.bin" ' + f'monitor program_esp ' + f'"{to_unix_path(debug_config.build_data["prog_path"][:-4])}.bin" ' f'{build_extra_data.get("application_offset", DEFAULT_APP_OFFSET)} verify' ) debug_config.load_cmds = load_cmds - From 37952328090d44841c196728bd05f1f4f7895d86 Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Sat, 31 May 2025 14:44:58 +0200 Subject: [PATCH 82/97] Python style code formatting / adding doc strings --- builder/main.py | 493 +++++++++++++++++++++++++++++------------------- 1 file changed, 299 insertions(+), 194 deletions(-) diff --git a/builder/main.py b/builder/main.py index 29de2614b..00ec4a398 100644 --- a/builder/main.py +++ b/builder/main.py @@ -12,34 +12,41 @@ # See the License for the specific language governing permissions and # limitations under the License. +import locale import os import re -import subprocess import shlex +import subprocess import sys -import locale from os.path import isfile, join from SCons.Script import ( - ARGUMENTS, COMMAND_LINE_TARGETS, AlwaysBuild, Builder, Default, - DefaultEnvironment) + ARGUMENTS, + COMMAND_LINE_TARGETS, + AlwaysBuild, + Builder, + Default, + DefaultEnvironment, +) -from platformio.util import get_serial_ports from platformio.project.helpers import get_project_dir +from platformio.util import get_serial_ports - +# Initialize environment and configuration env = DefaultEnvironment() platform = env.PioPlatform() projectconfig = env.GetProjectConfig() terminal_cp = locale.getpreferredencoding().lower() -# -# Helpers -# - +# Framework directory path FRAMEWORK_DIR = platform.get_package_dir("framework-arduinoespressif32") + def BeforeUpload(target, source, env): + """ + Prepare the environment before uploading firmware. + Handles port detection and special upload configurations. + """ upload_options = {} if "BOARD" in env: upload_options = env.BoardConfig().get("upload", {}) @@ -56,6 +63,10 @@ def BeforeUpload(target, source, env): def _get_board_memory_type(env): + """ + Determine the memory type configuration for the board. + Returns the appropriate memory type string based on board configuration. + """ board_config = env.BoardConfig() default_type = "%s_%s" % ( board_config.get("build.flash_mode", "dio"), @@ -71,33 +82,46 @@ def _get_board_memory_type(env): ), ) + def _normalize_frequency(frequency): + """ + Convert frequency value to normalized string format (e.g., "40m"). + Removes 'L' suffix and converts to MHz format. + """ frequency = str(frequency).replace("L", "") return str(int(int(frequency) / 1000000)) + "m" + def _get_board_f_flash(env): + """Get the flash frequency for the board.""" frequency = env.subst("$BOARD_F_FLASH") return _normalize_frequency(frequency) + def _get_board_f_image(env): + """Get the image frequency for the board, fallback to flash frequency.""" board_config = env.BoardConfig() if "build.f_image" in board_config: return _normalize_frequency(board_config.get("build.f_image")) return _get_board_f_flash(env) + def _get_board_f_boot(env): + """Get the boot frequency for the board, fallback to flash frequency.""" board_config = env.BoardConfig() if "build.f_boot" in board_config: return _normalize_frequency(board_config.get("build.f_boot")) return _get_board_f_flash(env) + def _get_board_flash_mode(env): - if _get_board_memory_type(env) in ( - "opi_opi", - "opi_qspi", - ): + """ + Determine the appropriate flash mode for the board. + Handles special cases for OPI memory types. + """ + if _get_board_memory_type(env) in ("opi_opi", "opi_qspi"): return "dout" mode = env.subst("$BOARD_FLASH_MODE") @@ -105,14 +129,24 @@ def _get_board_flash_mode(env): return "dio" return mode + def _get_board_boot_mode(env): + """ + Determine the boot mode for the board. + Handles special cases for OPI memory types. + """ memory_type = env.BoardConfig().get("build.arduino.memory_type", "") build_boot = env.BoardConfig().get("build.boot", "$BOARD_FLASH_MODE") if memory_type in ("opi_opi", "opi_qspi"): build_boot = "opi" return build_boot + def _parse_size(value): + """ + Parse size values from various formats (int, hex, K/M suffixes). + Returns the size in bytes as an integer. + """ if isinstance(value, int): return value elif value.isdigit(): @@ -124,17 +158,25 @@ def _parse_size(value): return int(value[:-1]) * base return value + def _parse_partitions(env): + """ + Parse the partition table CSV file and return partition information. + Also sets the application offset for the environment. + """ partitions_csv = env.subst("$PARTITIONS_TABLE_CSV") if not isfile(partitions_csv): - sys.stderr.write("Could not find the file %s with partitions " - "table.\n" % partitions_csv) + sys.stderr.write( + "Could not find the file %s with partitions table.\n" + % partitions_csv + ) env.Exit(1) return result = [] next_offset = 0 - app_offset = 0x10000 # default address for firmware + app_offset = 0x10000 # Default address for firmware + with open(partitions_csv) as fp: for line in fp.readlines(): line = line.strip() @@ -151,24 +193,34 @@ def _parse_partitions(env): "subtype": tokens[2], "offset": tokens[3] or calculated_offset, "size": tokens[4], - "flags": tokens[5] if len(tokens) > 5 else None + "flags": tokens[5] if len(tokens) > 5 else None, } result.append(partition) next_offset = _parse_size(partition["offset"]) - if (partition["subtype"] == "ota_0"): + if partition["subtype"] == "ota_0": app_offset = next_offset next_offset = next_offset + _parse_size(partition["size"]) + # Configure application partition offset env.Replace(ESP32_APP_OFFSET=str(hex(app_offset))) # Propagate application offset to debug configurations - env["INTEGRATION_EXTRA_DATA"].update({"application_offset": str(hex(app_offset))}) + env["INTEGRATION_EXTRA_DATA"].update( + {"application_offset": str(hex(app_offset))} + ) return result + def _update_max_upload_size(env): + """ + Update the maximum upload size based on partition table configuration. + Prioritizes user-specified partition names. + """ if not env.get("PARTITIONS_TABLE_CSV"): return + sizes = { - p["subtype"]: _parse_size(p["size"]) for p in _parse_partitions(env) + p["subtype"]: _parse_size(p["size"]) + for p in _parse_partitions(env) if p["type"] in ("0", "app") } @@ -179,12 +231,15 @@ def _update_max_upload_size(env): if custom_app_partition_name: selected_partition = partitions.get(custom_app_partition_name, {}) if selected_partition: - board.update("upload.maximum_size", _parse_size(selected_partition["size"])) + board.update( + "upload.maximum_size", _parse_size(selected_partition["size"]) + ) return else: print( - "Warning! Selected partition `%s` is not available in the partition " \ - "table! Default partition will be used!" % custom_app_partition_name + "Warning! Selected partition `%s` is not available in the " + "partition table! Default partition will be used!" + % custom_app_partition_name ) for p in partitions.values(): @@ -192,17 +247,24 @@ def _update_max_upload_size(env): board.update("upload.maximum_size", _parse_size(p["size"])) break + def _to_unix_slashes(path): + """Convert Windows-style backslashes to Unix-style forward slashes.""" return path.replace("\\", "/") -# -# Filesystem helpers -# def fetch_fs_size(env): + """ + Extract filesystem size and offset information from partition table. + Sets FS_START, FS_SIZE, FS_PAGE, and FS_BLOCK environment variables. + """ fs = None for p in _parse_partitions(env): - if p["type"] == "data" and p["subtype"] in ("spiffs", "fat", "littlefs"): + if p["type"] == "data" and p["subtype"] in ( + "spiffs", + "fat", + "littlefs", + ): fs = p if not fs: sys.stderr.write( @@ -211,6 +273,7 @@ def fetch_fs_size(env): ) env.Exit(1) return + env["FS_START"] = _parse_size(fs["offset"]) env["FS_SIZE"] = _parse_size(fs["size"]) env["FS_PAGE"] = int("0x100", 16) @@ -222,20 +285,93 @@ def fetch_fs_size(env): env["FS_START"] += 4096 env["FS_SIZE"] -= 4096 + def __fetch_fs_size(target, source, env): + """Wrapper function for fetch_fs_size to be used as SCons emitter.""" fetch_fs_size(env) return (target, source) + +def check_lib_archive_exists(): + """ + Check if lib_archive is set in platformio.ini configuration. + Returns True if found, False otherwise. + """ + for section in projectconfig.sections(): + if "lib_archive" in projectconfig.options(section): + return True + return False + + +def firmware_metrics(target, source, env): + """ + Custom target to run esp-idf-size with support for command line parameters. + Usage: pio run -t metrics -- [esp-idf-size arguments] + """ + if terminal_cp != "utf-8": + print( + 'Firmware metrics can not be shown. Set the terminal codepage ' + 'to "utf-8"' + ) + return + + map_file = os.path.join( + env.subst("$BUILD_DIR"), env.subst("$PROGNAME") + ".map" + ) + if not os.path.isfile(map_file): + # Map file can be in project dir + map_file = os.path.join( + get_project_dir(), env.subst("$PROGNAME") + ".map" + ) + + if not os.path.isfile(map_file): + print(f"Error: Map file not found: {map_file}") + print("Make sure the project is built first with 'pio run'") + return + + cmd = [env.subst("$PYTHONEXE"), "-m", "esp_idf_size", "--ng"] + + # Parameters from platformio.ini + extra_args = env.GetProjectOption("custom_esp_idf_size_args", "") + if extra_args: + cmd.extend(shlex.split(extra_args)) + + # Command Line Parameter, after -- + cli_args = [] + if "--" in sys.argv: + dash_index = sys.argv.index("--") + if dash_index + 1 < len(sys.argv): + cli_args = sys.argv[dash_index + 1 :] + + # Map-file as last argument + cmd.append(map_file) + + # Debug-Info if wanted + if env.GetProjectOption("custom_esp_idf_size_verbose", False): + print(f"Running command: {' '.join(cmd)}") + + # Call esp-idf-size + result = subprocess.run(cmd, check=False, capture_output=False) + + if result.returncode != 0: + print(f"Warning: esp-idf-size exited with code {result.returncode}") + + +# Initialize board configuration and MCU settings board = env.BoardConfig() mcu = board.get("build.mcu", "esp32") toolchain_arch = "xtensa-%s" % mcu filesystem = board.get("build.filesystem", "littlefs") + +# Set toolchain architecture for RISC-V based ESP32 variants if mcu in ("esp32c2", "esp32c3", "esp32c5", "esp32c6", "esp32h2", "esp32p4"): toolchain_arch = "riscv32-esp" +# Initialize integration extra data if not present if "INTEGRATION_EXTRA_DATA" not in env: env["INTEGRATION_EXTRA_DATA"] = {} +# Configure build tools and environment variables env.Replace( __get_board_boot_mode=_get_board_boot_mode, __get_board_f_flash=_get_board_f_flash, @@ -243,7 +379,6 @@ def __fetch_fs_size(target, source, env): __get_board_f_boot=_get_board_f_boot, __get_board_flash_mode=_get_board_flash_mode, __get_board_memory_type=_get_board_memory_type, - AR="%s-elf-gcc-ar" % toolchain_arch, AS="%s-elf-as" % toolchain_arch, CC="%s-elf-gcc" % toolchain_arch, @@ -251,7 +386,14 @@ def __fetch_fs_size(target, source, env): GDB=join( platform.get_package_dir( "tool-riscv32-esp-elf-gdb" - if mcu in ("esp32c2", "esp32c3", "esp32c5", "esp32c6", "esp32h2", "esp32p4") + if mcu in ( + "esp32c2", + "esp32c3", + "esp32c5", + "esp32c6", + "esp32h2", + "esp32p4", + ) else "tool-xtensa-esp-elf-gdb" ) or "", @@ -261,20 +403,14 @@ def __fetch_fs_size(target, source, env): OBJCOPY=join(platform.get_package_dir("tool-esptoolpy") or "", "esptool.py"), RANLIB="%s-elf-gcc-ranlib" % toolchain_arch, SIZETOOL="%s-elf-size" % toolchain_arch, - ARFLAGS=["rc"], - - SIZEPROGREGEXP=r"^(?:\.iram0\.text|\.iram0\.vectors|\.dram0\.data|\.flash\.text|\.flash\.rodata|)\s+([0-9]+).*", + SIZEPROGREGEXP=r"^(?:\.iram0\.text|\.iram0\.vectors|\.dram0\.data|" + r"\.flash\.text|\.flash\.rodata|)\s+([0-9]+).*", SIZEDATAREGEXP=r"^(?:\.dram0\.data|\.dram0\.bss|\.noinit)\s+([0-9]+).*", SIZECHECKCMD="$SIZETOOL -A -d $SOURCES", SIZEPRINTCMD="$SIZETOOL -B -d $SOURCES", - - ERASEFLAGS=[ - "--chip", mcu, - "--port", '"$UPLOAD_PORT"' - ], + ERASEFLAGS=["--chip", mcu, "--port", '"$UPLOAD_PORT"'], ERASECMD='"$PYTHONEXE" "$OBJCOPY" $ERASEFLAGS erase_flash', - # mkspiffs package contains two different binaries for IDF and Arduino MKFSTOOL="mk%s" % filesystem + ( @@ -289,61 +425,61 @@ def __fetch_fs_size(target, source, env): if filesystem == "spiffs" else "" ), - # Legacy `ESP32_SPIFFS_IMAGE_NAME` is used as the second fallback value for - # backward compatibility + # Legacy `ESP32_SPIFFS_IMAGE_NAME` is used as the second fallback value + # for backward compatibility ESP32_FS_IMAGE_NAME=env.get( - "ESP32_FS_IMAGE_NAME", env.get("ESP32_SPIFFS_IMAGE_NAME", filesystem) + "ESP32_FS_IMAGE_NAME", + env.get("ESP32_SPIFFS_IMAGE_NAME", filesystem), + ), + ESP32_APP_OFFSET=env.get("INTEGRATION_EXTRA_DATA").get( + "application_offset" ), - - ESP32_APP_OFFSET=env.get("INTEGRATION_EXTRA_DATA").get("application_offset"), ARDUINO_LIB_COMPILE_FLAG="Inactive", - - PROGSUFFIX=".elf" + PROGSUFFIX=".elf", ) # Check if lib_archive is set in platformio.ini and set it to False # if not found. This makes weak defs in framework and libs possible. -def check_lib_archive_exists(): - for section in projectconfig.sections(): - if "lib_archive" in projectconfig.options(section): - #print(f"lib_archive in [{section}] found with value: {projectconfig.get(section, 'lib_archive')}") - return True - #print("lib_archive was not found in platformio.ini") - return False - if not check_lib_archive_exists(): env_section = "env:" + env["PIOENV"] projectconfig.set(env_section, "lib_archive", "False") - #print(f"lib_archive is set to False in [{env_section}]") # Allow user to override via pre:script if env.get("PROGNAME", "program") == "program": env.Replace(PROGNAME="firmware") +# Configure build actions and builders env.Append( BUILDERS=dict( ElfToBin=Builder( - action=env.VerboseAction(" ".join([ - '"$PYTHONEXE" "$OBJCOPY"', - "--chip", mcu, "elf2image", - "--flash_mode", "${__get_board_flash_mode(__env__)}", - "--flash_freq", "${__get_board_f_image(__env__)}", - "--flash_size", board.get("upload.flash_size", "4MB"), - "-o", "$TARGET", "$SOURCES" - ]), "Building $TARGET"), - suffix=".bin" + action=env.VerboseAction( + " ".join( + [ + '"$PYTHONEXE" "$OBJCOPY"', + "--chip", + mcu, + "elf2image", + "--flash_mode", + "${__get_board_flash_mode(__env__)}", + "--flash_freq", + "${__get_board_f_image(__env__)}", + "--flash_size", + board.get("upload.flash_size", "4MB"), + "-o", + "$TARGET", + "$SOURCES", + ] + ), + "Building $TARGET", + ), + suffix=".bin", ), DataToBin=Builder( action=env.VerboseAction( " ".join( ['"$MKFSTOOL"', "-c", "$SOURCES", "-s", "$FS_SIZE"] + ( - [ - "-p", - "$FS_PAGE", - "-b", - "$FS_BLOCK", - ] + ["-p", "$FS_PAGE", "-b", "$FS_BLOCK"] if filesystem in ("littlefs", "spiffs") else [] ) @@ -358,60 +494,11 @@ def check_lib_archive_exists(): ) ) +# Load framework-specific configuration if not env.get("PIOFRAMEWORK"): env.SConscript("frameworks/_bare.py", exports="env") - -def firmware_metrics(target, source, env): - """ - Custom target to run esp-idf-size with support for command line parameters - Usage: pio run -t metrics -- [esp-idf-size arguments] - """ - if terminal_cp != "utf-8": - print("Firmware metrics can not be shown. Set the terminal codepage to \"utf-8\"") - return - - map_file = os.path.join(env.subst("$BUILD_DIR"), env.subst("$PROGNAME") + ".map") - if not os.path.isfile(map_file): - # map file can be in project dir - map_file = os.path.join(get_project_dir(), env.subst("$PROGNAME") + ".map") - - if not os.path.isfile(map_file): - print(f"Error: Map file not found: {map_file}") - print("Make sure the project is built first with 'pio run'") - return - - cmd = [env.subst("$PYTHONEXE"), "-m", "esp_idf_size", "--ng"] - - # Parameters from platformio.ini - extra_args = env.GetProjectOption("custom_esp_idf_size_args", "") - if extra_args: - cmd.extend(shlex.split(extra_args)) - - # Command Line Parameter, after -- - cli_args = [] - if "--" in sys.argv: - dash_index = sys.argv.index("--") - if dash_index + 1 < len(sys.argv): - cli_args = sys.argv[dash_index + 1:] - - # Map-file as last argument - cmd.append(map_file) - - # Debug-Info if wanted - if env.GetProjectOption("custom_esp_idf_size_verbose", False): - print(f"Running command: {' '.join(cmd)}") - - # Call esp-idf-size - result = subprocess.run(cmd, check=False, capture_output=False) - - if result.returncode != 0: - print(f"Warning: esp-idf-size exited with code {result.returncode}") - -# -# Target: Build executable and linkable firmware or FS image -# - +# Build executable and linkable firmware or FS image target_elf = None if "nobuild" in COMMAND_LINE_TARGETS: target_elf = join("$BUILD_DIR", "${PROGNAME}.elf") @@ -423,7 +510,8 @@ def firmware_metrics(target, source, env): else: target_elf = env.BuildProgram() silent_action = env.Action(firmware_metrics) - silent_action.strfunction = lambda target, source, env: '' # hack to silence scons command output + # Hack to silence scons command output + silent_action.strfunction = lambda target, source, env: "" env.AddPostAction(target_elf, silent_action) if set(["buildfs", "uploadfs", "uploadfsota"]) & set(COMMAND_LINE_TARGETS): target_firm = env.DataToBin( @@ -432,26 +520,27 @@ def firmware_metrics(target, source, env): env.NoCache(target_firm) AlwaysBuild(target_firm) else: - target_firm = env.ElfToBin( - join("$BUILD_DIR", "${PROGNAME}"), target_elf) + target_firm = env.ElfToBin(join("$BUILD_DIR", "${PROGNAME}"), target_elf) env.Depends(target_firm, "checkprogsize") -env.AddPlatformTarget("buildfs", target_firm, target_firm, "Build Filesystem Image") +# Configure platform targets +env.AddPlatformTarget( + "buildfs", target_firm, target_firm, "Build Filesystem Image" +) AlwaysBuild(env.Alias("nobuild", target_firm)) target_buildprog = env.Alias("buildprog", target_firm, target_firm) -# update max upload size based on CSV file +# Update max upload size based on CSV file if env.get("PIOMAINPROG"): env.AddPreAction( "checkprogsize", env.VerboseAction( lambda source, target, env: _update_max_upload_size(env), - "Retrieving maximum program size $SOURCES")) + "Retrieving maximum program size $SOURCES", + ), + ) -# # Target: Print binary size -# - target_size = env.AddPlatformTarget( "size", target_elf, @@ -460,25 +549,25 @@ def firmware_metrics(target, source, env): "Calculate program size", ) -# # Target: Upload firmware or FS image -# - upload_protocol = env.subst("$UPLOAD_PROTOCOL") debug_tools = board.get("debug.tools", {}) upload_actions = [] # Compatibility with old OTA configurations -if (upload_protocol != "espota" - and re.match(r"\"?((([0-9]{1,3}\.){3}[0-9]{1,3})|[^\\/]+\.local)\"?$", - env.get("UPLOAD_PORT", ""))): +if upload_protocol != "espota" and re.match( + r"\"?((([0-9]{1,3}\.){3}[0-9]{1,3})|[^\\/]+\.local)\"?$", + env.get("UPLOAD_PORT", ""), +): upload_protocol = "espota" sys.stderr.write( "Warning! We have just detected `upload_port` as IP address or host " "name of ESP device. `upload_protocol` is switched to `espota`.\n" "Please specify `upload_protocol = espota` in `platformio.ini` " - "project configuration file.\n") + "project configuration file.\n" + ) +# Configure upload protocol: ESP OTA if upload_protocol == "espota": if not env.subst("$UPLOAD_PORT"): sys.stderr.write( @@ -486,32 +575,45 @@ def firmware_metrics(target, source, env): "using `upload_port` for build environment or use " "global `--upload-port` option.\n" "See https://docs.platformio.org/page/platforms/" - "espressif32.html#over-the-air-ota-update\n") + "espressif32.html#over-the-air-ota-update\n" + ) env.Replace( - UPLOADER=join(FRAMEWORK_DIR,"tools", "espota.py"), + UPLOADER=join(FRAMEWORK_DIR, "tools", "espota.py"), UPLOADERFLAGS=["--debug", "--progress", "-i", "$UPLOAD_PORT"], - UPLOADCMD='"$PYTHONEXE" "$UPLOADER" $UPLOADERFLAGS -f $SOURCE' + UPLOADCMD='"$PYTHONEXE" "$UPLOADER" $UPLOADERFLAGS -f $SOURCE', ) if set(["uploadfs", "uploadfsota"]) & set(COMMAND_LINE_TARGETS): env.Append(UPLOADERFLAGS=["--spiffs"]) upload_actions = [env.VerboseAction("$UPLOADCMD", "Uploading $SOURCE")] +# Configure upload protocol: esptool elif upload_protocol == "esptool": env.Replace( UPLOADER=join( - platform.get_package_dir("tool-esptoolpy") or "", "esptool.py"), + platform.get_package_dir("tool-esptoolpy") or "", "esptool.py" + ), UPLOADERFLAGS=[ - "--chip", mcu, - "--port", '"$UPLOAD_PORT"', - "--baud", "$UPLOAD_SPEED", - "--before", board.get("upload.before_reset", "default_reset"), - "--after", board.get("upload.after_reset", "hard_reset"), - "write_flash", "-z", - "--flash_mode", "${__get_board_flash_mode(__env__)}", - "--flash_freq", "${__get_board_f_image(__env__)}", - "--flash_size", "detect" + "--chip", + mcu, + "--port", + '"$UPLOAD_PORT"', + "--baud", + "$UPLOAD_SPEED", + "--before", + board.get("upload.before_reset", "default_reset"), + "--after", + board.get("upload.after_reset", "hard_reset"), + "write_flash", + "-z", + "--flash_mode", + "${__get_board_flash_mode(__env__)}", + "--flash_freq", + "${__get_board_f_image(__env__)}", + "--flash_size", + "detect", ], - UPLOADCMD='"$PYTHONEXE" "$UPLOADER" $UPLOADERFLAGS $ESP32_APP_OFFSET $SOURCE' + UPLOADCMD='"$PYTHONEXE" "$UPLOADER" $UPLOADERFLAGS ' + "$ESP32_APP_OFFSET $SOURCE", ) for image in env.get("FLASH_EXTRA_IMAGES", []): env.Append(UPLOADERFLAGS=[image[0], env.subst(image[1])]) @@ -519,27 +621,36 @@ def firmware_metrics(target, source, env): if "uploadfs" in COMMAND_LINE_TARGETS: env.Replace( UPLOADERFLAGS=[ - "--chip", mcu, - "--port", '"$UPLOAD_PORT"', - "--baud", "$UPLOAD_SPEED", - "--before", board.get("upload.before_reset", "default_reset"), - "--after", board.get("upload.after_reset", "hard_reset"), - "write_flash", "-z", - "--flash_mode", "${__get_board_flash_mode(__env__)}", - "--flash_freq", "${__get_board_f_image(__env__)}", - "--flash_size", "detect", - "$FS_START" + "--chip", + mcu, + "--port", + '"$UPLOAD_PORT"', + "--baud", + "$UPLOAD_SPEED", + "--before", + board.get("upload.before_reset", "default_reset"), + "--after", + board.get("upload.after_reset", "hard_reset"), + "write_flash", + "-z", + "--flash_mode", + "${__get_board_flash_mode(__env__)}", + "--flash_freq", + "${__get_board_f_image(__env__)}", + "--flash_size", + "detect", + "$FS_START", ], UPLOADCMD='"$PYTHONEXE" "$UPLOADER" $UPLOADERFLAGS $SOURCE', ) upload_actions = [ env.VerboseAction(BeforeUpload, "Looking for upload port..."), - env.VerboseAction("$UPLOADCMD", "Uploading $SOURCE") + env.VerboseAction("$UPLOADCMD", "Uploading $SOURCE"), ] +# Configure upload protocol: DFU elif upload_protocol == "dfu": - hwids = board.get("build.hwids", [["0x2341", "0x0070"]]) vid = hwids[0][0] pid = hwids[0][1] @@ -554,16 +665,18 @@ def firmware_metrics(target, source, env): "-d", ",".join(["%s:%s" % (hwid[0], hwid[1]) for hwid in hwids]), "-Q", - "-D" + "-D", ], UPLOADCMD='"$UPLOADER" $UPLOADERFLAGS "$SOURCE"', ) +# Configure upload protocol: Debug tools (OpenOCD) elif upload_protocol in debug_tools: _parse_partitions(env) openocd_args = ["-d%d" % (2 if int(ARGUMENTS.get("PIOVERBOSE", 0)) else 1)] openocd_args.extend( - debug_tools.get(upload_protocol).get("server").get("arguments", [])) + debug_tools.get(upload_protocol).get("server").get("arguments", []) + ) openocd_args.extend( [ "-c", @@ -591,7 +704,9 @@ def firmware_metrics(target, source, env): f.replace( "$PACKAGE_DIR", _to_unix_slashes( - platform.get_package_dir("tool-openocd-esp32") or "")) + platform.get_package_dir("tool-openocd-esp32") or "" + ), + ) for f in openocd_args ] env.Replace( @@ -601,81 +716,71 @@ def firmware_metrics(target, source, env): ) upload_actions = [env.VerboseAction("$UPLOADCMD", "Uploading $SOURCE")] -# custom upload tool +# Configure upload protocol: Custom elif upload_protocol == "custom": upload_actions = [env.VerboseAction("$UPLOADCMD", "Uploading $SOURCE")] else: sys.stderr.write("Warning! Unknown upload protocol %s\n" % upload_protocol) - +# Register upload targets env.AddPlatformTarget("upload", target_firm, upload_actions, "Upload") -env.AddPlatformTarget("uploadfs", target_firm, upload_actions, "Upload Filesystem Image") env.AddPlatformTarget( - "uploadfsota", target_firm, upload_actions, "Upload Filesystem Image OTA") + "uploadfs", target_firm, upload_actions, "Upload Filesystem Image" +) +env.AddPlatformTarget( + "uploadfsota", + target_firm, + upload_actions, + "Upload Filesystem Image OTA", +) -# # Target: Erase Flash and Upload -# - env.AddPlatformTarget( "erase_upload", target_firm, [ env.VerboseAction(BeforeUpload, "Looking for upload port..."), env.VerboseAction("$ERASECMD", "Erasing..."), - env.VerboseAction("$UPLOADCMD", "Uploading $SOURCE") + env.VerboseAction("$UPLOADCMD", "Uploading $SOURCE"), ], "Erase Flash and Upload", ) -# # Target: Erase Flash -# - env.AddPlatformTarget( "erase", None, [ env.VerboseAction(BeforeUpload, "Looking for upload port..."), - env.VerboseAction("$ERASECMD", "Erasing...") + env.VerboseAction("$ERASECMD", "Erasing..."), ], "Erase Flash", ) -# -# Register Custom Target -# +# Register Custom Target for firmware metrics env.AddCustomTarget( name="metrics", dependencies="$BUILD_DIR/${PROGNAME}.elf", actions=firmware_metrics, title="Firmware Size Metrics", - description="Analyze firmware size using esp-idf-size (supports CLI args after --)", - always_build=True + description="Analyze firmware size using esp-idf-size " + "(supports CLI args after --)", + always_build=True, ) -# # Additional Target without Build-Dependency when already compiled -# env.AddCustomTarget( name="metrics-only", dependencies=None, actions=firmware_metrics, title="Firmware Size Metrics (No Build)", description="Analyze firmware size without building first", - always_build=True + always_build=True, ) - -# # Override memory inspection behavior -# - env.SConscript("sizedata.py", exports="env") -# -# Default targets -# - +# Set default targets Default([target_buildprog, target_size]) From 1ef3e9091f2e82d9a2d2feff95bc12a775aa12d7 Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Sat, 31 May 2025 15:25:19 +0200 Subject: [PATCH 83/97] nitpick changes --- builder/frameworks/espidf.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/builder/frameworks/espidf.py b/builder/frameworks/espidf.py index fe077750e..bc1bb05e0 100644 --- a/builder/frameworks/espidf.py +++ b/builder/frameworks/espidf.py @@ -238,7 +238,7 @@ def load_custom_sdkconfig_file(): print(f"Path Traversal detected: {url} check your URL path") else: try: - response = requests.get(file_entry.split(" ")[0]) + response = requests.get(file_entry.split(" ")[0], timeout=10) if response.ok: return response.content.decode('utf-8') except requests.RequestException as e: @@ -318,6 +318,9 @@ def add_flash_configuration(config_flags): return config_flags def write_sdkconfig_file(idf_config_flags, checksum_source): + if "arduino" not in env.subst("$PIOFRAMEWORK"): + print("Error: Arduino framework required for sdkconfig processing") + return """Write the final sdkconfig.defaults file with checksum.""" sdkconfig_src = join(arduino_libs_mcu, "sdkconfig") sdkconfig_dst = join(PROJECT_DIR, "sdkconfig.defaults") From 4eaac3396e8e90f16124b8bda4dca8a9415bdcfd Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Sat, 31 May 2025 15:31:08 +0200 Subject: [PATCH 84/97] catch possible exception from subprocess more specific --- platform.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platform.py b/platform.py index 12152880b..658855cbe 100644 --- a/platform.py +++ b/platform.py @@ -191,7 +191,7 @@ def _run_idf_tools_install(self, tools_json_path: str, idf_tools_path: str) -> b except subprocess.TimeoutExpired: logger.error(f"Timeout in idf_tools.py after {SUBPROCESS_TIMEOUT}s") return False - except Exception as e: + except (subprocess.SubprocessError, OSError) as e: logger.error(f"Error in idf_tools.py: {e}") return False From 798e22a6519b3d488fc4d2145bd062c01b48ea01 Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Sat, 31 May 2025 15:33:25 +0200 Subject: [PATCH 85/97] whitespaces --- builder/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/builder/main.py b/builder/main.py index 00ec4a398..a37db9382 100644 --- a/builder/main.py +++ b/builder/main.py @@ -176,7 +176,7 @@ def _parse_partitions(env): result = [] next_offset = 0 app_offset = 0x10000 # Default address for firmware - + with open(partitions_csv) as fp: for line in fp.readlines(): line = line.strip() @@ -200,7 +200,7 @@ def _parse_partitions(env): if partition["subtype"] == "ota_0": app_offset = next_offset next_offset = next_offset + _parse_size(partition["size"]) - + # Configure application partition offset env.Replace(ESP32_APP_OFFSET=str(hex(app_offset))) # Propagate application offset to debug configurations From f35b79636cd83312a580f9c326fa4c35783e9f7e Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Sat, 31 May 2025 15:38:07 +0200 Subject: [PATCH 86/97] simplify logic --- platform.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/platform.py b/platform.py index 658855cbe..0dac6167c 100644 --- a/platform.py +++ b/platform.py @@ -287,11 +287,11 @@ def _handle_existing_tool( self.packages[tool_name]["optional"] = False logger.debug(f"Tool {tool_name} found with correct version") return True - else: - # Wrong version, reinstall - logger.info(f"Reinstalling {tool_name} due to version mismatch") - safe_remove_directory(paths['tool_path']) - return self.install_tool(tool_name, retry_count + 1) + + # Wrong version, reinstall + logger.info(f"Reinstalling {tool_name} due to version mismatch") + safe_remove_directory(paths['tool_path']) + return self.install_tool(tool_name, retry_count + 1) def _configure_arduino_framework(self, frameworks: List[str]) -> None: """Configure Arduino framework with dynamic library URL fetching.""" From efed85646216f2289c0bdbbac9b2cf90a31a65e4 Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Sat, 31 May 2025 15:40:10 +0200 Subject: [PATCH 87/97] PEP8 convention --- builder/frameworks/arduino.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/builder/frameworks/arduino.py b/builder/frameworks/arduino.py index c7ab1aeec..04b26c961 100644 --- a/builder/frameworks/arduino.py +++ b/builder/frameworks/arduino.py @@ -24,7 +24,6 @@ import subprocess import json -import semantic_version import os import sys import shutil @@ -36,6 +35,7 @@ from pathlib import Path from typing import Union, List +import semantic_version from SCons.Script import DefaultEnvironment, SConscript from platformio import fs from platformio.package.version import pepver_to_semver From 8cb1de0180e2437f6e32d114b87ab90092c0a6a0 Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Sat, 31 May 2025 16:14:37 +0200 Subject: [PATCH 88/97] better check for entry in board_sdkconfig --- platform.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platform.py b/platform.py index 0dac6167c..2ef94be6c 100644 --- a/platform.py +++ b/platform.py @@ -323,7 +323,7 @@ def _configure_espidf_framework( board_config.get("espidf.custom_sdkconfig", "") ) - if custom_sdkconfig is not None or len(str(board_sdkconfig)) > 3: + if custom_sdkconfig is not None or board_sdkconfig is not None: frameworks.append("espidf") self.packages["framework-espidf"]["optional"] = False if mcu == "esp32c2": From cfc3720b437229d4587c9e6b3b5940facb532536 Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Sat, 31 May 2025 16:29:11 +0200 Subject: [PATCH 89/97] even more short and correct --- platform.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platform.py b/platform.py index 2ef94be6c..8bad9f8b6 100644 --- a/platform.py +++ b/platform.py @@ -323,7 +323,7 @@ def _configure_espidf_framework( board_config.get("espidf.custom_sdkconfig", "") ) - if custom_sdkconfig is not None or board_sdkconfig is not None: + if custom_sdkconfig or board_sdkconfig: frameworks.append("espidf") self.packages["framework-espidf"]["optional"] = False if mcu == "esp32c2": From 46ac6a7938e5287aeeff1ed06b2b07705458c77d Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Sat, 31 May 2025 16:39:40 +0200 Subject: [PATCH 90/97] solid check, does only fire when entrys there --- platform.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/platform.py b/platform.py index 8bad9f8b6..933299e60 100644 --- a/platform.py +++ b/platform.py @@ -323,7 +323,8 @@ def _configure_espidf_framework( board_config.get("espidf.custom_sdkconfig", "") ) - if custom_sdkconfig or board_sdkconfig: + if (custom_sdkconfig and custom_sdkconfig.strip()) or \ + (board_sdkconfig and board_sdkconfig.strip()): frameworks.append("espidf") self.packages["framework-espidf"]["optional"] = False if mcu == "esp32c2": From 60ee5318f8ba42bca014966725d2aa6315b66cee Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Sat, 31 May 2025 17:00:13 +0200 Subject: [PATCH 91/97] implement function has_valid_sdkconfig() for "clean" checking of config settings --- platform.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/platform.py b/platform.py index 933299e60..a988a8969 100644 --- a/platform.py +++ b/platform.py @@ -117,6 +117,11 @@ def safe_copy_file(src: str, dst: str) -> bool: return True +def has_valid_sdkconfig(config_value: str) -> bool: + """Check if sdkconfig value is valid and non-empty.""" + return bool(config_value and config_value.strip()) + + class Espressif32Platform(PlatformBase): """ESP32 platform implementation for PlatformIO with optimized toolchain management.""" @@ -323,8 +328,7 @@ def _configure_espidf_framework( board_config.get("espidf.custom_sdkconfig", "") ) - if (custom_sdkconfig and custom_sdkconfig.strip()) or \ - (board_sdkconfig and board_sdkconfig.strip()): + if has_valid_sdkconfig(custom_sdkconfig) or has_valid_sdkconfig(board_sdkconfig): frameworks.append("espidf") self.packages["framework-espidf"]["optional"] = False if mcu == "esp32c2": From 4eff679a1dc8242b069212b92f9657822e7ce527 Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Sat, 31 May 2025 17:12:03 +0200 Subject: [PATCH 92/97] fix function has_valid_sdkconfig() --- platform.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/platform.py b/platform.py index a988a8969..a28486b22 100644 --- a/platform.py +++ b/platform.py @@ -117,9 +117,11 @@ def safe_copy_file(src: str, dst: str) -> bool: return True -def has_valid_sdkconfig(config_value: str) -> bool: +def has_valid_sdkconfig(config_value) -> bool: """Check if sdkconfig value is valid and non-empty.""" - return bool(config_value and config_value.strip()) + if not config_value: + return False + return bool(config_value.strip()) class Espressif32Platform(PlatformBase): From ec5a431ce1d9b2a6a9aa006ade8e392b3b96bb82 Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Sat, 31 May 2025 17:18:48 +0200 Subject: [PATCH 93/97] back to refactor 1 soution --- platform.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/platform.py b/platform.py index a28486b22..933299e60 100644 --- a/platform.py +++ b/platform.py @@ -117,13 +117,6 @@ def safe_copy_file(src: str, dst: str) -> bool: return True -def has_valid_sdkconfig(config_value) -> bool: - """Check if sdkconfig value is valid and non-empty.""" - if not config_value: - return False - return bool(config_value.strip()) - - class Espressif32Platform(PlatformBase): """ESP32 platform implementation for PlatformIO with optimized toolchain management.""" @@ -330,7 +323,8 @@ def _configure_espidf_framework( board_config.get("espidf.custom_sdkconfig", "") ) - if has_valid_sdkconfig(custom_sdkconfig) or has_valid_sdkconfig(board_sdkconfig): + if (custom_sdkconfig and custom_sdkconfig.strip()) or \ + (board_sdkconfig and board_sdkconfig.strip()): frameworks.append("espidf") self.packages["framework-espidf"]["optional"] = False if mcu == "esp32c2": From 598bbce5aaebe0fc8ebbe7f7747b3c0355a17d34 Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Sat, 31 May 2025 17:25:02 +0200 Subject: [PATCH 94/97] back to working "dirty" if check --- platform.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/platform.py b/platform.py index 933299e60..0dac6167c 100644 --- a/platform.py +++ b/platform.py @@ -323,8 +323,7 @@ def _configure_espidf_framework( board_config.get("espidf.custom_sdkconfig", "") ) - if (custom_sdkconfig and custom_sdkconfig.strip()) or \ - (board_sdkconfig and board_sdkconfig.strip()): + if custom_sdkconfig is not None or len(str(board_sdkconfig)) > 3: frameworks.append("espidf") self.packages["framework-espidf"]["optional"] = False if mcu == "esp32c2": From 1581adb49efccacf064e3b3a390db40deb33584c Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Sat, 31 May 2025 18:41:48 +0200 Subject: [PATCH 95/97] file handling with UTF-8 --- builder/frameworks/espidf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/builder/frameworks/espidf.py b/builder/frameworks/espidf.py index bc1bb05e0..80a52889a 100644 --- a/builder/frameworks/espidf.py +++ b/builder/frameworks/espidf.py @@ -328,7 +328,7 @@ def write_sdkconfig_file(idf_config_flags, checksum_source): # Generate checksum for validation (maintains original logic) checksum = get_MD5_hash(checksum_source.strip() + mcu) - with open(sdkconfig_src, 'r') as src, open(sdkconfig_dst, 'w') as dst: + with open(sdkconfig_src, 'r', encoding='utf-8') as src, open(sdkconfig_dst, 'w', encoding='utf-8') as dst: # Write checksum header (critical for compilation decision logic) dst.write(f"# TASMOTA__{checksum}\n") From 5f350b7be958351e094fa43eb55861c9b527449c Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Sat, 31 May 2025 18:47:24 +0200 Subject: [PATCH 96/97] utf-8 --- platform.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/platform.py b/platform.py index 0dac6167c..239c804b4 100644 --- a/platform.py +++ b/platform.py @@ -200,7 +200,7 @@ def _check_tool_version(self, tool_name: str) -> bool: paths = self._get_tool_paths(tool_name) try: - with open(paths['package_path'], 'r') as f: + with open(paths['package_path'], 'r', encoding='utf-8') as f: package_data = json.load(f) required_version = self.packages.get(tool_name, {}).get("package-version") @@ -406,7 +406,7 @@ def _ensure_mklittlefs_version(self) -> None: if os.path.exists(piopm_path): try: - with open(piopm_path, 'r') as f: + with open(piopm_path, 'r', encoding='utf-8') as f: package_data = json.load(f) if package_data.get('version') != MKLITTLEFS_VERSION_320: os.remove(piopm_path) From 58e43679c7d88f7c9d5fa2aec32d16ebfac15032 Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Sat, 31 May 2025 19:04:13 +0200 Subject: [PATCH 97/97] Update main.py --- builder/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/builder/main.py b/builder/main.py index a37db9382..f3dee1b40 100644 --- a/builder/main.py +++ b/builder/main.py @@ -217,7 +217,7 @@ def _update_max_upload_size(env): """ if not env.get("PARTITIONS_TABLE_CSV"): return - + sizes = { p["subtype"]: _parse_size(p["size"]) for p in _parse_partitions(env)