diff --git a/Tools/ardupilotwaf/chibios.py b/Tools/ardupilotwaf/chibios.py index 6e342d5dd559e..b50dee2504c35 100644 --- a/Tools/ardupilotwaf/chibios.py +++ b/Tools/ardupilotwaf/chibios.py @@ -17,9 +17,13 @@ import struct import base64 import subprocess +import traceback import hal_common +# sys.path already set up at the top of boards.py +import chibios_hwdef + _dynamic_env_data = {} def _load_dynamic_env_data(bld): bldnode = bld.bldnode.make_node('modules/ChibiOS') @@ -511,16 +515,16 @@ def setup_canperiph_build(cfg): cfg.get_board().with_can = True def load_env_vars_handle_kv_pair(env, kv_pair): - '''handle a key/value pair out of the pickled environment dictionary''' + '''handle a key/value pair out of the hwdef generator''' (k, v) = kv_pair if k == 'ROMFS_FILES': env.ROMFS_FILES += v return hal_common.load_env_vars_handle_kv_pair(env, kv_pair) -def load_env_vars(env): - '''optionally load extra environment variables from env.py in the build directory''' - hal_common.load_env_vars(env, kv_handler=load_env_vars_handle_kv_pair) +def load_env_vars(env, hwdef_env): + '''load environment variables from the hwdef generator''' + hal_common.load_env_vars(env, hwdef_env, kv_handler=load_env_vars_handle_kv_pair) if env.DEBUG or env.DEBUG_SYMBOLS: env.CHIBIOS_BUILD_FLAGS += ' ENABLE_DEBUG_SYMBOLS=yes' @@ -597,12 +601,11 @@ def bldpath(path): env.DEFAULT_PARAMETERS = cfg.options.default_parameters try: - ret = generate_hwdef_h(env) + hwdef_env = generate_hwdef_h(env) except Exception: + traceback.print_exc() cfg.fatal("Failed to process hwdef.dat") - if ret != 0: - cfg.fatal("Failed to process hwdef.dat ret=%d" % ret) - load_env_vars(cfg.env) + load_env_vars(cfg.env, hwdef_env) if env.HAL_NUM_CAN_IFACES and not env.AP_PERIPH: setup_canmgr_build(cfg) if env.HAL_NUM_CAN_IFACES and env.AP_PERIPH and not env.BOOTLOADER: @@ -613,38 +616,42 @@ def bldpath(path): def generate_hwdef_h(env): '''run chibios_hwdef.py''' - import subprocess if env.BOOTLOADER: if len(env.HWDEF) == 0: env.HWDEF = os.path.join(env.SRCROOT, 'libraries/AP_HAL_ChibiOS/hwdef/%s/hwdef-bl.dat' % env.BOARD) else: # update to using hwdef-bl.dat env.HWDEF = env.HWDEF.replace('hwdef.dat', 'hwdef-bl.dat') - env.BOOTLOADER_OPTION="--bootloader" + bootloader_flag = True else: if len(env.HWDEF) == 0: env.HWDEF = os.path.join(env.SRCROOT, 'libraries/AP_HAL_ChibiOS/hwdef/%s/hwdef.dat' % env.BOARD) - env.BOOTLOADER_OPTION="" + bootloader_flag = False - if env.AP_SIGNED_FIRMWARE: - print(env.BOOTLOADER_OPTION) - env.BOOTLOADER_OPTION += " --signed-fw" - print(env.BOOTLOADER_OPTION) hwdef_script = os.path.join(env.SRCROOT, 'libraries/AP_HAL_ChibiOS/hwdef/scripts/chibios_hwdef.py') hwdef_out = env.BUILDROOT if not os.path.exists(hwdef_out): os.mkdir(hwdef_out) - python = sys.executable - cmd = "{0} '{1}' -D '{2}' --params '{3}' '{4}'".format(python, hwdef_script, hwdef_out, env.DEFAULT_PARAMETERS, env.HWDEF) + + hwdef = [env.HWDEF] if env.HWDEF_EXTRA: - cmd += " '{0}'".format(env.HWDEF_EXTRA) - if env.BOOTLOADER_OPTION: - cmd += " " + env.BOOTLOADER_OPTION - return subprocess.call(cmd, shell=True) + hwdef.append(env.HWDEF_EXTRA) + + c = chibios_hwdef.ChibiOSHWDef( + outdir=hwdef_out, + bootloader=bootloader_flag, + signed_fw=bool(env.AP_SIGNED_FIRMWARE), + hwdef=hwdef, + # stringify like old subprocess based invocation. note that no error is + # generated if this path is missing! + default_params_filepath=str(env.DEFAULT_PARAMETERS), + quiet=False, + ) + c.run() + return c.env_vars def pre_build(bld): '''pre-build hook to change dynamic sources''' - load_env_vars(bld.env) if bld.env.HAL_NUM_CAN_IFACES: bld.get_board().with_can = True if bld.env.WITH_LITTLEFS: @@ -653,37 +660,16 @@ def pre_build(bld): if not os.path.exists(hwdef_h): print("Generating hwdef.h") try: - ret = generate_hwdef_h(bld.env) + generate_hwdef_h(bld.env) + # TRAP: env vars are not reloaded! except Exception: bld.fatal("Failed to process hwdef.dat") - if ret != 0: - bld.fatal("Failed to process hwdef.dat ret=%d" % ret) setup_optimization(bld.env) def build(bld): # make ccache effective on ChibiOS builds os.environ['CCACHE_IGNOREOPTIONS'] = '--specs=nano.specs --specs=nosys.specs' - - hwdef_rule="%s '%s/hwdef/scripts/chibios_hwdef.py' -D '%s' --params '%s' '%s'" % ( - bld.env.get_flat('PYTHON'), - bld.env.AP_HAL_ROOT, - bld.env.BUILDROOT, - bld.env.default_parameters, - bld.env.HWDEF) - if bld.env.HWDEF_EXTRA: - hwdef_rule += " " + bld.env.HWDEF_EXTRA - if bld.env.BOOTLOADER_OPTION: - hwdef_rule += " " + bld.env.BOOTLOADER_OPTION - bld( - # build hwdef.h from hwdef.dat. This is needed after a waf clean - source=bld.path.ant_glob(bld.env.HWDEF), - rule=hwdef_rule, - group='dynamic_sources', - target=[bld.bldnode.find_or_declare('hwdef.h'), - bld.bldnode.find_or_declare('ldscript.ld'), - bld.bldnode.find_or_declare('hw.dat')] - ) bld( # create the file modules/ChibiOS/include_dirs diff --git a/Tools/ardupilotwaf/esp32.py b/Tools/ardupilotwaf/esp32.py index b46a5a962332d..c0c9bd9ea8bed 100644 --- a/Tools/ardupilotwaf/esp32.py +++ b/Tools/ardupilotwaf/esp32.py @@ -31,11 +31,6 @@ def esp32_dynamic_env(self): hal_common.common_dynamic_env(self) -def load_env_vars(env): - '''optionally load extra environment variables from env.py in the build directory''' - hal_common.load_env_vars(env) - - def configure(cfg): mcu_esp32s3 = True if (cfg.variant[0:7] == "esp32s3") else False target = "esp32s3" if mcu_esp32s3 else "esp32" @@ -69,18 +64,11 @@ def bldpath(path): print("USING EXPRESSIF IDF:"+str(env.IDF)) try: - generate_hwdef_h(env) - except Exception as e: - print(get_exception_stacktrace(e)) + hwdef_env = generate_hwdef_h(env) + except Exception: + traceback.print_exc() cfg.fatal("Failed to generate hwdef") - load_env_vars(cfg.env) - -def get_exception_stacktrace(e): - ret = "%s\n" % e - ret += ''.join(traceback.format_exception(type(e), - e, - tb=e.__traceback__)) - return ret + hal_common.load_env_vars(cfg.env, hwdef_env) def generate_hwdef_h(env): '''run esp32_hwdef.py''' @@ -100,6 +88,7 @@ def generate_hwdef_h(env): quiet=False, ) eh.run() + return eh.env_vars # delete the output sdkconfig file when the input defaults changes. we take the # stamp as the output so we can compute the path to the sdkconfig, yet it @@ -166,7 +155,6 @@ def run(tsk): self.add_to_group(tsk) # hwdef pre-build: - load_env_vars(self.env) # if bld.env.HAL_NUM_CAN_IFACES: # bld.get_board().with_can = True hwdef_h = os.path.join(self.env.BUILDROOT, 'hwdef.h') @@ -174,20 +162,10 @@ def run(tsk): print("Generating hwdef.h") try: generate_hwdef_h(self.env) + # TRAP: env vars are not reloaded! except Exception: self.fatal(f"Failed to process hwdef.dat {hwdef_h}") -def build(bld): - bld( - # build hwdef.h from hwdef.dat. This is needed after a waf clean - source=bld.path.ant_glob(bld.env.HWDEF), - rule="", - group='dynamic_sources', - target=[ - bld.bldnode.find_or_declare('hwdef.h'), - ] - ) - @feature('esp32_ap_program') @after_method('process_source') def esp32_firmware(self): diff --git a/Tools/ardupilotwaf/hal_common.py b/Tools/ardupilotwaf/hal_common.py index c6f0319ee5abb..27e03d068a58b 100644 --- a/Tools/ardupilotwaf/hal_common.py +++ b/Tools/ardupilotwaf/hal_common.py @@ -4,19 +4,12 @@ AP_FLAKE8_CLEAN """ -import os -import pickle import sys -def load_env_vars(env, kv_handler=None): - '''optionally load extra environment variables from env.py in the build directory''' - env_py = os.path.join(env.BUILDROOT, 'env.py') - if not os.path.exists(env_py): - print("No env.py found") - return - e = pickle.load(open(env_py, 'rb')) - for kv in e.items(): +def load_env_vars(env, hwdef_env, kv_handler=None): + '''load environment variables from the hwdef generator''' + for kv in hwdef_env.items(): if kv_handler is not None: kv_handler(env, kv) continue @@ -24,7 +17,7 @@ def load_env_vars(env, kv_handler=None): def load_env_vars_handle_kv_pair(env, kv_pair): - '''handle a key/value pair out of the pickled environment dictionary''' + '''handle a key/value pair out of the hwdef generator''' (k, v) = kv_pair if k in env: if isinstance(env[k], dict): diff --git a/Tools/ardupilotwaf/linux.py b/Tools/ardupilotwaf/linux.py index 09e3dfb05c800..85683d1b1a1c2 100644 --- a/Tools/ardupilotwaf/linux.py +++ b/Tools/ardupilotwaf/linux.py @@ -30,11 +30,6 @@ def linux_firmware(self): pass -def load_env_vars(env): - '''optionally load extra environment variables from env.py in the build directory''' - hal_common.load_env_vars(env) - - def configure(cfg): def srcpath(path): @@ -56,19 +51,11 @@ def bldpath(path): env.AP_PROGRAM_FEATURES += ['linux_ap_program'] try: - generate_hwdef_h(env) - except Exception as e: - print(get_exception_stacktrace(e)) + hwdef_env = generate_hwdef_h(env) + except Exception: + traceback.print_exc() cfg.fatal("Failed to generate hwdef") - load_env_vars(cfg.env) - - -def get_exception_stacktrace(e): - ret = "%s\n" % e - ret += ''.join(traceback.format_exception(type(e), - e, - tb=e.__traceback__)) - return ret + hal_common.load_env_vars(cfg.env, hwdef_env) def generate_hwdef_h(env): @@ -89,11 +76,11 @@ def generate_hwdef_h(env): quiet=False, ) lh.run() + return lh.env_vars def pre_build(bld): '''pre-build hook to change dynamic sources''' - load_env_vars(bld.env) if bld.env.HAL_NUM_CAN_IFACES: bld.get_board().with_can = True if bld.env.WITH_LITTLEFS: @@ -103,17 +90,6 @@ def pre_build(bld): print("Generating hwdef.h") try: generate_hwdef_h(bld.env) + # TRAP: env vars are not reloaded! except Exception: bld.fatal(f"Failed to process hwdef.dat {hwdef_h}") - - -def build(bld): - bld( - # build hwdef.h from hwdef.dat. This is needed after a waf clean - source=bld.path.ant_glob(bld.env.HWDEF), - rule="", - group='dynamic_sources', - target=[ - bld.bldnode.find_or_declare('hwdef.h'), - ] - ) diff --git a/Tools/scripts/run_flake8.py b/Tools/scripts/run_flake8.py index 44ece77473cfa..d0c9037953be5 100755 --- a/Tools/scripts/run_flake8.py +++ b/Tools/scripts/run_flake8.py @@ -41,10 +41,6 @@ def run(self): for filename in filenames: if os.path.splitext(filename)[1] != ".py": continue - if filename == 'env.py': - # we are generating content into these files which - # is not actually Python... - continue filepath = os.path.join(dirpath, filename) content = pathlib.Path(filepath).read_text() if "AP_FLAKE8_CLEAN" not in content: diff --git a/libraries/AP_HAL/hwdef/scripts/hwdef.py b/libraries/AP_HAL/hwdef/scripts/hwdef.py index 1a7721f06e50d..8d69259634877 100644 --- a/libraries/AP_HAL/hwdef/scripts/hwdef.py +++ b/libraries/AP_HAL/hwdef/scripts/hwdef.py @@ -6,7 +6,6 @@ import filecmp import os -import pickle import re import shlex import sys @@ -403,7 +402,3 @@ def write_device_table(self, f, description, define_name, devlist): f.write(f'#define {define_name} \\\n') f.write(',\\\n'.join([f" {x}" for x in devlist])) f.write("\n") - - def write_env_py(self, filename): - '''write out env.py for environment variables to control the build process''' - pickle.dump(self.env_vars, open(filename, "wb")) diff --git a/libraries/AP_HAL_ChibiOS/hwdef/scripts/chibios_hwdef.py b/libraries/AP_HAL_ChibiOS/hwdef/scripts/chibios_hwdef.py index ffb6fd7a20434..3da98d6c5bc4a 100644 --- a/libraries/AP_HAL_ChibiOS/hwdef/scripts/chibios_hwdef.py +++ b/libraries/AP_HAL_ChibiOS/hwdef/scripts/chibios_hwdef.py @@ -2729,7 +2729,7 @@ def get_processed_defaults_file(self, defaults_filepath, depth=0): def write_processed_defaults_file(self, filepath): # see if board has a defaults.parm file or a --default-parameters file was specified defaults_filename = os.path.join(os.path.dirname(self.hwdef[0]), 'defaults.parm') - defaults_path = os.path.join(os.path.dirname(self.hwdef[0]), args.params) + defaults_path = os.path.join(os.path.dirname(self.hwdef[0]), self.default_params_filepath) defaults_abspath = None if os.path.exists(defaults_path): @@ -3131,7 +3131,6 @@ def run(self): # CHIBIOS_BUILD_FLAGS is passed to the ChibiOS makefile self.env_vars['CHIBIOS_BUILD_FLAGS'] = ' '.join(self.build_flags) - self.write_env_py(os.path.join(self.outdir, "env.py")) if __name__ == '__main__': diff --git a/libraries/AP_HAL_ESP32/hwdef/scripts/esp32_hwdef.py b/libraries/AP_HAL_ESP32/hwdef/scripts/esp32_hwdef.py index f92f6c80f10df..c20d76bf4bcdf 100755 --- a/libraries/AP_HAL_ESP32/hwdef/scripts/esp32_hwdef.py +++ b/libraries/AP_HAL_ESP32/hwdef/scripts/esp32_hwdef.py @@ -46,8 +46,6 @@ def write_hwdef_header_content(self, f): self.write_SERIAL_config(f) self.write_RCOUT_config(f) - self.write_env_py(os.path.join(self.outdir, "env.py")) - def process_line(self, line, depth): '''process one line of pin definition file''' # keep all config lines for later use diff --git a/libraries/AP_HAL_Linux/hwdef/scripts/linux_hwdef.py b/libraries/AP_HAL_Linux/hwdef/scripts/linux_hwdef.py index 98cdd0bb58811..b00b13de2672f 100755 --- a/libraries/AP_HAL_Linux/hwdef/scripts/linux_hwdef.py +++ b/libraries/AP_HAL_Linux/hwdef/scripts/linux_hwdef.py @@ -32,8 +32,6 @@ def write_hwdef_header_content(self, f): self.write_MAG_config(f) self.write_BARO_config(f) - self.write_env_py(os.path.join(self.outdir, "env.py")) - def process_line(self, line, depth): '''process one line of pin definition file''' # keep all config lines for later use