ardupilotwaf: kill env.py

Tools/ardupilotwaf/chibios.py

@@ -17,9 +17,13 @@
 import struct
 import base64
 import subprocess
+import traceback
 
 import hal_common
 
+# sys.path already set up at the top of boards.py
+import chibios_hwdef
+
 _dynamic_env_data = {}
 def _load_dynamic_env_data(bld):
     bldnode = bld.bldnode.make_node('modules/ChibiOS')
@@ -511,16 +515,16 @@ def setup_canperiph_build(cfg):
     cfg.get_board().with_can = True
 
 def load_env_vars_handle_kv_pair(env, kv_pair):
-    '''handle a key/value pair out of the pickled environment dictionary'''
+    '''handle a key/value pair out of the hwdef generator'''
     (k, v) = kv_pair
     if k == 'ROMFS_FILES':
         env.ROMFS_FILES += v
         return
     hal_common.load_env_vars_handle_kv_pair(env, kv_pair)
 
-def load_env_vars(env):
-    '''optionally load extra environment variables from env.py in the build directory'''
-    hal_common.load_env_vars(env, kv_handler=load_env_vars_handle_kv_pair)
+def load_env_vars(env, hwdef_env):
+    '''load environment variables from the hwdef generator'''
+    hal_common.load_env_vars(env, hwdef_env, kv_handler=load_env_vars_handle_kv_pair)
 
     if env.DEBUG or env.DEBUG_SYMBOLS:
         env.CHIBIOS_BUILD_FLAGS += ' ENABLE_DEBUG_SYMBOLS=yes'
@@ -597,12 +601,11 @@ def bldpath(path):
         env.DEFAULT_PARAMETERS = cfg.options.default_parameters
 
     try:
-        ret = generate_hwdef_h(env)
+        hwdef_env = generate_hwdef_h(env)
     except Exception:
+        traceback.print_exc()
         cfg.fatal("Failed to process hwdef.dat")
-    if ret != 0:
-        cfg.fatal("Failed to process hwdef.dat ret=%d" % ret)
-    load_env_vars(cfg.env)
+    load_env_vars(cfg.env, hwdef_env)
     if env.HAL_NUM_CAN_IFACES and not env.AP_PERIPH:
         setup_canmgr_build(cfg)
     if env.HAL_NUM_CAN_IFACES and env.AP_PERIPH and not env.BOOTLOADER:
@@ -613,38 +616,42 @@ def bldpath(path):
 
 def generate_hwdef_h(env):
     '''run chibios_hwdef.py'''
-    import subprocess
     if env.BOOTLOADER:
         if len(env.HWDEF) == 0:
             env.HWDEF = os.path.join(env.SRCROOT, 'libraries/AP_HAL_ChibiOS/hwdef/%s/hwdef-bl.dat' % env.BOARD)
         else:
             # update to using hwdef-bl.dat
             env.HWDEF = env.HWDEF.replace('hwdef.dat', 'hwdef-bl.dat')
-        env.BOOTLOADER_OPTION="--bootloader"
+        bootloader_flag = True
     else:
         if len(env.HWDEF) == 0:
             env.HWDEF = os.path.join(env.SRCROOT, 'libraries/AP_HAL_ChibiOS/hwdef/%s/hwdef.dat' % env.BOARD)
-        env.BOOTLOADER_OPTION=""
+        bootloader_flag = False
 
-    if env.AP_SIGNED_FIRMWARE:
-        print(env.BOOTLOADER_OPTION)
-        env.BOOTLOADER_OPTION += " --signed-fw"
-        print(env.BOOTLOADER_OPTION)
     hwdef_script = os.path.join(env.SRCROOT, 'libraries/AP_HAL_ChibiOS/hwdef/scripts/chibios_hwdef.py')
     hwdef_out = env.BUILDROOT
     if not os.path.exists(hwdef_out):
         os.mkdir(hwdef_out)
-    python = sys.executable
-    cmd = "{0} '{1}' -D '{2}' --params '{3}' '{4}'".format(python, hwdef_script, hwdef_out, env.DEFAULT_PARAMETERS, env.HWDEF)
+
+    hwdef = [env.HWDEF]
     if env.HWDEF_EXTRA:
-        cmd += " '{0}'".format(env.HWDEF_EXTRA)
-    if env.BOOTLOADER_OPTION:
-        cmd += " " + env.BOOTLOADER_OPTION
-    return subprocess.call(cmd, shell=True)
+        hwdef.append(env.HWDEF_EXTRA)
+
+    c = chibios_hwdef.ChibiOSHWDef(
+        outdir=hwdef_out,
+        bootloader=bootloader_flag,
+        signed_fw=bool(env.AP_SIGNED_FIRMWARE),
+        hwdef=hwdef,
+        # stringify like old subprocess based invocation. note that no error is
+        # generated if this path is missing!
+        default_params_filepath=str(env.DEFAULT_PARAMETERS),
+        quiet=False,
+    )
+    c.run()
+    return c.env_vars
 
 def pre_build(bld):
     '''pre-build hook to change dynamic sources'''
-    load_env_vars(bld.env)
     if bld.env.HAL_NUM_CAN_IFACES:
         bld.get_board().with_can = True
     if bld.env.WITH_LITTLEFS:
@@ -653,37 +660,16 @@ def pre_build(bld):
     if not os.path.exists(hwdef_h):
         print("Generating hwdef.h")
         try:
-            ret = generate_hwdef_h(bld.env)
+            generate_hwdef_h(bld.env)
+            # TRAP: env vars are not reloaded!
         except Exception:
             bld.fatal("Failed to process hwdef.dat")
-        if ret != 0:
-            bld.fatal("Failed to process hwdef.dat ret=%d" % ret)
     setup_optimization(bld.env)
 
 def build(bld):
 
     # make ccache effective on ChibiOS builds
     os.environ['CCACHE_IGNOREOPTIONS'] = '--specs=nano.specs --specs=nosys.specs'
-
-    hwdef_rule="%s '%s/hwdef/scripts/chibios_hwdef.py' -D '%s' --params '%s' '%s'" % (
-            bld.env.get_flat('PYTHON'),
-            bld.env.AP_HAL_ROOT,
-            bld.env.BUILDROOT,
-            bld.env.default_parameters,
-            bld.env.HWDEF)
-    if bld.env.HWDEF_EXTRA:
-        hwdef_rule += " " + bld.env.HWDEF_EXTRA
-    if bld.env.BOOTLOADER_OPTION:
-        hwdef_rule += " " + bld.env.BOOTLOADER_OPTION
-    bld(
-        # build hwdef.h from hwdef.dat. This is needed after a waf clean
-        source=bld.path.ant_glob(bld.env.HWDEF),
-        rule=hwdef_rule,
-        group='dynamic_sources',
-        target=[bld.bldnode.find_or_declare('hwdef.h'),
-                bld.bldnode.find_or_declare('ldscript.ld'),
-                bld.bldnode.find_or_declare('hw.dat')]
-    )
 
     bld(
         # create the file modules/ChibiOS/include_dirs

Tools/ardupilotwaf/esp32.py

@@ -31,11 +31,6 @@ def esp32_dynamic_env(self):
     hal_common.common_dynamic_env(self)
 
 
-def load_env_vars(env):
-    '''optionally load extra environment variables from env.py in the build directory'''
-    hal_common.load_env_vars(env)
-
-
 def configure(cfg):
     mcu_esp32s3 = True if (cfg.variant[0:7] == "esp32s3") else False
     target = "esp32s3" if mcu_esp32s3 else "esp32"
@@ -69,18 +64,11 @@ def bldpath(path):
     print("USING EXPRESSIF IDF:"+str(env.IDF))
 
     try:
-        generate_hwdef_h(env)
-    except Exception as e:
-        print(get_exception_stacktrace(e))
+        hwdef_env = generate_hwdef_h(env)
+    except Exception:
+        traceback.print_exc()
         cfg.fatal("Failed to generate hwdef")
-    load_env_vars(cfg.env)
-
-def get_exception_stacktrace(e):
-    ret = "%s\n" % e
-    ret += ''.join(traceback.format_exception(type(e),
-                                              e,
-                                              tb=e.__traceback__))
-    return ret
+    hal_common.load_env_vars(cfg.env, hwdef_env)
 
 def generate_hwdef_h(env):
     '''run esp32_hwdef.py'''
@@ -100,6 +88,7 @@ def generate_hwdef_h(env):
         quiet=False,
     )
     eh.run()
+    return eh.env_vars
 
 # delete the output sdkconfig file when the input defaults changes. we take the
 # stamp as the output so we can compute the path to the sdkconfig, yet it
@@ -166,28 +155,17 @@ def run(tsk):
     self.add_to_group(tsk)
 
     # hwdef pre-build:
-    load_env_vars(self.env)
 #    if bld.env.HAL_NUM_CAN_IFACES:
 #        bld.get_board().with_can = True
     hwdef_h = os.path.join(self.env.BUILDROOT, 'hwdef.h')
     if not os.path.exists(hwdef_h):
         print("Generating hwdef.h")
         try:
             generate_hwdef_h(self.env)
+            # TRAP: env vars are not reloaded!
         except Exception:
             self.fatal(f"Failed to process hwdef.dat {hwdef_h}")
 
-def build(bld):
-    bld(
-        # build hwdef.h from hwdef.dat. This is needed after a waf clean
-        source=bld.path.ant_glob(bld.env.HWDEF),
-        rule="",
-        group='dynamic_sources',
-        target=[
-            bld.bldnode.find_or_declare('hwdef.h'),
-        ]
-    )
-
 @feature('esp32_ap_program')
 @after_method('process_source')
 def esp32_firmware(self):

Tools/ardupilotwaf/hal_common.py

@@ -4,27 +4,20 @@
 AP_FLAKE8_CLEAN
 """
 
-import os
-import pickle
 import sys
 
 
-def load_env_vars(env, kv_handler=None):
-    '''optionally load extra environment variables from env.py in the build directory'''
-    env_py = os.path.join(env.BUILDROOT, 'env.py')
-    if not os.path.exists(env_py):
-        print("No env.py found")
-        return
-    e = pickle.load(open(env_py, 'rb'))
-    for kv in e.items():
+def load_env_vars(env, hwdef_env, kv_handler=None):
+    '''load environment variables from the hwdef generator'''
+    for kv in hwdef_env.items():
         if kv_handler is not None:
             kv_handler(env, kv)
             continue
         load_env_vars_handle_kv_pair(env, kv)
 
 
 def load_env_vars_handle_kv_pair(env, kv_pair):
-    '''handle a key/value pair out of the pickled environment dictionary'''
+    '''handle a key/value pair out of the hwdef generator'''
     (k, v) = kv_pair
     if k in env:
         if isinstance(env[k], dict):

Tools/ardupilotwaf/linux.py

@@ -30,11 +30,6 @@ def linux_firmware(self):
     pass
 
 
-def load_env_vars(env):
-    '''optionally load extra environment variables from env.py in the build directory'''
-    hal_common.load_env_vars(env)
-
-
 def configure(cfg):
 
     def srcpath(path):
@@ -56,19 +51,11 @@ def bldpath(path):
     env.AP_PROGRAM_FEATURES += ['linux_ap_program']
 
     try:
-        generate_hwdef_h(env)
-    except Exception as e:
-        print(get_exception_stacktrace(e))
+        hwdef_env = generate_hwdef_h(env)
+    except Exception:
+        traceback.print_exc()
         cfg.fatal("Failed to generate hwdef")
-    load_env_vars(cfg.env)
-
-
-def get_exception_stacktrace(e):
-    ret = "%s\n" % e
-    ret += ''.join(traceback.format_exception(type(e),
-                                              e,
-                                              tb=e.__traceback__))
-    return ret
+    hal_common.load_env_vars(cfg.env, hwdef_env)
 
 
 def generate_hwdef_h(env):
@@ -89,11 +76,11 @@ def generate_hwdef_h(env):
         quiet=False,
     )
     lh.run()
+    return lh.env_vars
 
 
 def pre_build(bld):
     '''pre-build hook to change dynamic sources'''
-    load_env_vars(bld.env)
     if bld.env.HAL_NUM_CAN_IFACES:
         bld.get_board().with_can = True
     if bld.env.WITH_LITTLEFS:
@@ -103,17 +90,6 @@ def pre_build(bld):
         print("Generating hwdef.h")
         try:
             generate_hwdef_h(bld.env)
+            # TRAP: env vars are not reloaded!
         except Exception:
             bld.fatal(f"Failed to process hwdef.dat {hwdef_h}")
-
-
-def build(bld):
-    bld(
-        # build hwdef.h from hwdef.dat. This is needed after a waf clean
-        source=bld.path.ant_glob(bld.env.HWDEF),
-        rule="",
-        group='dynamic_sources',
-        target=[
-            bld.bldnode.find_or_declare('hwdef.h'),
-        ]
-    )

Tools/scripts/run_flake8.py

@@ -41,10 +41,6 @@ def run(self):
             for filename in filenames:
                 if os.path.splitext(filename)[1] != ".py":
                     continue
-                if filename == 'env.py':
-                    # we are generating content into these files which
-                    # is not actually Python...
-                    continue
                 filepath = os.path.join(dirpath, filename)
                 content = pathlib.Path(filepath).read_text()
                 if "AP_FLAKE8_CLEAN" not in content:

libraries/AP_HAL/hwdef/scripts/hwdef.py

@@ -6,7 +6,6 @@
 
 import filecmp
 import os
-import pickle
 import re
 import shlex
 import sys
@@ -403,7 +402,3 @@ def write_device_table(self, f, description, define_name, devlist):
         f.write(f'#define {define_name} \\\n')
         f.write(',\\\n'.join([f"   {x}" for x in devlist]))
         f.write("\n")
-
-    def write_env_py(self, filename):
-        '''write out env.py for environment variables to control the build process'''
-        pickle.dump(self.env_vars, open(filename, "wb"))

libraries/AP_HAL_ChibiOS/hwdef/scripts/chibios_hwdef.py

@@ -2729,7 +2729,7 @@ def get_processed_defaults_file(self, defaults_filepath, depth=0):
     def write_processed_defaults_file(self, filepath):
         # see if board has a defaults.parm file or a --default-parameters file was specified
         defaults_filename = os.path.join(os.path.dirname(self.hwdef[0]), 'defaults.parm')
-        defaults_path = os.path.join(os.path.dirname(self.hwdef[0]), args.params)
+        defaults_path = os.path.join(os.path.dirname(self.hwdef[0]), self.default_params_filepath)
 
         defaults_abspath = None
         if os.path.exists(defaults_path):
@@ -3131,7 +3131,6 @@ def run(self):
 
         # CHIBIOS_BUILD_FLAGS is passed to the ChibiOS makefile
         self.env_vars['CHIBIOS_BUILD_FLAGS'] = ' '.join(self.build_flags)
-        self.write_env_py(os.path.join(self.outdir, "env.py"))
 
 
 if __name__ == '__main__':

libraries/AP_HAL_ESP32/hwdef/scripts/esp32_hwdef.py

@@ -46,8 +46,6 @@ def write_hwdef_header_content(self, f):
         self.write_SERIAL_config(f)
         self.write_RCOUT_config(f)
 
-        self.write_env_py(os.path.join(self.outdir, "env.py"))
-
     def process_line(self, line, depth):
         '''process one line of pin definition file'''
         # keep all config lines for later use

libraries/AP_HAL_Linux/hwdef/scripts/linux_hwdef.py

@@ -32,8 +32,6 @@ def write_hwdef_header_content(self, f):
         self.write_MAG_config(f)
         self.write_BARO_config(f)
 
-        self.write_env_py(os.path.join(self.outdir, "env.py"))
-
     def process_line(self, line, depth):
         '''process one line of pin definition file'''
         # keep all config lines for later use