split boards dynamically

Author: microdev1

.github/actions/deps/external/action.yml

@@ -9,54 +9,31 @@ inputs:
     - cache
     - restore
 
-  platform:
+  port:
     required: false
     default: none
-    type: choice
-    options:
-    - arm
-    - aarch
-    - esp
-    - riscv
-    - none
+    type: string
 
 runs:
   using: composite
   steps:
-    # aarch
-    - name: Get aarch toolchain
-      if: inputs.platform == 'aarch'
-      run: |
-        wget --no-verbose https://adafruit-circuit-python.s3.amazonaws.com/gcc-arm-10.3-2021.07-x86_64-aarch64-none-elf.tar.xz
-        sudo tar -C /usr --strip-components=1 -xaf gcc-arm-10.3-2021.07-x86_64-aarch64-none-elf.tar.xz
-        sudo apt-get install -y mtools
-      shell: bash
-    - name: Install mkfs.fat
-      if: inputs.platform == 'aarch'
-      run: |
-        wget https://github.com/dosfstools/dosfstools/releases/download/v4.2/dosfstools-4.2.tar.gz
-        tar -xaf dosfstools-4.2.tar.gz
-        cd dosfstools-4.2
-        ./configure
-        make -j 2
-        cd src
-        echo >> $GITHUB_PATH $(pwd)
-      shell: bash
-
     # arm
     - name: Get arm toolchain
-      if: inputs.platform == 'aarch' || inputs.platform == 'arm'
+      if: >-
+        inputs.port != 'none' &&
+        inputs.port != 'litex' &&
+        inputs.port != 'espressif'
       uses: carlosperate/arm-none-eabi-gcc-action@v1
       with:
         release: '10-2020-q4'
 
-    # esp
-    - name: Get esp toolchain
-      if: inputs.platform == 'esp'
+    # espressif
+    - name: Get espressif toolchain
+      if: inputs.port == 'espressif'
       run: sudo apt-get install -y ninja-build
       shell: bash
     - name: Install IDF tools
-      if: inputs.platform == 'esp'
+      if: inputs.port == 'espressif'
       run: |
         echo "Installing ESP-IDF tools"
         $IDF_PATH/tools/idf_tools.py --non-interactive install required
@@ -66,24 +43,16 @@ runs:
         rm -rf $IDF_TOOLS_PATH/dist
       shell: bash
     - name: Set environment
-      if: inputs.platform == 'esp'
+      if: inputs.port == 'espressif'
       run: |
         source $IDF_PATH/export.sh
         echo >> $GITHUB_ENV "IDF_PYTHON_ENV_PATH=$IDF_PYTHON_ENV_PATH"
         echo >> $GITHUB_PATH "$PATH"
       shell: bash
 
-    # riscv
-    - name: Get riscv toolchain
-      if: inputs.platform == 'riscv'
-      run: |
-        wget https://static.dev.sifive.com/dev-tools/riscv64-unknown-elf-gcc-8.3.0-2019.08.0-x86_64-linux-centos6.tar.gz
-        sudo tar -C /usr --strip-components=1 -xaf riscv64-unknown-elf-gcc-8.3.0-2019.08.0-x86_64-linux-centos6.tar.gz
-      shell: bash
-
     # common
     - name: Cache python dependencies
-      if: inputs.platform != 'esp'
+      if: inputs.port != 'espressif'
       uses: ./.github/actions/deps/python
       with:
         action: ${{ inputs.action }}

.github/actions/deps/ports/action.yml

@@ -19,10 +19,18 @@ runs:
         if [ -z $PORT ]; then (exit 1); else echo >> $GITHUB_OUTPUT "port=$PORT"; fi
       shell: bash
 
-    - name: Set up espressif port
+    - name: Set up broadcom
+      if: steps.board-to-port.outputs.port == 'broadcom'
+      uses: ./.github/actions/deps/ports/broadcom
+
+    - name: Set up espressif
       if: steps.board-to-port.outputs.port == 'espressif'
       uses: ./.github/actions/deps/ports/espressif
 
-    - name: Set up nrf port
+    - name: Set up litex
+      if: steps.board-to-port.outputs.port == 'litex'
+      uses: ./.github/actions/deps/ports/litex
+
+    - name: Set up nrf
       if: steps.board-to-port.outputs.port == 'nrf'
       uses: ./.github/actions/deps/ports/nrf

.github/actions/deps/ports/broadcom/action.yml

@@ -0,0 +1,21 @@
+name: Fetch broadcom port deps
+
+runs:
+  using: composite
+  steps:
+    - name: Get broadcom toolchain
+      run: |
+        wget --no-verbose https://adafruit-circuit-python.s3.amazonaws.com/gcc-arm-10.3-2021.07-x86_64-broadcom64-none-elf.tar.xz
+        sudo tar -C /usr --strip-components=1 -xaf gcc-arm-10.3-2021.07-x86_64-broadcom64-none-elf.tar.xz
+        sudo apt-get install -y mtools
+      shell: bash
+    - name: Install mkfs.fat
+      run: |
+        wget https://github.com/dosfstools/dosfstools/releases/download/v4.2/dosfstools-4.2.tar.gz
+        tar -xaf dosfstools-4.2.tar.gz
+        cd dosfstools-4.2
+        ./configure
+        make -j 2
+        cd src
+        echo >> $GITHUB_PATH $(pwd)
+      shell: bash

.github/actions/deps/ports/litex/action.yml

@@ -0,0 +1,10 @@
+name: Fetch litex port deps
+
+runs:
+  using: composite
+  steps:
+    - name: Get litex toolchain
+      run: |
+        wget https://static.dev.sifive.com/dev-tools/riscv64-unknown-elf-gcc-8.3.0-2019.08.0-x86_64-linux-centos6.tar.gz
+        sudo tar -C /usr --strip-components=1 -xaf riscv64-unknown-elf-gcc-8.3.0-2019.08.0-x86_64-linux-centos6.tar.gz
+      shell: bash

.github/workflows/build-boards-custom.yml

@@ -32,16 +32,6 @@ run-name: ${{ inputs.board }}-${{ inputs.language }}-${{ inputs.version }}${{ in
 jobs:
   build:
     runs-on: ubuntu-22.04
-    env:
-      PLATFORM_atmel-samd: arm
-      PLATFORM_broadcom: aarch
-      PLATFORM_cxd56: arm
-      PLATFORM_espressif: esp
-      PLATFORM_litex: riscv
-      PLATFORM_mimxrt10xx: arm
-      PLATFORM_nrf: arm
-      PLATFORM_raspberrypi: arm
-      PLATFORM_stm: arm
     steps:
     - name: Set up repository
       run: |
@@ -70,7 +60,7 @@ jobs:
       uses: ./.github/actions/deps/external
       with:
         action: cache
-        platform: ${{ env[format('PLATFORM_{0}', steps.set-up-port.outputs.port)] }}
+        port: ${{ steps.set-up-port.outputs.port }}
     - name: Set up mpy-cross
       if: steps.set-up-submodules.outputs.frozen == 'True'
       uses: ./.github/actions/mpy_cross

.github/workflows/build-boards.yml

@@ -3,9 +3,6 @@ name: Build boards
 on:
   workflow_call:
     inputs:
-      platform:
-        required: true
-        type: string
       boards:
         required: true
         type: string
@@ -38,6 +35,7 @@ jobs:
       with:
         python-version: 3.x
     - name: Set up port
+      id: set-up-port
       uses: ./.github/actions/deps/ports
       with:
         board: ${{ matrix.board }}
@@ -47,7 +45,7 @@ jobs:
     - name: Set up external
       uses: ./.github/actions/deps/external
       with:
-        platform: ${{ inputs.platform }}
+        port: ${{ steps.set-up-port.outputs.port }}
     - name: Set up mpy-cross
       if: steps.set-up-submodules.outputs.frozen == 'True'
       uses: ./.github/actions/mpy_cross

.github/workflows/build.yml

@@ -20,14 +20,9 @@ jobs:
   scheduler:
     runs-on: ubuntu-20.04
     outputs:
-      build-doc: ${{ steps.set-matrix.outputs.build-doc }}
-      build-boards: ${{ steps.set-matrix.outputs.build-boards }}
-      build-windows: ${{ steps.set-matrix.outputs.build-windows }}
-      boards-aarch: ${{ steps.set-matrix.outputs.boards-aarch }}
-      boards-arm: ${{ steps.set-matrix.outputs.boards-arm }}
-      boards-atmel: ${{ steps.set-matrix.outputs.boards-atmel }}
-      boards-esp: ${{ steps.set-matrix.outputs.boards-esp }}
-      boards-riscv: ${{ steps.set-matrix.outputs.boards-riscv }}
+      docs: ${{ steps.set-matrix.outputs.docs }}
+      boards: ${{ steps.set-matrix.outputs.boards }}
+      windows: ${{ steps.set-matrix.outputs.windows }}
       cp-version: ${{ steps.set-up-submodules.outputs.version }}
     steps:
     - name: Dump GitHub context
@@ -99,7 +94,7 @@ jobs:
 
   mpy-cross:
     needs: scheduler
-    if: needs.scheduler.outputs.build-boards == 'True'
+    if: needs.scheduler.outputs.boards != '{}'
     uses: ./.github/workflows/build-mpy-cross.yml
     secrets: inherit
     with:
@@ -108,7 +103,7 @@ jobs:
   mpy-cross-mac:
     runs-on: macos-11
     needs: scheduler
-    if: needs.scheduler.outputs.build-boards == 'True'
+    if: needs.scheduler.outputs.boards != '{}'
     env:
       CP_VERSION: ${{ needs.scheduler.outputs.cp-version }}
     steps:
@@ -160,10 +155,10 @@ jobs:
         AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
         AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
 
-  build-doc:
+  docs:
     runs-on: ubuntu-22.04
     needs: scheduler
-    if: needs.scheduler.outputs.build-doc == 'True'
+    if: needs.scheduler.outputs.docs == 'True'
     env:
       CP_VERSION: ${{ needs.scheduler.outputs.cp-version }}
     steps:
@@ -218,10 +213,10 @@ jobs:
         [ -z "$TWINE_USERNAME" ] || echo "Uploading dev release to PyPi"
         [ -z "$TWINE_USERNAME" ] || twine upload circuitpython-stubs/dist/*
 
-  build-windows:
+  windows:
     runs-on: windows-2022
     needs: scheduler
-    if: needs.scheduler.outputs.build-windows == 'True'
+    if: needs.scheduler.outputs.windows == 'True'
     env:
       CP_VERSION: ${{ needs.scheduler.outputs.cp-version }}
     defaults:
@@ -286,52 +281,14 @@ jobs:
     # ERROR: Platform MINGW64_NT-10.0-17763-x86_64 appears to be unsupported
     # https://github.com/espressif/esp-idf/issues/7062
 
-  aarch:
+  boards:
     needs: [scheduler, mpy-cross, tests]
-    if: ${{ needs.scheduler.outputs.boards-aarch != '[]' }}
+    if: needs.scheduler.outputs.boards != '{}'
     uses: ./.github/workflows/build-boards.yml
     secrets: inherit
+    strategy:
+      matrix:
+        port: ${{ fromJSON(needs.scheduler.outputs.boards).ports }}
     with:
-      platform: aarch
-      boards: ${{ needs.scheduler.outputs.boards-aarch }}
-      cp-version: ${{ needs.scheduler.outputs.cp-version }}
-
-  arm:
-    needs: [scheduler, mpy-cross, tests]
-    if: ${{ needs.scheduler.outputs.boards-arm != '[]' }}
-    uses: ./.github/workflows/build-boards.yml
-    secrets: inherit
-    with:
-      platform: arm
-      boards: ${{ needs.scheduler.outputs.boards-arm }}
-      cp-version: ${{ needs.scheduler.outputs.cp-version }}
-
-  atmel:
-    needs: [scheduler, mpy-cross, tests]
-    if: ${{ needs.scheduler.outputs.boards-atmel != '[]' }}
-    uses: ./.github/workflows/build-boards.yml
-    secrets: inherit
-    with:
-      platform: arm
-      boards: ${{ needs.scheduler.outputs.boards-atmel }}
-      cp-version: ${{ needs.scheduler.outputs.cp-version }}
-
-  esp:
-    needs: [scheduler, mpy-cross, tests]
-    if: ${{ needs.scheduler.outputs.boards-esp != '[]' }}
-    uses: ./.github/workflows/build-boards.yml
-    secrets: inherit
-    with:
-      platform: esp
-      boards: ${{ needs.scheduler.outputs.boards-esp }}
-      cp-version: ${{ needs.scheduler.outputs.cp-version }}
-
-  riscv:
-    needs: [scheduler, mpy-cross, tests]
-    if: ${{ needs.scheduler.outputs.boards-riscv != '[]' }}
-    uses: ./.github/workflows/build-boards.yml
-    secrets: inherit
-    with:
-      platform: riscv
-      boards: ${{ needs.scheduler.outputs.boards-riscv }}
+      boards: ${{ toJSON(fromJSON(needs.scheduler.outputs.boards)[matrix.port]) }}
       cp-version: ${{ needs.scheduler.outputs.cp-version }}

tools/ci_changes_per_commit.py

@@ -167,8 +167,8 @@ def get_commit_depth_and_check_suite(query_commits):
 
 
 def get_bad_check_runs(query_check_runs):
+    bad_runs = {}
     more_pages = True
-    bad_runs_by_matrix = {}
 
     run_types = ["failed", "incomplete"]
 
@@ -184,13 +184,12 @@ def get_bad_check_runs(query_check_runs):
 
             for check_run in check_runs[run_type]["nodes"]:
                 name = check_run["name"]
-                res_matrix = regex_matrix.search(name)
-                if res_matrix:
-                    matrix = name.split(" /", 1)[0]
-                    matrix_job = name.split(" (", 1)[1][:-1]
-                    bad_runs_by_matrix.setdefault(matrix, []).append(matrix_job)
+                if name.startswith("boards") or regex_matrix.search(name):
+                    matrix = name.split(" ", 1)[0]
+                    matrix_job = name.rsplit(" (", 1)[1][:-1]
+                    bad_runs.setdefault(matrix, []).append(matrix_job)
                 elif name != "scheduler":
-                    bad_runs_by_matrix[name] = True
+                    bad_runs[name] = True
                 else:
                     return {}
 
@@ -200,7 +199,7 @@ def get_bad_check_runs(query_check_runs):
                 query_check_runs.variables["include" + run_type_camel] = True
                 more_pages = True
 
-    return bad_runs_by_matrix
+    return bad_runs
 
 
 def set_commit(commit):

tools/ci_fetch_deps.py

@@ -78,13 +78,13 @@ def main():
         submodules = ["extmod/ulab", "lib/", "tools/"]
     elif TARGET == "tests":
         submodules = ["extmod/ulab", "lib/", "tools/"]
-    elif TARGET == "build-doc":
+    elif TARGET == "docs":
         # used in .readthedocs.yml to generate RTD
         submodules = ["extmod/ulab"]
         submodules_tags = ["frozen/"]
     elif TARGET == "mpy-cross" or TARGET == "mpy-cross-mac":
         submodules = ["tools/"]  # for huffman
-    elif TARGET == "build-windows":
+    elif TARGET == "windows":
         # This builds one board from a number of ports so fill out a bunch of submodules
         submodules = ["extmod/ulab", "lib/", "tools/", "ports/", "data/nvm.toml"]
     elif TARGET == "website":