From 0fe4ac83ad425b50d7e3c688ebd96a91dd15d6ef Mon Sep 17 00:00:00 2001 From: Santi Manero Date: Mon, 17 Mar 2025 11:16:37 +0100 Subject: [PATCH 01/12] [BLAZ-1548] Improve performance creation docker image --- .dockerignore | 14 ++++++++++++++ deployment/Dockerfile.application | 31 ++++++++++++++----------------- setup.py | 12 ++++++------ 3 files changed, 34 insertions(+), 23 deletions(-) create mode 100644 .dockerignore diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..20c477ef --- /dev/null +++ b/.dockerignore @@ -0,0 +1,14 @@ +venv +build +dist +deployment +test-reports +**/tests +.semgrep +.idea +.github +.eggs +.pre-commit-config.yaml +*.egg-info/ +**/*.zip +**/*.7z \ No newline at end of file diff --git a/deployment/Dockerfile.application b/deployment/Dockerfile.application index baf20945..1c329d08 100644 --- a/deployment/Dockerfile.application +++ b/deployment/Dockerfile.application @@ -1,30 +1,27 @@ -FROM python:3.11-alpine AS startleft-base +FROM python:3.12-alpine3.21 AS startleft-base -WORKDIR /usr/src/app - -RUN apk update && \ - apk upgrade && \ - apk add --repository=https://dl-cdn.alpinelinux.org/alpine/v3.20/main --repository=https://dl-cdn.alpinelinux.org/alpine/v3.20/community \ - g++~=13.2 gcc~=13.2 gfortran~=13.2 libgcc~=13.2 libstdc++~=13.2 && \ - apk --no-cache add geos geos-dev git graphviz-dev lapack lapack-dev libmagic musl-dev py3-pybind11-dev re2 re2-dev +WORKDIR /app COPY . . -RUN pip install --upgrade pip && pip install . +RUN apk -U upgrade && \ + apk add git && \ + apk add --no-cache cblas geos git lapack libmagic py3-pygraphviz py3-re2 py3-shapely re2 && \ + cp -R /usr/lib/python3.12/site-packages /usr/local/lib/python3.12/ +RUN ls -a /usr/local/lib/python3.12/site-packages -FROM python:3.11-alpine +RUN pip install --upgrade pip && pip install . -WORKDIR /app -RUN apk update && \ - apk upgrade && \ - apk --no-cache add cblas geos graphviz-dev lapack libmagic re2 && \ - adduser --disabled-password --no-create-home startleft && \ - rm -rf /usr/local/lib/python3.11/site-packages +FROM python:3.12-alpine3.21 -COPY --from=startleft-base /usr/local/lib/python3.11/site-packages /usr/local/lib/python3.11/site-packages +RUN apk -U upgrade && \ + apk add --no-cache libmagic py3-pygraphviz py3-re2 py3-shapely && \ + adduser --disabled-password --no-create-home startleft && \ + rm -Rf /usr/local/lib/python3.12/site-packages +COPY --from=startleft-base /usr/local/lib/python3.12/site-packages /usr/local/lib/python3.12/site-packages COPY --from=startleft-base /usr/local/bin/startleft /usr/local/bin/startleft USER startleft diff --git a/setup.py b/setup.py index 7b605bf6..269ba5e1 100644 --- a/setup.py +++ b/setup.py @@ -13,7 +13,7 @@ keywords=['threat modeling', 'cyber security', 'appsec'], packages=find_packages(), include_package_data=True, - python_requires='>= 3.9, <= 3.12', + python_requires='>= 3.9, < 3.13', install_requires=[ 'pyyaml==6.0.1', 'jsonschema==4.19.0', @@ -25,19 +25,19 @@ "python-multipart==0.0.19", 'click==8.1.7', 'uvicorn==0.23.2', - 'shapely==2.0.1', 'vsdx==0.5.19', 'python-magic==0.4.27', 'setuptools==70.3.0', 'setuptools-scm==8.1.0', 'defusedxml==0.7.1', 'networkx==3.1', - 'dependency-injector==4.41.0', - 'google-re2==1.0', + 'dependency-injector==4.46.0', 'xmlschema==2.5.0', 'word2number==1.1', - # Do not upgrade pygraphviz unless security issues because it is heavily dependent on the underlying OS - 'pygraphviz==1.10', + # These dependencies are heavily dependent on the underlying OS + 'pygraphviz==1.13', + 'shapely==2.0.6', + 'google-re2', # Numpy is a transitive dependency of fastapi, requests and python-multipart # They require different v1 versions, while v2 versions lead to import errors 'numpy<2.0' From fd3cf779c2b826d74e5750f939c51201dab86451 Mon Sep 17 00:00:00 2001 From: Santi Manero Date: Mon, 17 Mar 2025 16:03:56 +0100 Subject: [PATCH 02/12] [BLAZ-1548] Fix pygraphviz version for windows pipelines --- .github/actions/install-startleft/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/install-startleft/action.yml b/.github/actions/install-startleft/action.yml index 3c6bcb54..065d0511 100644 --- a/.github/actions/install-startleft/action.yml +++ b/.github/actions/install-startleft/action.yml @@ -29,7 +29,7 @@ runs: if: runner.os == 'Windows' shell: bash run: | - pip install --global-option=build_ext --global-option="-IC:\Program files\Graphviz\include" --global-option="-LC:\Program files\Graphviz\lib" pygraphviz==1.10 + pip install --global-option=build_ext --global-option="-IC:\Program files\Graphviz\include" --global-option="-LC:\Program files\Graphviz\lib" pygraphviz==1.13 echo "C:\Program Files\Graphviz\bin" >> $GITHUB_PATH - name: Install dependencies From f1ac9dad99bdeb425fdb73fb38038394694246da Mon Sep 17 00:00:00 2001 From: Santi Manero Date: Mon, 17 Mar 2025 16:21:50 +0100 Subject: [PATCH 03/12] [BLAZ-1548] Fixed dependencies on python 3.9. Added 3.12 version to qa test --- .github/workflows/qa.yml | 2 +- setup.py | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/qa.yml b/.github/workflows/qa.yml index bf44690a..0ba74b52 100644 --- a/.github/workflows/qa.yml +++ b/.github/workflows/qa.yml @@ -21,7 +21,7 @@ jobs: strategy: matrix: os: [ ubuntu-latest, windows-latest ] - python-version: [ "3.9", "3.10", "3.11" ] + python-version: [ "3.9", "3.10", "3.11", "3.12" ] uses: ./.github/workflows/test.yml with: os: ${{ matrix.os }} diff --git a/setup.py b/setup.py index 269ba5e1..2516baba 100644 --- a/setup.py +++ b/setup.py @@ -1,8 +1,12 @@ +import sys + from setuptools import setup, find_packages from startleft.startleft._version.local_scheme import guess_startleft_semver_suffix from startleft.startleft._version.version_scheme import guess_startleft_semver +pygraphviz_version = '1.10' if sys.version_info < (3, 9) else '1.13' + setup( name='startleft', description='Parse Infrastructure as Code files to the Open Threat Model format and upload them to IriusRisk', @@ -35,7 +39,7 @@ 'xmlschema==2.5.0', 'word2number==1.1', # These dependencies are heavily dependent on the underlying OS - 'pygraphviz==1.13', + f'pygraphviz=={pygraphviz_version}', 'shapely==2.0.6', 'google-re2', # Numpy is a transitive dependency of fastapi, requests and python-multipart From af84b25c9e6b128ab826a98f6383d2a603c951c0 Mon Sep 17 00:00:00 2001 From: Santi Manero Date: Tue, 18 Mar 2025 08:07:07 +0100 Subject: [PATCH 04/12] [BLAZ-1548] Fixed dependencies on python 3.9 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 2516baba..f7550de1 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ from startleft.startleft._version.local_scheme import guess_startleft_semver_suffix from startleft.startleft._version.version_scheme import guess_startleft_semver -pygraphviz_version = '1.10' if sys.version_info < (3, 9) else '1.13' +pygraphviz_version = '1.10' if sys.version_info < (3, 10) else '1.13' setup( name='startleft', From fd65ecc6c36457bb46fdd89dd8328f7d3b38a03a Mon Sep 17 00:00:00 2001 From: Santi Manero Date: Tue, 18 Mar 2025 08:28:07 +0100 Subject: [PATCH 05/12] [BLAZ-1548] Removed qa action for 3.12 --- .github/workflows/qa.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/qa.yml b/.github/workflows/qa.yml index 0ba74b52..9dd45d4f 100644 --- a/.github/workflows/qa.yml +++ b/.github/workflows/qa.yml @@ -21,7 +21,7 @@ jobs: strategy: matrix: os: [ ubuntu-latest, windows-latest ] - python-version: [ "3.9", "3.10", "3.11", "3.12" ] + python-version: [ "3.9", "3.10", "3.11"] uses: ./.github/workflows/test.yml with: os: ${{ matrix.os }} From 5dea3a4d48b17a399f636b2a1c1ac83f6ed35794 Mon Sep 17 00:00:00 2001 From: Santi Manero Date: Tue, 18 Mar 2025 08:58:00 +0100 Subject: [PATCH 06/12] [BLAZ-1548] Fix github action for windows and python3.9 --- .github/actions/install-startleft/action.yml | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/.github/actions/install-startleft/action.yml b/.github/actions/install-startleft/action.yml index 065d0511..5208e36b 100644 --- a/.github/actions/install-startleft/action.yml +++ b/.github/actions/install-startleft/action.yml @@ -25,11 +25,26 @@ runs: - name: Setup Graphviz uses: ts-graphviz/setup-graphviz@55fcdfa9328aed658432c22011bec2873cd8e69f # Securely point to hash commit for v1 + - name: Set pygraphviz default version + env: + PYGRAPHVIZ_VERSION: "1.13" + run: echo "PYGRAPHVIZ_VERSION=$PYGRAPHVIZ_VERSION" >> "$GITHUB_ENV" + shell: bash + + - name: Set pygraphviz version for Python 3.9 + if: ${{ inputs.python-version == '3.9' }} + env: + PYGRAPHVIZ_VERSION: "1.10" + run: echo "PYGRAPHVIZ_VERSION=$PYGRAPHVIZ_VERSION" >> "$GITHUB_ENV" + shell: bash + + + - name: Configure Graphviz in Windows if: runner.os == 'Windows' shell: bash run: | - pip install --global-option=build_ext --global-option="-IC:\Program files\Graphviz\include" --global-option="-LC:\Program files\Graphviz\lib" pygraphviz==1.13 + pip install --global-option=build_ext --global-option="-IC:\Program files\Graphviz\include" --global-option="-LC:\Program files\Graphviz\lib" pygraphviz=="$PYGRAPHVIZ_VERSION" echo "C:\Program Files\Graphviz\bin" >> $GITHUB_PATH - name: Install dependencies From afe5b53cc07773918be6fd631d7a25c2a1efb83f Mon Sep 17 00:00:00 2001 From: Santi Manero Date: Wed, 19 Mar 2025 13:19:47 +0100 Subject: [PATCH 07/12] [BLAZ-1548] Removed windows tests --- .github/actions/install-startleft/action.yml | 27 -------------------- .github/workflows/qa.yml | 4 +-- 2 files changed, 2 insertions(+), 29 deletions(-) diff --git a/.github/actions/install-startleft/action.yml b/.github/actions/install-startleft/action.yml index 5208e36b..488cc4b3 100644 --- a/.github/actions/install-startleft/action.yml +++ b/.github/actions/install-startleft/action.yml @@ -25,35 +25,8 @@ runs: - name: Setup Graphviz uses: ts-graphviz/setup-graphviz@55fcdfa9328aed658432c22011bec2873cd8e69f # Securely point to hash commit for v1 - - name: Set pygraphviz default version - env: - PYGRAPHVIZ_VERSION: "1.13" - run: echo "PYGRAPHVIZ_VERSION=$PYGRAPHVIZ_VERSION" >> "$GITHUB_ENV" - shell: bash - - - name: Set pygraphviz version for Python 3.9 - if: ${{ inputs.python-version == '3.9' }} - env: - PYGRAPHVIZ_VERSION: "1.10" - run: echo "PYGRAPHVIZ_VERSION=$PYGRAPHVIZ_VERSION" >> "$GITHUB_ENV" - shell: bash - - - - - name: Configure Graphviz in Windows - if: runner.os == 'Windows' - shell: bash - run: | - pip install --global-option=build_ext --global-option="-IC:\Program files\Graphviz\include" --global-option="-LC:\Program files\Graphviz\lib" pygraphviz=="$PYGRAPHVIZ_VERSION" - echo "C:\Program Files\Graphviz\bin" >> $GITHUB_PATH - - name: Install dependencies run: pip install . && pip install -e ".[setup,test]" shell: bash - # This step MUST be after the general installation of StartLeft - - name: Install libmagic in Windows - if: runner.os == 'Windows' - run: pip install python-magic-bin - shell: bash diff --git a/.github/workflows/qa.yml b/.github/workflows/qa.yml index 9dd45d4f..ccde9e9c 100644 --- a/.github/workflows/qa.yml +++ b/.github/workflows/qa.yml @@ -20,8 +20,8 @@ jobs: name: StartLeft Tests strategy: matrix: - os: [ ubuntu-latest, windows-latest ] - python-version: [ "3.9", "3.10", "3.11"] + os: [ ubuntu-latest ] + python-version: [ "3.10", "3.11"] uses: ./.github/workflows/test.yml with: os: ${{ matrix.os }} From a18157f3e0f4b37e6a0f33c82cefdbc1919d5846 Mon Sep 17 00:00:00 2001 From: Santi Manero Date: Wed, 19 Mar 2025 14:50:14 +0100 Subject: [PATCH 08/12] [BLAZ-1548] Add python 3.12 qa tests to github action. Remove compatibility for python 3.9 --- .github/workflows/qa.yml | 2 +- setup.py | 7 ++----- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/.github/workflows/qa.yml b/.github/workflows/qa.yml index ccde9e9c..019de386 100644 --- a/.github/workflows/qa.yml +++ b/.github/workflows/qa.yml @@ -21,7 +21,7 @@ jobs: strategy: matrix: os: [ ubuntu-latest ] - python-version: [ "3.10", "3.11"] + python-version: [ "3.10", "3.11", "3.12"] uses: ./.github/workflows/test.yml with: os: ${{ matrix.os }} diff --git a/setup.py b/setup.py index f7550de1..58cff643 100644 --- a/setup.py +++ b/setup.py @@ -1,11 +1,8 @@ -import sys - from setuptools import setup, find_packages from startleft.startleft._version.local_scheme import guess_startleft_semver_suffix from startleft.startleft._version.version_scheme import guess_startleft_semver -pygraphviz_version = '1.10' if sys.version_info < (3, 10) else '1.13' setup( name='startleft', @@ -17,7 +14,7 @@ keywords=['threat modeling', 'cyber security', 'appsec'], packages=find_packages(), include_package_data=True, - python_requires='>= 3.9, < 3.13', + python_requires='>= 3.10, < 3.13', install_requires=[ 'pyyaml==6.0.1', 'jsonschema==4.19.0', @@ -39,7 +36,7 @@ 'xmlschema==2.5.0', 'word2number==1.1', # These dependencies are heavily dependent on the underlying OS - f'pygraphviz=={pygraphviz_version}', + 'pygraphviz==1.13', 'shapely==2.0.6', 'google-re2', # Numpy is a transitive dependency of fastapi, requests and python-multipart From 8531332c81ae3f42ca444cac4cc6130536112d0e Mon Sep 17 00:00:00 2001 From: Santi Manero Date: Wed, 19 Mar 2025 15:15:04 +0100 Subject: [PATCH 09/12] [BLAZ-1548] Fix github action qa tests over python 3.12 --- .github/actions/install-startleft/action.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/actions/install-startleft/action.yml b/.github/actions/install-startleft/action.yml index 488cc4b3..037ebfb7 100644 --- a/.github/actions/install-startleft/action.yml +++ b/.github/actions/install-startleft/action.yml @@ -14,19 +14,19 @@ runs: with: python-version: ${{ inputs.python-version }} - - name: Update pip version to 23.0.1 - run: python -m pip install --use-pep517 --upgrade pip==23.0.1 + - name: Ensure pip is up-to-date + run: python -m ensurepip --upgrade pip shell: bash - - name: Set setuptools to v70 for compatibility with setuptools-scm - run: python -m pip install setuptools==70.3.0 + - name: Upgrade setuptools for compatibility with setuptools-scm + run: python -m pip install --upgrade setuptools shell: bash - name: Setup Graphviz uses: ts-graphviz/setup-graphviz@55fcdfa9328aed658432c22011bec2873cd8e69f # Securely point to hash commit for v1 - name: Install dependencies - run: pip install . && pip install -e ".[setup,test]" + run: python -m pip install . && python -m pip install -e ".[setup,test]" shell: bash From 6d5bd99ed161d339663c220556bf0adf2e7094aa Mon Sep 17 00:00:00 2001 From: Santi Manero Date: Wed, 19 Mar 2025 15:17:20 +0100 Subject: [PATCH 10/12] [BLAZ-1548] Fix github action qa tests over python 3.12 --- .github/actions/install-startleft/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/install-startleft/action.yml b/.github/actions/install-startleft/action.yml index 037ebfb7..ac7d8ace 100644 --- a/.github/actions/install-startleft/action.yml +++ b/.github/actions/install-startleft/action.yml @@ -15,7 +15,7 @@ runs: python-version: ${{ inputs.python-version }} - name: Ensure pip is up-to-date - run: python -m ensurepip --upgrade pip + run: python -m ensurepip --upgrade shell: bash - name: Upgrade setuptools for compatibility with setuptools-scm From 11325b2a730d20ee370e764060d1bfd732b4369a Mon Sep 17 00:00:00 2001 From: Santi Manero Date: Thu, 20 Mar 2025 07:44:47 +0100 Subject: [PATCH 11/12] [BLAZ-1548] Minor changes on Dockerfile --- deployment/Dockerfile.application | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/deployment/Dockerfile.application b/deployment/Dockerfile.application index 1c329d08..2d8f8720 100644 --- a/deployment/Dockerfile.application +++ b/deployment/Dockerfile.application @@ -5,13 +5,9 @@ WORKDIR /app COPY . . RUN apk -U upgrade && \ - apk add git && \ apk add --no-cache cblas geos git lapack libmagic py3-pygraphviz py3-re2 py3-shapely re2 && \ - cp -R /usr/lib/python3.12/site-packages /usr/local/lib/python3.12/ - -RUN ls -a /usr/local/lib/python3.12/site-packages - -RUN pip install --upgrade pip && pip install . + cp -R /usr/lib/python3.12/site-packages /usr/local/lib/python3.12/ && \ + pip install --upgrade pip && pip install . FROM python:3.12-alpine3.21 From 7297b69d8031173c84c35b14983f71d01cd3a1a6 Mon Sep 17 00:00:00 2001 From: Santi Manero Date: Thu, 20 Mar 2025 09:58:07 +0100 Subject: [PATCH 12/12] [BLAZ-1548] Added uvicorn to docker --- deployment/Dockerfile.application | 1 + 1 file changed, 1 insertion(+) diff --git a/deployment/Dockerfile.application b/deployment/Dockerfile.application index 2d8f8720..f7a40c71 100644 --- a/deployment/Dockerfile.application +++ b/deployment/Dockerfile.application @@ -19,6 +19,7 @@ RUN apk -U upgrade && \ COPY --from=startleft-base /usr/local/lib/python3.12/site-packages /usr/local/lib/python3.12/site-packages COPY --from=startleft-base /usr/local/bin/startleft /usr/local/bin/startleft +COPY --from=startleft-base /usr/local/bin/uvicorn /usr/local/bin/uvicorn USER startleft