From 34d9044a2cf8595077ae0acf9de477ea65bd8ed5 Mon Sep 17 00:00:00 2001 From: abikouo Date: Thu, 23 Feb 2023 16:01:50 +0100 Subject: [PATCH 01/20] update changelog workflow --- .github/scripts/validate_changelog.py | 147 ++++++++++++++++++++++++++ .github/workflows/changelog.yml | 27 +++-- 2 files changed, 167 insertions(+), 7 deletions(-) create mode 100644 .github/scripts/validate_changelog.py diff --git a/.github/scripts/validate_changelog.py b/.github/scripts/validate_changelog.py new file mode 100644 index 00000000..5a1dc2fa --- /dev/null +++ b/.github/scripts/validate_changelog.py @@ -0,0 +1,147 @@ +#!/usr/bin/python + +import re +import os +from collections import defaultdict +import yaml +import subprocess +import sys +import argparse + + +def is_valid_change_log(ref): + return re.match("^changelogs/fragments/(.*)\.(yaml|yml)$", ref) + +def is_module_or_plugin(ref): + prefix_list = ( + "plugins/modules", + "plugins/action", + "plugins/inventory", + "plugins/lookup", + "plugins/filter", + "plugins/connection", + "plugins/become", + "plugins/cache", + "plugins/callback", + "plugins/cliconf", + "plugins/httpapi", + "plugins/netconf", + "plugins/shell", + "plugins/strategy", + "plugins/terminal", + "plugins/test", + "plugins/vars", + ) + return ref.startswith(prefix_list) + + +def is_documentation_file(ref): + prefix_list = ( + "docs/", + "plugins/doc_fragments", + ) + return ref.startswith(prefix_list) + +def is_added_module_or_plugin_or_documentation_changes(changes): + + # Validate Pull request add new modules and plugins + if any([is_module_or_plugin(x) for x in changes["A"]]): + return True + + # Validate documentation changes only + all_files = changes["A"] + changes["M"] + changes["D"] + if all([is_documentation_file(x) for x in all_files]): + return True + + return False + +def validate_changelog(path): + + try: + # https://github.com/ansible-community/antsibull-changelog/blob/main/docs/changelogs.rst#changelog-fragment-categories + changes_type = ( + "release_summary", + "breaking_changes", + "major_changes", + "minor_changes", + "removed_features", + "deprecated_features", + "security_fixes", + "bugfixes", + "known_issues", + "trivial", + ) + with open(path, "rb") as f: + result = list(yaml.safe_load_all(f)) + + for section in result: + for key in section.keys(): + if key not in changes_type: + print("Unexpected changelog section {0} from file {1}".format(key, os.path.basename(path))) + return False + if not isinstance(section[key], list): + print( + "Changelog section {0} from file {1} must be a list, {2} found instead.".format( + key, + os.path.basename(path), + type(section[key]) + ) + ) + return False + return True + except (IOError, yaml.YAMLError) as exc: + print("Error loading changelog file {0}: {1}".format(os.path.basename(path),exc)) + return False + +def run_command(cmd): + params = { + "stdout": subprocess.PIPE, + "stderr": subprocess.PIPE, + "shell": True + } + proc = subprocess.Popen(cmd, **params) + out, err = proc.communicate() + return proc.returncode, out, err + +def list_files(head_ref, base_ref): + cmd = "git diff origin/{0} {1} --name-status".format(base_ref, head_ref) + print("Running command '{0}'".format(cmd)) + rc, stdout, stderr = run_command(cmd) + if rc != 0: + raise ValueError(stderr) + + changes = defaultdict(list) + for file in stdout.decode("utf-8").split("\n"): + v = file.split("\t") + if len(v) == 2: + changes[v[0]].append(v[1]) + return changes + +def main(head_ref, base_ref): + + changes = list_files(head_ref, base_ref) + if changes: + changelog = [x for x in changes["A"] if is_valid_change_log(x)] + if not changelog: + if not is_added_module_or_plugin_or_documentation_changes(changes): + print( + "Missing changelog fragment. This is not required only if "\ + "PR adds new modules and plugins or contain only documentation changes." + ) + sys.exit(1) + print( + "Changelog not required as PR adds new modules and/or plugins or "\ + "contain only documentation changes." + ) + elif any(not validate_changelog(f) for f in changelog): + sys.exit(1) + sys.exit(0) + +if __name__ == "__main__": + + parser = argparse.ArgumentParser(description="Validate changelog file from new commit") + parser.add_argument("--base-ref", required=True) + parser.add_argument("--head-ref", required=True) + + args = parser.parse_args() + main(args.head_ref, args.base_ref) diff --git a/.github/workflows/changelog.yml b/.github/workflows/changelog.yml index 0b365a25..0fc56784 100644 --- a/.github/workflows/changelog.yml +++ b/.github/workflows/changelog.yml @@ -5,6 +5,10 @@ jobs: changelog: runs-on: ubuntu-latest name: Require a changelog + env: + PY_COLORS: "1" + source_directory: "./source" + download_directory: "./downloads" steps: - name: Checkout the collection repository uses: actions/checkout@v3 @@ -12,12 +16,21 @@ jobs: path: ${{ env.source_directory }} ref: ${{ github.event.pull_request.head.sha }} fetch-depth: "0" - if: "!contains(github.event.pull_request.labels.*.name, 'skip-changelog')" - - name: Ensure a new changelog entry exists + - name: setup python + uses: actions/setup-python@v4 + with: + python-version: '3.9' + + - name: Download python script + run: curl -o ${{ env.download_directory }}/validate_changelog.py https://raw.githubusercontent.com/ansible-network/github_actions/blob/main/scripts/validate_changelog.py + + - name: Install python required libraries + run: pip install -U pyyaml + + - name: Ensure a valid changelog entry exists run: >- - git show origin/${{ github.event.pull_request.base.ref }}..HEAD - --name-status - --oneline | - grep -E -e "A\s+changelogs/fragments/" -e "M\s+CHANGELOG.rst" - if: "!contains(github.event.pull_request.labels.*.name, 'skip-changelog')" + python ${{ env.download_directory }}/validate_changelog.py + --base-ref ${{ github.event.pull_request.base.ref }} + --head-ref ${{ github.event.pull_request.head.sha }} + working-directory: ${{ env.source_directory }} From ce6ff128d8b3c8b98e41050c0d2b7c136d513977 Mon Sep 17 00:00:00 2001 From: abikouo Date: Wed, 22 Mar 2023 09:27:56 +0100 Subject: [PATCH 02/20] add the possibility to skip changelog validation --- .github/workflows/changelog.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/changelog.yml b/.github/workflows/changelog.yml index 0fc56784..205f0253 100644 --- a/.github/workflows/changelog.yml +++ b/.github/workflows/changelog.yml @@ -5,10 +5,10 @@ jobs: changelog: runs-on: ubuntu-latest name: Require a changelog + if: "!contains(github.event.pull_request.labels.*.name, 'skip-changelog')" env: PY_COLORS: "1" source_directory: "./source" - download_directory: "./downloads" steps: - name: Checkout the collection repository uses: actions/checkout@v3 @@ -23,14 +23,14 @@ jobs: python-version: '3.9' - name: Download python script - run: curl -o ${{ env.download_directory }}/validate_changelog.py https://raw.githubusercontent.com/ansible-network/github_actions/blob/main/scripts/validate_changelog.py + run: curl -o /tmp/validate_changelog.py https://raw.githubusercontent.com/ansible-network/github_actions/blob/main/scripts/validate_changelog.py - name: Install python required libraries run: pip install -U pyyaml - name: Ensure a valid changelog entry exists run: >- - python ${{ env.download_directory }}/validate_changelog.py + python /tmp/validate_changelog.py --base-ref ${{ github.event.pull_request.base.ref }} --head-ref ${{ github.event.pull_request.head.sha }} working-directory: ${{ env.source_directory }} From 4b96b1a2384fb21fc9a5ef957be217644babc8d1 Mon Sep 17 00:00:00 2001 From: Kate Case Date: Wed, 22 Mar 2023 11:21:47 -0400 Subject: [PATCH 03/20] pre-commit fixes --- .github/scripts/validate_changelog.py | 2 +- .github/workflows/changelog.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/scripts/validate_changelog.py b/.github/scripts/validate_changelog.py index 5a1dc2fa..0b05a014 100644 --- a/.github/scripts/validate_changelog.py +++ b/.github/scripts/validate_changelog.py @@ -40,7 +40,7 @@ def is_documentation_file(ref): "docs/", "plugins/doc_fragments", ) - return ref.startswith(prefix_list) + return ref.startswith(prefix_list) def is_added_module_or_plugin_or_documentation_changes(changes): diff --git a/.github/workflows/changelog.yml b/.github/workflows/changelog.yml index 205f0253..2d80ea14 100644 --- a/.github/workflows/changelog.yml +++ b/.github/workflows/changelog.yml @@ -20,7 +20,7 @@ jobs: - name: setup python uses: actions/setup-python@v4 with: - python-version: '3.9' + python-version: "3.9" - name: Download python script run: curl -o /tmp/validate_changelog.py https://raw.githubusercontent.com/ansible-network/github_actions/blob/main/scripts/validate_changelog.py From ac8bbcdcd6f37b9ddd76a6396afe47b276a2b1b7 Mon Sep 17 00:00:00 2001 From: Kate Case Date: Wed, 22 Mar 2023 11:35:55 -0400 Subject: [PATCH 04/20] Add black ans isort to pre-commit --- .github/scripts/validate_changelog.py | 48 +++++++++++++++------------ .pre-commit-config.yaml | 11 ++++++ 2 files changed, 38 insertions(+), 21 deletions(-) diff --git a/.github/scripts/validate_changelog.py b/.github/scripts/validate_changelog.py index 0b05a014..8ba64374 100644 --- a/.github/scripts/validate_changelog.py +++ b/.github/scripts/validate_changelog.py @@ -1,17 +1,19 @@ #!/usr/bin/python -import re +import argparse import os -from collections import defaultdict -import yaml +import re import subprocess import sys -import argparse +from collections import defaultdict + +import yaml def is_valid_change_log(ref): return re.match("^changelogs/fragments/(.*)\.(yaml|yml)$", ref) + def is_module_or_plugin(ref): prefix_list = ( "plugins/modules", @@ -42,8 +44,8 @@ def is_documentation_file(ref): ) return ref.startswith(prefix_list) -def is_added_module_or_plugin_or_documentation_changes(changes): +def is_added_module_or_plugin_or_documentation_changes(changes): # Validate Pull request add new modules and plugins if any([is_module_or_plugin(x) for x in changes["A"]]): return True @@ -55,8 +57,8 @@ def is_added_module_or_plugin_or_documentation_changes(changes): return False -def validate_changelog(path): +def validate_changelog(path): try: # https://github.com/ansible-community/antsibull-changelog/blob/main/docs/changelogs.rst#changelog-fragment-categories changes_type = ( @@ -77,32 +79,34 @@ def validate_changelog(path): for section in result: for key in section.keys(): if key not in changes_type: - print("Unexpected changelog section {0} from file {1}".format(key, os.path.basename(path))) + print( + "Unexpected changelog section {0} from file {1}".format( + key, os.path.basename(path) + ) + ) return False if not isinstance(section[key], list): print( "Changelog section {0} from file {1} must be a list, {2} found instead.".format( - key, - os.path.basename(path), - type(section[key]) + key, os.path.basename(path), type(section[key]) ) ) return False return True except (IOError, yaml.YAMLError) as exc: - print("Error loading changelog file {0}: {1}".format(os.path.basename(path),exc)) + print( + "Error loading changelog file {0}: {1}".format(os.path.basename(path), exc) + ) return False + def run_command(cmd): - params = { - "stdout": subprocess.PIPE, - "stderr": subprocess.PIPE, - "shell": True - } + params = {"stdout": subprocess.PIPE, "stderr": subprocess.PIPE, "shell": True} proc = subprocess.Popen(cmd, **params) out, err = proc.communicate() return proc.returncode, out, err + def list_files(head_ref, base_ref): cmd = "git diff origin/{0} {1} --name-status".format(base_ref, head_ref) print("Running command '{0}'".format(cmd)) @@ -117,29 +121,31 @@ def list_files(head_ref, base_ref): changes[v[0]].append(v[1]) return changes -def main(head_ref, base_ref): +def main(head_ref, base_ref): changes = list_files(head_ref, base_ref) if changes: changelog = [x for x in changes["A"] if is_valid_change_log(x)] if not changelog: if not is_added_module_or_plugin_or_documentation_changes(changes): print( - "Missing changelog fragment. This is not required only if "\ + "Missing changelog fragment. This is not required only if " "PR adds new modules and plugins or contain only documentation changes." ) sys.exit(1) print( - "Changelog not required as PR adds new modules and/or plugins or "\ + "Changelog not required as PR adds new modules and/or plugins or " "contain only documentation changes." ) elif any(not validate_changelog(f) for f in changelog): sys.exit(1) sys.exit(0) -if __name__ == "__main__": - parser = argparse.ArgumentParser(description="Validate changelog file from new commit") +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Validate changelog file from new commit" + ) parser.add_argument("--base-ref", required=True) parser.add_argument("--head-ref", required=True) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2a7cfa0e..98335e6a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -15,3 +15,14 @@ repos: additional_dependencies: - prettier - prettier-plugin-toml + + - repo: https://github.com/PyCQA/isort + rev: 5.12.0 + hooks: + - id: isort + args: ["--filter-files"] + + - repo: https://github.com/psf/black + rev: 23.1.0 + hooks: + - id: black From a5711b60d774cc44aed79a070a34a6f001c40edf Mon Sep 17 00:00:00 2001 From: abikouo Date: Wed, 22 Mar 2023 16:42:48 +0100 Subject: [PATCH 05/20] add test workflow --- .github/scripts/validate_changelog.py | 46 +++++++++++++---------- mypy.ini | 14 +++++++ tox.ini | 54 +++++++++++++++++++++++++++ 3 files changed, 95 insertions(+), 19 deletions(-) create mode 100644 mypy.ini create mode 100644 tox.ini diff --git a/.github/scripts/validate_changelog.py b/.github/scripts/validate_changelog.py index 8ba64374..1b65f953 100644 --- a/.github/scripts/validate_changelog.py +++ b/.github/scripts/validate_changelog.py @@ -1,7 +1,7 @@ #!/usr/bin/python import argparse -import os +import logging import re import subprocess import sys @@ -9,9 +9,15 @@ import yaml +FORMAT = "[%(asctime)s] - %(message)s" +logging.basicConfig(format=FORMAT) +logger = logging.getLogger("validate_changelog") +logger.setLevel(logging.DEBUG) + def is_valid_change_log(ref): - return re.match("^changelogs/fragments/(.*)\.(yaml|yml)$", ref) + return re.match(r"^changelogs/fragments/(.*)\.(yaml|yml)$", ref) + def is_module_or_plugin(ref): @@ -79,29 +85,28 @@ def validate_changelog(path): for section in result: for key in section.keys(): if key not in changes_type: - print( - "Unexpected changelog section {0} from file {1}".format( - key, os.path.basename(path) - ) - ) + msg = f"{key} from {path} is not a valid changelog type" + logger.error(msg) return False if not isinstance(section[key], list): - print( - "Changelog section {0} from file {1} must be a list, {2} found instead.".format( - key, os.path.basename(path), type(section[key]) + logger.error( + "Changelog section {0} from file {1} must be a list," + " {2} found instead.".format( + key, + path, + type(section[key]), ) ) return False return True except (IOError, yaml.YAMLError) as exc: - print( - "Error loading changelog file {0}: {1}".format(os.path.basename(path), exc) - ) + msg = "yaml loading error for file {0} -> {1}".format(path, exc) + logger.error(msg) return False def run_command(cmd): - params = {"stdout": subprocess.PIPE, "stderr": subprocess.PIPE, "shell": True} + params = dict(stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) proc = subprocess.Popen(cmd, **params) out, err = proc.communicate() return proc.returncode, out, err @@ -109,7 +114,7 @@ def run_command(cmd): def list_files(head_ref, base_ref): cmd = "git diff origin/{0} {1} --name-status".format(base_ref, head_ref) - print("Running command '{0}'".format(cmd)) + logger.info("Executing '{0}'".format(cmd)) rc, stdout, stderr = run_command(cmd) if rc != 0: raise ValueError(stderr) @@ -122,6 +127,8 @@ def list_files(head_ref, base_ref): return changes +def main(head_ref, base_ref): + def main(head_ref, base_ref): changes = list_files(head_ref, base_ref) if changes: @@ -129,13 +136,14 @@ def main(head_ref, base_ref): if not changelog: if not is_added_module_or_plugin_or_documentation_changes(changes): print( - "Missing changelog fragment. This is not required only if " - "PR adds new modules and plugins or contain only documentation changes." + "Missing changelog fragment. This is not required" + " only if PR adds new modules and plugins or contain" + " only documentation changes." ) sys.exit(1) print( - "Changelog not required as PR adds new modules and/or plugins or " - "contain only documentation changes." + "Changelog not required as PR adds new modules and/or" + " plugins or contain only documentation changes." ) elif any(not validate_changelog(f) for f in changelog): sys.exit(1) diff --git a/mypy.ini b/mypy.ini new file mode 100644 index 00000000..62498c62 --- /dev/null +++ b/mypy.ini @@ -0,0 +1,14 @@ +[mypy] +files = + docs/, + share/, + src/, + tests/ +install_types = true +namespace_packages = true +no_implicit_optional = true +non_interactive = true +pretty = true +show_column_numbers = true +show_error_codes = true +show_error_context = true \ No newline at end of file diff --git a/tox.ini b/tox.ini new file mode 100644 index 00000000..3470ba68 --- /dev/null +++ b/tox.ini @@ -0,0 +1,54 @@ +[tox] +minversion = 1.4.2 +skipsdist = True + +[testenv:black] +deps = + black >= 22.0, < 23.0 + +commands = + black -v --check --diff .github + +[testenv:flake8] +deps = + flake8 + +commands = + flake8 .github + +[testenv:isort] +deps = + isort + +commands = + isort --check .github + +[testenv:mypy] +deps = + mypy + +commands = + mypy .github + +[testenv:black_format] +deps = + {[testenv:black]deps} + +commands = + black -v {posargs} + +[testenv:linters] + +deps = + + {[testenv:black]deps} + {[testenv:flake8]deps} + {[testenv:isort]deps} + {[testenv:mypy]deps} + +commands = + + {[testenv:black]commands} + {[testenv:flake8]commands} + {[testenv:isort]commands} + {[testenv:mypy]commands} \ No newline at end of file From 0317a4ac6fc1bee940e9afd9d3bbac09462679bb Mon Sep 17 00:00:00 2001 From: abikouo Date: Wed, 22 Mar 2023 16:46:27 +0100 Subject: [PATCH 06/20] add ci validation --- .github/workflows/local.yml | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 .github/workflows/local.yml diff --git a/.github/workflows/local.yml b/.github/workflows/local.yml new file mode 100644 index 00000000..ea34d4f1 --- /dev/null +++ b/.github/workflows/local.yml @@ -0,0 +1,36 @@ +name: Tests +concurrency: + group: ${{ github.head_ref }} + cancel-in-progress: true + +on: + pull_request: + types: + - opened + - reopened + - labeled + - unlabeled + - synchronize + branches: + - '*' + tags: + - '*' + +jobs: + linters: + runs-on: ubuntu-latest + steps: + - name: Checkout the repository + uses: actions/checkout@v3 + + - name: setup python + uses: actions/setup-python@v4 + with: + python-version: '3.9' + + - name: install tox + run: pip install -U tox + shell: bash + + - name: Run unit tests + run: tox -e linters -vv \ No newline at end of file From 18d3d554a0d922d6d5b3952cddf55978c178b134 Mon Sep 17 00:00:00 2001 From: abikouo Date: Wed, 22 Mar 2023 16:50:50 +0100 Subject: [PATCH 07/20] rebase --- .github/scripts/validate_changelog.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/scripts/validate_changelog.py b/.github/scripts/validate_changelog.py index 1b65f953..b9bacf35 100644 --- a/.github/scripts/validate_changelog.py +++ b/.github/scripts/validate_changelog.py @@ -19,7 +19,6 @@ def is_valid_change_log(ref): return re.match(r"^changelogs/fragments/(.*)\.(yaml|yml)$", ref) - def is_module_or_plugin(ref): prefix_list = ( "plugins/modules", @@ -127,21 +126,19 @@ def list_files(head_ref, base_ref): return changes -def main(head_ref, base_ref): - def main(head_ref, base_ref): changes = list_files(head_ref, base_ref) if changes: changelog = [x for x in changes["A"] if is_valid_change_log(x)] if not changelog: if not is_added_module_or_plugin_or_documentation_changes(changes): - print( + logger.error( "Missing changelog fragment. This is not required" " only if PR adds new modules and plugins or contain" " only documentation changes." ) sys.exit(1) - print( + logger.info( "Changelog not required as PR adds new modules and/or" " plugins or contain only documentation changes." ) From 89046e696afa7ecdd7af284bfb32e649a910943d Mon Sep 17 00:00:00 2001 From: abikouo Date: Wed, 22 Mar 2023 17:04:24 +0100 Subject: [PATCH 08/20] move scripts from .github dir --- {.github/scripts => scripts}/validate_changelog.py | 0 tox.ini | 8 ++++---- 2 files changed, 4 insertions(+), 4 deletions(-) rename {.github/scripts => scripts}/validate_changelog.py (100%) diff --git a/.github/scripts/validate_changelog.py b/scripts/validate_changelog.py similarity index 100% rename from .github/scripts/validate_changelog.py rename to scripts/validate_changelog.py diff --git a/tox.ini b/tox.ini index 3470ba68..eacc2a80 100644 --- a/tox.ini +++ b/tox.ini @@ -7,28 +7,28 @@ deps = black >= 22.0, < 23.0 commands = - black -v --check --diff .github + black -v --check --diff scripts [testenv:flake8] deps = flake8 commands = - flake8 .github + flake8 scripts [testenv:isort] deps = isort commands = - isort --check .github + isort --check scripts [testenv:mypy] deps = mypy commands = - mypy .github + mypy scripts [testenv:black_format] deps = From 627b170110b0bcb91d3f55aa02b99f1acc66de7d Mon Sep 17 00:00:00 2001 From: abikouo Date: Wed, 22 Mar 2023 17:09:23 +0100 Subject: [PATCH 09/20] fix end-of-file linters --- .github/workflows/local.yml | 2 +- mypy.ini | 2 +- tox.ini | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/local.yml b/.github/workflows/local.yml index ea34d4f1..923d3a4e 100644 --- a/.github/workflows/local.yml +++ b/.github/workflows/local.yml @@ -33,4 +33,4 @@ jobs: shell: bash - name: Run unit tests - run: tox -e linters -vv \ No newline at end of file + run: tox -e linters -vv diff --git a/mypy.ini b/mypy.ini index 62498c62..b7e91a29 100644 --- a/mypy.ini +++ b/mypy.ini @@ -11,4 +11,4 @@ non_interactive = true pretty = true show_column_numbers = true show_error_codes = true -show_error_context = true \ No newline at end of file +show_error_context = true diff --git a/tox.ini b/tox.ini index eacc2a80..2f2c450e 100644 --- a/tox.ini +++ b/tox.ini @@ -51,4 +51,4 @@ commands = {[testenv:black]commands} {[testenv:flake8]commands} {[testenv:isort]commands} - {[testenv:mypy]commands} \ No newline at end of file + {[testenv:mypy]commands} From d875632f31484bef1709748c1e2604c9431714eb Mon Sep 17 00:00:00 2001 From: abikouo Date: Wed, 22 Mar 2023 17:35:01 +0100 Subject: [PATCH 10/20] Remove github workflow --- .github/workflows/local.yml | 36 ------------------------- tox.ini | 54 ------------------------------------- 2 files changed, 90 deletions(-) delete mode 100644 .github/workflows/local.yml delete mode 100644 tox.ini diff --git a/.github/workflows/local.yml b/.github/workflows/local.yml deleted file mode 100644 index 923d3a4e..00000000 --- a/.github/workflows/local.yml +++ /dev/null @@ -1,36 +0,0 @@ -name: Tests -concurrency: - group: ${{ github.head_ref }} - cancel-in-progress: true - -on: - pull_request: - types: - - opened - - reopened - - labeled - - unlabeled - - synchronize - branches: - - '*' - tags: - - '*' - -jobs: - linters: - runs-on: ubuntu-latest - steps: - - name: Checkout the repository - uses: actions/checkout@v3 - - - name: setup python - uses: actions/setup-python@v4 - with: - python-version: '3.9' - - - name: install tox - run: pip install -U tox - shell: bash - - - name: Run unit tests - run: tox -e linters -vv diff --git a/tox.ini b/tox.ini deleted file mode 100644 index 2f2c450e..00000000 --- a/tox.ini +++ /dev/null @@ -1,54 +0,0 @@ -[tox] -minversion = 1.4.2 -skipsdist = True - -[testenv:black] -deps = - black >= 22.0, < 23.0 - -commands = - black -v --check --diff scripts - -[testenv:flake8] -deps = - flake8 - -commands = - flake8 scripts - -[testenv:isort] -deps = - isort - -commands = - isort --check scripts - -[testenv:mypy] -deps = - mypy - -commands = - mypy scripts - -[testenv:black_format] -deps = - {[testenv:black]deps} - -commands = - black -v {posargs} - -[testenv:linters] - -deps = - - {[testenv:black]deps} - {[testenv:flake8]deps} - {[testenv:isort]deps} - {[testenv:mypy]deps} - -commands = - - {[testenv:black]commands} - {[testenv:flake8]commands} - {[testenv:isort]commands} - {[testenv:mypy]commands} From 48d03be7efecb6717e775aa13cf94f20071368a6 Mon Sep 17 00:00:00 2001 From: Bikouo Aubin <79859644+abikouo@users.noreply.github.com> Date: Wed, 22 Mar 2023 18:41:25 +0100 Subject: [PATCH 11/20] Update mypy.ini Co-authored-by: Kate Case --- mypy.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/mypy.ini b/mypy.ini index b7e91a29..7ad8a0fe 100644 --- a/mypy.ini +++ b/mypy.ini @@ -12,3 +12,4 @@ pretty = true show_column_numbers = true show_error_codes = true show_error_context = true +strict = true From 055ce4cdfe1556f67258bf8d191144b99f4fd7d2 Mon Sep 17 00:00:00 2001 From: Bikouo Aubin <79859644+abikouo@users.noreply.github.com> Date: Wed, 22 Mar 2023 18:41:33 +0100 Subject: [PATCH 12/20] Update .pre-commit-config.yaml Co-authored-by: Kate Case --- .pre-commit-config.yaml | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 98335e6a..bfc0339a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -26,3 +26,25 @@ repos: rev: 23.1.0 hooks: - id: black + + - repo: https://github.com/pycqa/flake8 + rev: 6.0.0 + hooks: + - id: flake8 + additional_dependencies: + - flake8-2020 >= 1.6.0 + - flake8-isort >= 4.1.1 + - darglint + - flake8-docstrings # uses pydocstyle + + - repo: https://github.com/pre-commit/mirrors-mypy + rev: v1.1.1 + hooks: + - id: mypy + additional_dependencies: + - types-PyYAML + + - repo: https://github.com/pycqa/pylint.git + rev: v2.17.0 + hooks: + - id: pylint From e194af2a4184d586a27414914d501b65365e62e4 Mon Sep 17 00:00:00 2001 From: abikouo Date: Thu, 23 Mar 2023 13:33:48 +0100 Subject: [PATCH 13/20] Fix linters issues --- .github/workflows/changelog.yml | 1 - .pylintrc | 116 ++++++++++++++++++++++++++++++++ scripts/validate_changelog.py | 80 +++++++++++----------- setup.cfg | 4 ++ 4 files changed, 162 insertions(+), 39 deletions(-) create mode 100644 .pylintrc create mode 100644 setup.cfg diff --git a/.github/workflows/changelog.yml b/.github/workflows/changelog.yml index 2d80ea14..1aba2fda 100644 --- a/.github/workflows/changelog.yml +++ b/.github/workflows/changelog.yml @@ -32,5 +32,4 @@ jobs: run: >- python /tmp/validate_changelog.py --base-ref ${{ github.event.pull_request.base.ref }} - --head-ref ${{ github.event.pull_request.head.sha }} working-directory: ${{ env.source_directory }} diff --git a/.pylintrc b/.pylintrc new file mode 100644 index 00000000..a2cefc77 --- /dev/null +++ b/.pylintrc @@ -0,0 +1,116 @@ +[MAIN] + +# Analyse import fallback blocks. This can be used to support both Python 2 and +# 3 compatible code, which means that the block might have code that exists +# only in one or another interpreter, leading to false positives when analysed. +analyse-fallback-blocks=no + +# Clear in-memory caches upon conclusion of linting. Useful if running pylint +# in a server-like mode. +clear-cache-post-run=no + +# Load and enable all available extensions. Use --list-extensions to see a list +# all available extensions. +#enable-all-extensions= + +# In error mode, messages with a category besides ERROR or FATAL are +# suppressed, and no reports are done by default. Error mode is compatible with +# disabling specific errors. +#errors-only= + +# Always return a 0 (non-error) status code, even if lint errors are found. +# This is primarily useful in continuous integration scripts. +#exit-zero= + +# A comma-separated list of package or module names from where C extensions may +# be loaded. Extensions are loading into the active Python interpreter and may +# run arbitrary code. +extension-pkg-allow-list= + +# A comma-separated list of package or module names from where C extensions may +# be loaded. Extensions are loading into the active Python interpreter and may +# run arbitrary code. (This is an alternative name to extension-pkg-allow-list +# for backward compatibility.) +extension-pkg-whitelist= + +# Return non-zero exit code if any of these messages/categories are detected, +# even if score is above --fail-under value. Syntax same as enable. Messages +# specified are enabled, while categories only check already-enabled messages. +fail-on= + +# Specify a score threshold under which the program will exit with error. +fail-under=10 + +# Interpret the stdin as a python script, whose filename needs to be passed as +# the module_or_package argument. +#from-stdin= + +# Files or directories to be skipped. They should be base names, not paths. +ignore=CVS + +# Add files or directories matching the regular expressions patterns to the +# ignore-list. The regex matches against paths and can be in Posix or Windows +# format. Because '\\' represents the directory delimiter on Windows systems, +# it can't be used as an escape character. +ignore-paths= + +# Files or directories matching the regular expression patterns are skipped. +# The regex matches against base names, not paths. The default value ignores +# Emacs file locks +ignore-patterns=^\.# + +# List of module names for which member attributes should not be checked +# (useful for modules/projects where namespaces are manipulated during runtime +# and thus existing member attributes cannot be deduced by static analysis). It +# supports qualified module names, as well as Unix pattern matching. +ignored-modules= + +# Python code to execute, usually for sys.path manipulation such as +# pygtk.require(). +#init-hook= + +# Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the +# number of processors available to use, and will cap the count on Windows to +# avoid hangs. +jobs=1 + +# Control the amount of potential inferred values when inferring a single +# object. This can help the performance when dealing with large functions or +# complex, nested conditions. +limit-inference-results=100 + +# List of plugins (as comma separated values of python module names) to load, +# usually to register additional checkers. +load-plugins= + +# Pickle collected data for later comparisons. +persistent=yes + +# Minimum Python version to use for version dependent checks. Will default to +# the version used to run pylint. +py-version=3.9 + +# Discover python modules and packages in the file system subtree. +recursive=no + +# Add paths to the list of the source roots. Supports globbing patterns. The +# source root is an absolute path or a path relative to the current working +# directory used to determine a package namespace for modules located under the +# source root. +source-roots= + +# When enabled, pylint would attempt to guess common misconfiguration and emit +# user-friendly hints instead of false-positive error messages. +suggestion-mode=yes + +# Allow loading of arbitrary C extensions. Extensions are imported into the +# active Python interpreter and may run arbitrary code. +unsafe-load-any-extension=no + +# In verbose mode, extra non-checker-related info will be displayed. +#verbose= + +[MESSAGES CONTROL] + +disable= + missing-docstring \ No newline at end of file diff --git a/scripts/validate_changelog.py b/scripts/validate_changelog.py index b9bacf35..d3621c4e 100644 --- a/scripts/validate_changelog.py +++ b/scripts/validate_changelog.py @@ -6,8 +6,12 @@ import subprocess import sys from collections import defaultdict +from typing import Any, Dict, Tuple, Union -import yaml +try: + import yaml +except ImportError: + pass FORMAT = "[%(asctime)s] - %(message)s" logging.basicConfig(format=FORMAT) @@ -15,11 +19,12 @@ logger.setLevel(logging.DEBUG) -def is_valid_change_log(ref): - return re.match(r"^changelogs/fragments/(.*)\.(yaml|yml)$", ref) +def is_valid_change_log(ref: str) -> bool: + match = re.match(r"^changelogs/fragments/(.*)\.(yaml|yml)$", ref) + return bool(match) -def is_module_or_plugin(ref): +def is_module_or_plugin(ref: str) -> bool: prefix_list = ( "plugins/modules", "plugins/action", @@ -42,7 +47,7 @@ def is_module_or_plugin(ref): return ref.startswith(prefix_list) -def is_documentation_file(ref): +def is_documentation_file(ref: str) -> bool: prefix_list = ( "docs/", "plugins/doc_fragments", @@ -50,20 +55,20 @@ def is_documentation_file(ref): return ref.startswith(prefix_list) -def is_added_module_or_plugin_or_documentation_changes(changes): +def should_skip_changelog(changes: Dict[Any, Any]) -> bool: # Validate Pull request add new modules and plugins - if any([is_module_or_plugin(x) for x in changes["A"]]): + if any(is_module_or_plugin(x) for x in changes["A"]): return True # Validate documentation changes only all_files = changes["A"] + changes["M"] + changes["D"] - if all([is_documentation_file(x) for x in all_files]): + if all(is_documentation_file(x) for x in all_files): return True return False -def validate_changelog(path): +def validate_changelog(path: str) -> bool: try: # https://github.com/ansible-community/antsibull-changelog/blob/main/docs/changelogs.rst#changelog-fragment-categories changes_type = ( @@ -78,8 +83,8 @@ def validate_changelog(path): "known_issues", "trivial", ) - with open(path, "rb") as f: - result = list(yaml.safe_load_all(f)) + with open(path, "rb") as file_desc: + result = list(yaml.safe_load_all(file_desc)) for section in result: for key in section.keys(): @@ -89,49 +94,49 @@ def validate_changelog(path): return False if not isinstance(section[key], list): logger.error( - "Changelog section {0} from file {1} must be a list," - " {2} found instead.".format( - key, - path, - type(section[key]), - ) + "Changelog section %s from file %s must be a list," + " '%s' found instead.", + key, + path, + type(section[key]), ) return False return True except (IOError, yaml.YAMLError) as exc: - msg = "yaml loading error for file {0} -> {1}".format(path, exc) + msg = f"yaml loading error for file {path} -> {exc}" logger.error(msg) return False -def run_command(cmd): - params = dict(stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) - proc = subprocess.Popen(cmd, **params) - out, err = proc.communicate() - return proc.returncode, out, err +def run_command(cmd: str) -> Tuple[Union[int, Any], str, str]: + with subprocess.Popen( + cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True + ) as proc: + out, err = proc.communicate() + return proc.returncode, out.decode("utf-8"), err.decode("utf-8") -def list_files(head_ref, base_ref): - cmd = "git diff origin/{0} {1} --name-status".format(base_ref, head_ref) - logger.info("Executing '{0}'".format(cmd)) - rc, stdout, stderr = run_command(cmd) - if rc != 0: +def list_files(ref: str) -> Dict[Any, Any]: + command = "git diff origin/" + ref + " --name-status" + logger.info("Executing -> %s", command) + ret_code, stdout, stderr = run_command(command) + if ret_code != 0: raise ValueError(stderr) changes = defaultdict(list) - for file in stdout.decode("utf-8").split("\n"): - v = file.split("\t") - if len(v) == 2: - changes[v[0]].append(v[1]) + for file in stdout.split("\n"): + file_attr = file.split("\t") + if len(file_attr) == 2: + changes[file_attr[0]].append(file_attr[1]) return changes -def main(head_ref, base_ref): - changes = list_files(head_ref, base_ref) +def main(ref: str) -> None: + changes = list_files(ref) if changes: changelog = [x for x in changes["A"] if is_valid_change_log(x)] if not changelog: - if not is_added_module_or_plugin_or_documentation_changes(changes): + if not should_skip_changelog(changes): logger.error( "Missing changelog fragment. This is not required" " only if PR adds new modules and plugins or contain" @@ -151,8 +156,7 @@ def main(head_ref, base_ref): parser = argparse.ArgumentParser( description="Validate changelog file from new commit" ) - parser.add_argument("--base-ref", required=True) - parser.add_argument("--head-ref", required=True) + parser.add_argument("--ref", required=True, help="Pull request base ref") args = parser.parse_args() - main(args.head_ref, args.base_ref) + main(args.ref) diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 00000000..10de8514 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,4 @@ +[flake8] +max-line-length = 160 +ignore = D103,D100 +exclude = From fea62012bb28b75c7c605df80418b5ad297a7ba8 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 23 Mar 2023 15:06:19 +0000 Subject: [PATCH 14/20] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .pylintrc | 2 +- setup.cfg | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.pylintrc b/.pylintrc index a2cefc77..7d481cf0 100644 --- a/.pylintrc +++ b/.pylintrc @@ -113,4 +113,4 @@ unsafe-load-any-extension=no [MESSAGES CONTROL] disable= - missing-docstring \ No newline at end of file + missing-docstring diff --git a/setup.cfg b/setup.cfg index 10de8514..4d8185e0 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,4 +1,4 @@ [flake8] max-line-length = 160 ignore = D103,D100 -exclude = +exclude = From 46467379dfed112dfe532816e1dd996ad2cb4d58 Mon Sep 17 00:00:00 2001 From: abikouo Date: Thu, 23 Mar 2023 16:32:07 +0100 Subject: [PATCH 15/20] fix script argument --- .github/workflows/changelog.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/changelog.yml b/.github/workflows/changelog.yml index 1aba2fda..6f7cc02f 100644 --- a/.github/workflows/changelog.yml +++ b/.github/workflows/changelog.yml @@ -31,5 +31,5 @@ jobs: - name: Ensure a valid changelog entry exists run: >- python /tmp/validate_changelog.py - --base-ref ${{ github.event.pull_request.base.ref }} + --ref ${{ github.event.pull_request.base.ref }} working-directory: ${{ env.source_directory }} From a92c81542b52b7e72f58deddcf7f3027198ffbf4 Mon Sep 17 00:00:00 2001 From: abikouo Date: Thu, 23 Mar 2023 16:39:10 +0100 Subject: [PATCH 16/20] add some debug information --- scripts/validate_changelog.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/scripts/validate_changelog.py b/scripts/validate_changelog.py index d3621c4e..3489e7bb 100644 --- a/scripts/validate_changelog.py +++ b/scripts/validate_changelog.py @@ -19,7 +19,7 @@ logger.setLevel(logging.DEBUG) -def is_valid_change_log(ref: str) -> bool: +def is_changelog_file(ref: str) -> bool: match = re.match(r"^changelogs/fragments/(.*)\.(yaml|yml)$", ref) return bool(match) @@ -68,7 +68,7 @@ def should_skip_changelog(changes: Dict[Any, Any]) -> bool: return False -def validate_changelog(path: str) -> bool: +def is_valid_changelog_format(path: str) -> bool: try: # https://github.com/ansible-community/antsibull-changelog/blob/main/docs/changelogs.rst#changelog-fragment-categories changes_type = ( @@ -128,13 +128,15 @@ def list_files(ref: str) -> Dict[Any, Any]: file_attr = file.split("\t") if len(file_attr) == 2: changes[file_attr[0]].append(file_attr[1]) + logger.info("changes -> %s", changes) return changes def main(ref: str) -> None: changes = list_files(ref) if changes: - changelog = [x for x in changes["A"] if is_valid_change_log(x)] + changelog = [x for x in changes["A"] if is_changelog_file(x)] + logger.info("changelog files -> %s", changelog) if not changelog: if not should_skip_changelog(changes): logger.error( @@ -147,8 +149,16 @@ def main(ref: str) -> None: "Changelog not required as PR adds new modules and/or" " plugins or contain only documentation changes." ) - elif any(not validate_changelog(f) for f in changelog): - sys.exit(1) + else: + invalid_changelog_files = [ + x for x in changelog if not is_valid_changelog_format(x) + ] + if invalid_changelog_files: + logger.error( + "The following changelog files are not valid -> %s", + invalid_changelog_files, + ) + sys.exit(1) sys.exit(0) From 0c6565b33972d2cb9ee30e7be060127c3a70c2a6 Mon Sep 17 00:00:00 2001 From: Bikouo Aubin <79859644+abikouo@users.noreply.github.com> Date: Fri, 24 Mar 2023 08:58:46 +0100 Subject: [PATCH 17/20] Update .pre-commit-config.yaml Co-authored-by: Kate Case --- .pre-commit-config.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index bfc0339a..c6c5d255 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -48,3 +48,5 @@ repos: rev: v2.17.0 hooks: - id: pylint + additional_dependencies: + - PyYAML From 35142cad56ca0b2cfe9d8d9257dced691e3f4267 Mon Sep 17 00:00:00 2001 From: Bikouo Aubin <79859644+abikouo@users.noreply.github.com> Date: Fri, 24 Mar 2023 08:58:54 +0100 Subject: [PATCH 18/20] Update scripts/validate_changelog.py Co-authored-by: Kate Case --- scripts/validate_changelog.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/validate_changelog.py b/scripts/validate_changelog.py index 3489e7bb..961ca1d0 100644 --- a/scripts/validate_changelog.py +++ b/scripts/validate_changelog.py @@ -110,10 +110,10 @@ def is_valid_changelog_format(path: str) -> bool: def run_command(cmd: str) -> Tuple[Union[int, Any], str, str]: with subprocess.Popen( - cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True + cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True, encoding="utf-8", ) as proc: out, err = proc.communicate() - return proc.returncode, out.decode("utf-8"), err.decode("utf-8") + return proc.returncode, out, err def list_files(ref: str) -> Dict[Any, Any]: From e0e01861e9268513f8195a807f949d36c3a1f8a1 Mon Sep 17 00:00:00 2001 From: Bikouo Aubin <79859644+abikouo@users.noreply.github.com> Date: Fri, 24 Mar 2023 08:59:01 +0100 Subject: [PATCH 19/20] Update scripts/validate_changelog.py Co-authored-by: Kate Case --- scripts/validate_changelog.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/scripts/validate_changelog.py b/scripts/validate_changelog.py index 961ca1d0..18e5c17c 100644 --- a/scripts/validate_changelog.py +++ b/scripts/validate_changelog.py @@ -8,10 +8,7 @@ from collections import defaultdict from typing import Any, Dict, Tuple, Union -try: - import yaml -except ImportError: - pass +import yaml FORMAT = "[%(asctime)s] - %(message)s" logging.basicConfig(format=FORMAT) From 37255e90e3a3ceadd64db2e58f977bb81249aa29 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 24 Mar 2023 07:59:14 +0000 Subject: [PATCH 20/20] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- scripts/validate_changelog.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/scripts/validate_changelog.py b/scripts/validate_changelog.py index 18e5c17c..d9c8498f 100644 --- a/scripts/validate_changelog.py +++ b/scripts/validate_changelog.py @@ -107,7 +107,11 @@ def is_valid_changelog_format(path: str) -> bool: def run_command(cmd: str) -> Tuple[Union[int, Any], str, str]: with subprocess.Popen( - cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True, encoding="utf-8", + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + shell=True, + encoding="utf-8", ) as proc: out, err = proc.communicate() return proc.returncode, out, err