gathering all new jobs

.github/actions/ansible_test_integration/action.yml

@@ -0,0 +1,104 @@
+name: ansible test integration
+description: configure cloud environment and run ansible-test integration tests
+inputs:
+  python_version:
+    description: Python version to use to run integration tests
+    required: true
+  ansible_version:
+    description: ansible-core version to use to run integration tests
+    required: true
+  source_path:
+    description: "Path to the collection source"
+    required: true
+  dependency_path:
+    description: "Path to the dependency collection"
+    required: false
+  create_k8s_cluster:
+    description: Wether integration tests required kubernetes cluster to execute
+    required: false
+  ansible_test_targets:
+    description: Integration tests targets
+    required: false
+  ansible_test_environment:
+    description: list of environment variables to set when running ansible-test
+    required: false
+
+runs:
+  using: composite
+  steps:
+
+    - name: Set up Python ${{ inputs.python_version }}
+      uses: actions/setup-python@v4
+      with:
+        python-version: ${{ inputs.python_version }}
+
+    - name: Install wheel now for faster builds
+      run: python3 -m pip install wheel --upgrade
+      shell: bash
+
+    - name: Install ansible-core (${{ inputs.ansible_version }})
+      run: python3 -m pip install https://github.com/ansible/ansible/archive/${{ inputs.ansible_version }}.tar.gz --disable-pip-version-check
+      shell: bash
+
+    # install source collection
+    - name: Read collection metadata from galaxy.yml
+      id: identify
+      uses: ansible-network/github_actions/.github/actions/identify_collection@main
+      with:
+        source_path: ${{ inputs.source_path }}
+
+    - name: Build and install the collection
+      uses: ansible-network/github_actions/.github/actions/build_install_collection@main
+      with:
+        install_python_dependencies: true
+        source_path: ${{ inputs.source_path }}
+        collection_path: ${{ steps.identify.outputs.collection_path }}
+        tar_file: ${{ steps.identify.outputs.tar_file }}
+
+    # install collection dependencies
+    - name: Read collection metadata from ${{ inputs.dependency_path }}
+      if: inputs.dependency_path != ''
+      id: identify-dependency
+      uses: ansible-network/github_actions/.github/actions/identify_collection@main
+      with:
+        source_path: ${{ inputs.dependency_path }}
+
+    - name: Build and install ${{ inputs.dependency_path }}
+      if: inputs.dependency_path != ''
+      uses: ansible-network/github_actions/.github/actions/build_install_collection@main
+      with:
+        install_python_dependencies: true
+        source_path: ${{ inputs.dependency_path }}
+        collection_path: ${{ steps.identify-dependency.outputs.collection_path }}
+        tar_file: ${{ steps.identify-dependency.outputs.tar_file }}
+
+    - name: Disable selinux with selinux_please_lie_to_me
+      run: |
+        python3 -m pip uninstall -y selinux
+        python3 -m pip install selinux_please_lie_to_me
+      shell: bash
+
+    - name: Create k8s Kind Cluster
+      uses: helm/[email protected]
+      if: inputs.create_k8s_cluster == 'true'
+
+    - name: Set environment variables
+      run: echo "${{ inputs.ansible_test_environment }}" >> $GITHUB_ENV
+      shell: bash
+      if: inputs.ansible_test_environment != ''
+
+    - name: Run integration tests
+      run: >-
+        ansible-test integration
+        --diff
+        --no-temp-workdir
+        --color
+        --skip-tags False
+        --retry-on-error
+        --continue-on-error
+        --python ${{ inputs.python_version }} 
+        -v
+        --coverage
+        ${{ inputs.ansible_test_targets }}
+      shell: bash
+      working-directory: ${{ steps.identify.outputs.collection_path }}

.github/actions/checkout_repository/action.yml

@@ -0,0 +1,53 @@
+name: 'Checkout Repository'
+description: |
+  checkout repository and override commit based on keyword 'depends-on' from pull request message
+inputs:
+  repository:
+    description: |
+      Repository to checkout, repository is defined by organisation and repository name
+      e.g: ansible-collections/kubernetes.core
+    required: true
+  ref:
+    description: |
+      The default branch, tag or SHA to checkout.
+    required: true
+  destination_directory:
+    description: |
+      Path where repositories will be checkout
+    required: true
+  pull_request_body:
+    description: 'Pull request body used to override version to checkout'
+    required: false
+    default: ${{ github.event.pull_request.body }}
+runs:
+  using: composite  
+  steps:
+    - name: Set up Python '3.9'
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.9'
+
+    - name: install PyGithub
+      run: |
+        pip install -U PyGithub
+      shell: bash
+
+    - id: dependency-resolution
+      shell: bash
+      run: |
+        python ${{ github.action_path }}/resolve_dependency.py
+      env:
+        RESOLVE_REF_PR_BODY: ${{ inputs.pull_request_body }}
+        RESOLVE_REF_REPOSITORY: ${{ inputs.repository }}
+
+    - name: display merge commit sha from dependency resolution
+      run: echo "[checkout ${{ inputs.repository }}] merge commit sha from dependency resolution => '${{ steps.dependency-resolution.outputs.merge_commit_sha }}'"
+      shell: bash
+
+    - name: checkout repository
+      uses: actions/checkout@v3
+      with:
+        repository: ${{ inputs.repository }}
+        path: ${{ inputs.destination_directory }}
+        ref: ${{ steps.dependency-resolution.outputs.merge_commit_sha || inputs.ref }}
+        fetch-depth: "0"

.github/actions/checkout_repository/resolve_dependency.py

@@ -0,0 +1,63 @@
+#!/usr/bin/python
+
+import re
+import os
+import sys
+from github import Github
+import logging
+
+
+FORMAT = '[%(asctime)s] - %(message)s'
+logging.basicConfig(format=FORMAT)
+logger = logging.getLogger('resolve_dependency')
+logger.setLevel(logging.DEBUG)
+
+"""
+    While trying to checkout a repository, this script will check if there is an override into the
+    Pull request body.
+    Example: 
+        assuming we are going to checkout 'ansible-collections/community.aws' on 'main' branch,
+        if the pull request body contains 'Depends-On: https://github.com/ansible-collections/community.aws/pull/1542'
+        we will checkout 'ansible-collections/community.aws' on 'merge_commit_sha' instead
+"""
+
+def get_pr_merge_commit_sha(repository, pr_number):
+    access_token = os.environ.get("GITHUB_TOKEN")
+    gh = Github(access_token)
+    repo = gh.get_repo(repository)
+
+    pr = repo.get_pull(pr_number)
+    if not pr.mergeable:
+        # raise an error when the pull request is not mergeable
+        sys.tracebacklimit = -1
+        raise ValueError("Pull request %d from %s is not mergeable" % (pr_number, repository))
+
+    return pr.merge_commit_sha
+
+
+def resolve_ref(pr_body, repository):
+    pr_regx = re.compile(rf"^Depends-On:[ ]*https://github.com/{repository}/pull/(\d+)\s*$", re.MULTILINE | re.IGNORECASE)
+    # Search for expression starting with depends-on not case-sensitive
+    m = pr_regx.findall(pr_body)
+    return int(m[0]) if m else None
+
+
+def main():
+
+    pr_body = os.environ.get("RESOLVE_REF_PR_BODY")
+    repository = os.environ.get("RESOLVE_REF_REPOSITORY")
+
+    pr_number = resolve_ref(pr_body, repository)
+    if pr_number is None:
+        return
+    logger.info("Override checkout with pr number: %d" % pr_number)
+
+    # get pull request merge commit sha
+    merge_commit_sha = get_pr_merge_commit_sha(repository, pr_number)
+    logger.info("merge commit sha for pull request %d => '%s'" % (pr_number, merge_commit_sha))
+    with open(os.environ.get("GITHUB_OUTPUT"), "a") as fw:
+        fw.write("merge_commit_sha={0}\n".format(merge_commit_sha))
+
+if __name__ == "__main__":
+
+    main()

.github/actions/cloud_integration_test_splitter/action.yml

@@ -1,14 +1,13 @@
-name: Retrieve collection information
-description: Extract collection infromation from the galaxy.yml file
+name: Cloud integration test splitter
+description: Evaluate which targets need to be tested.
 
 inputs:
-  collection_path:
-    description: Path to the collections to test
+  collections_to_test:
+    description: |
+      Path to the collections to test.
+      Provide as a comma-separated list of collection path and base ref to test against.
+      e.g: 'repo_path_1:main,repo_path_2:stable-2'
     required: true
-  base_ref:
-    description: The base branch to test against
-    required: false
-    default: 'main'
   total_jobs:
     description: The total number of jobs to share targets on
     required: false
@@ -26,25 +25,20 @@ runs:
       with:
         python-version: '3.9'
 
-    - name: Download script
-      run: curl -o ${{ github.workspace }}/list_changed_targets.py https://raw.githubusercontent.com/ansible-network/github_actions/cloud_splitter/.github/scripts/list_changed_targets.py
-      shell: bash
-
     - name: Install python required libraries
       run: pip install -U pyyaml
       shell: bash
 
     - name: Set variable to set test all targets
-      run: echo "ANSIBLE_TEST_ALL_THE_TARGETS=--test-all-the-targets" >> "$GITHUB_ENV"
+      run: echo "ANSIBLE_TEST_ALL_THE_TARGETS=true" >> "$GITHUB_ENV"
       shell: bash
       if: ${{ (contains(github.event.pull_request.labels.*.name, 'test-all-the-targets')) }}
 
-    - name: List changes for the pull request
+    - name: Evaluate targets to test
       id: splitter
       run: >-
-        python ${{ github.workspace }}/list_changed_targets.py
-        --collection-path ${{ inputs.collection_path }}
-        --total-jobs ${{ inputs.total_jobs }}
-        $ANSIBLE_TEST_ALL_THE_TARGETS
-        --base-ref ${{ inputs.base_ref }} >> $GITHUB_OUTPUT
+        python ${{ github.action_path }}/list_changed_targets.py
+      env:
+        COLLECTIONS_TO_TEST: '${{ inputs.collections_to_test }}'
+        TOTAL_JOBS: '${{ inputs.total_jobs }}'
       shell: bash

.github/scripts/list_changed_targets.py → .github/actions/cloud_integration_test_splitter/list_changed_targets.py

@@ -404,21 +404,39 @@ def _findslot(t):
     return chunks
 
 
+def parse_inputs():
+
+    test_all_the_targets = bool(os.environ.get("ANSIBLE_TEST_ALL_THE_TARGETS") == "true")
+    jobs = os.environ.get("TOTAL_JOBS")
+    total_jobs = int(jobs)
+
+    def _parse_collection(element):
+        info = element.split(":")
+        if len(info) != 2:
+            raise ValueError("The following '{}' is not a valid format for collection definition.".format(element))
+        path, ref = info[1]
+        if not PosixPath(path).exists():
+            raise ValueError("The following path '{}' does not exit.".format(path))
+        return path, ref
+
+    collections = list(map(_parse_collection, os.environ.get("COLLECTIONS_TO_TEST", "").split(",")))
+    return collections, total_jobs, test_all_the_targets
+
+
 if __name__ == "__main__":
-    args = parse_args(sys.argv[1:])
 
-    collections_path = [PosixPath(i.strip()) for i in args.collection_path.split(",") if i.strip()]
-    collections = [Collection(p) for p in collections_path]
+    collections_to_test, total_jobs, test_all_the_targets = parse_inputs()
+    collections = [Collection(p) for p, _ in collections_to_test]
     collections_names = [c.collection_name() for c in collections]
 
     changes = {}
-    if args.test_all_the_targets:
+    if test_all_the_targets:
         changes = {}
         for c in collections:
             c.cover_all()
             changes[c.collection_name()] = c.test_plan
     else:
-        for whc in [WhatHaveChanged(i, args.base_ref) for i in collections_path]:
+        for whc in [WhatHaveChanged(path, ref) for path, ref in collections_to_test]:
             changes[whc.collection_name()] = {
                 "modules": [],
                 "inventory": [],
@@ -462,6 +480,6 @@ def _findslot(t):
         changes = {x: unique_list(changes[x]["targets"]) for x in changes}
         trace_content(f"changes: %s" % json.dumps(changes, indent=2))
 
-    egs = ElGrandeSeparator(collections, args.total_jobs)
+    egs = ElGrandeSeparator(collections, total_jobs)
     output = egs.output()
     print("test_targets=%s" % json.dumps(output))