diff --git a/.circleci/README.md b/.circleci/README.md deleted file mode 100644 index c3757f5..0000000 --- a/.circleci/README.md +++ /dev/null @@ -1,33 +0,0 @@ -# Orb Development Pipeline - -This configuration file uses [orb-tools orb]() version 10 to automatically _pack_, _test_, and _publish_ CircleCI orbs using this project structure. View the comments within the config file for a full break down - -## Overview: - -**Imported Orbs** - -Both orb-tools and a development version of your orb will be imported into the config. On the first run, a `dev:alpha` development tag _must_ exist on your orb, but will be handled automatically from there on. - -**Jobs** - -In the _jobs_ key, you will define _integration tests_. These jobs will utilize the functionality of your orb at run-time and attempt to validate their usage with live examples. Integration tests can be an excellent way of determining issues with parameters and run-time execution. - -### Workflows - -There are two workflows which automate the pack, test, and publishing process. - -**test-pack** - -This is the first of the two workflows run. This workflow is responsible for any testing or prepping prior to integration tests. This is where linting occurs, shellchecking, BATS tests, or anything else that can be be tested without the need for further credentials. - -This Workflow will be placed on _hold_ prior to publishing a new development version of the orb (based on this commit), as this step requires access to specific publishing credentials. - -This allows users to fork the orb repository and begin the pipeline, while the code-owners review that the code is safe to test in an environment where publishing keys will be present. - -Once approved, the development version of the orb will publish and the _trigger-integration-tests-workflow_ job will run, kicking off the next workflow - -**integration-test_deploy** - -The second and final workflow is manually triggered by the _trigger-integration-tests-workflow_ job. In this run, the development version of the orb that was just published will be imported, and the integration tests will run. - -When running on the `master` branch (after merging to `master`), the workflow will additionally publish your new production orb. \ No newline at end of file diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index 434d080..0000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,34 +0,0 @@ -version: 2.1 - -setup: true -orbs: - orb-tools: circleci/orb-tools@11.5 - shellcheck: circleci/shellcheck@3.1 - -filters: &filters - tags: - only: /.+/ - -workflows: - lint-pack: - jobs: - - orb-tools/lint: - filters: *filters - - orb-tools/pack: - filters: *filters - - orb-tools/review: - filters: *filters - - shellcheck/check: - filters: *filters - - orb-tools/publish: - orb-name: circleci/path-filtering - vcs-type: << pipeline.project.type >> - requires: - [orb-tools/lint, orb-tools/review, orb-tools/pack, shellcheck/check] - context: orb-publisher - filters: *filters - - orb-tools/continue: - pipeline-number: << pipeline.number >> - vcs-type: << pipeline.project.type >> - requires: [orb-tools/publish] - filters: *filters diff --git a/.circleci/test-deploy.yml b/.circleci/test-deploy.yml deleted file mode 100644 index eaa147e..0000000 --- a/.circleci/test-deploy.yml +++ /dev/null @@ -1,53 +0,0 @@ -version: 2.1 -orbs: - path-filtering: circleci/path-filtering@dev:<> - orb-tools: circleci/orb-tools@11.5 - -filters: &filters - tags: - only: /.+/ - -jobs: - set-parameters-test: - docker: - - image: cimg/base:current - steps: - - checkout - - path-filtering/set-parameters: - mapping: | - src/.* test-changes true - src/examples/.* string-example "value" - - path-filtering/set-parameters: - config-path: ".circleci/test-deploy.yml" - mapping: | - src/commands/.* test-commands true .circleci/config.yml - src/examples/.* test-examples true .circleci/test-deploy.yml - src/jobs/.* test-jobs true .circleci/config.yml - src/tests/.* test-tests true .circleci/test-deploy.yml - - - path-filtering/generate-config: - config-list-path: /tmp/filtered-config-list - generated-config-path: "/tmp/generated-config.yml" - - - -workflows: - test-deploy: - jobs: - - set-parameters-test: - filters: *filters - - orb-tools/pack: - filters: *filters - - orb-tools/publish: - orb-name: circleci/path-filtering - vcs-type: << pipeline.project.type >> - pub-type: production - requires: - - orb-tools/pack - - set-parameters-test - context: orb-publisher - filters: - branches: - ignore: /.*/ - tags: - only: /^v[0-9]+\.[0-9]+\.[0-9]+$/ diff --git a/.gitignore b/.gitignore index 19eafbb..13665b4 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,4 @@ # orb.yml is "packed" from source, and not published directly from the repository. -orb.yml \ No newline at end of file +orb.yml + +.idea diff --git a/README.md b/README.md index 871be64..5faedea 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,5 @@ # Path Filtering Orb -[![CircleCI Build Status](https://circleci.com/gh/CircleCI-Public/path-filtering-orb.svg?style=shield "CircleCI Build Status")](https://circleci.com/gh/CircleCI-Public/path-filtering-orb) [![CircleCI Orb Version](https://badges.circleci.com/orbs/circleci/path-filtering.svg)](https://circleci.com/developer/orbs/orb/circleci/path-filtering) [![GitHub license](https://img.shields.io/badge/license-MIT-blue.svg)](https://raw.githubusercontent.com/circleci-public/path-filtering-orb/master/LICENSE) [![CircleCI Community](https://img.shields.io/badge/community-CircleCI%20Discuss-343434.svg)](https://discuss.circleci.com/c/ecosystem/orbs) - -A starter template for orb projects. Build, test, and publish orbs automatically on CircleCI with [Orb-Tools](https://circleci.com/orbs/registry/orb/circleci/orb-tools). +[![CircleCI Orb Version](https://badges.circleci.com/orbs/affinity/path-filtering-orb.svg)](https://circleci.com/developer/orbs/orb/affinity/path-filtering-orb) [![GitHub license](https://img.shields.io/badge/license-MIT-blue.svg)](https://raw.githubusercontent.com/affinity/path-filtering-orb/master/LICENSE) Additional READMEs are available in each directory. @@ -13,15 +11,34 @@ Additional READMEs are available in each directory. [Setup Workflows Documentation](https://github.com/CircleCI-Public/api-preview-docs/blob/master/docs/setup-workflows.md#concepts) - Doc explaining a special type of workflow used in dynamic config -[CircleCI Orb Registry Page](https://circleci.com/developer/orbs/orb/circleci/path-filtering) - The official registry page of this orb for all versions, executors, commands, and jobs described. +[CircleCI Orb Registry Page](https://circleci.com/developer/orbs/orb/affinity/path-filtering-orb) - The official registry page of this orb for all versions, executors, commands, and jobs described. [CircleCI Orb Docs](https://circleci.com/docs/2.0/orb-intro/#section=configuration) - Docs for using, creating, and publishing CircleCI Orbs. ### How to Contribute -We welcome [issues](https://github.com/CircleCI-Public/path-filtering-orb/issues) to and [pull requests](https://github.com/CircleCI-Public/path-filtering-orb/pulls) against this repository! +We welcome [issues](https://github.com/affinity/path-filtering-orb/issues) to and [pull requests](https://github.com/affinity/path-filtering-orb/pulls) against this repository! + +### Development + +To develop this orb, you can use the `circleci orb pack src > /tmp/orb.yml` command to generate and validate the orb.yml file. +Publish a dev version with: +```shell +circleci orb publish /tmp/orb.yml affinity/path-filtering@dev:first +``` + +Test it out in your CircleCI workflow by adding the following to your `.circleci/config.yml`: +```yaml +orbs: + path-filtering: affinity/path-filtering@dev:first +``` + +Publish it for use with: +```shell + circleci orb publish /tmp/orb.yml affinity/path-filtering@ +``` -### How to Publish An Update +### How to Publish A New Release 1. Merge pull requests with desired changes to the main branch. - For the best experience, squash-and-merge and use [Conventional Commit Messages](https://conventionalcommits.org/). 2. Find the current version of the orb. @@ -34,4 +51,3 @@ We welcome [issues](https://github.com/CircleCI-Public/path-filtering-orb/issues - If you have used _[Conventional Commit Messages](https://conventionalcommits.org/)_ it will be easy to determine what types of changes were made, allowing you to ensure the correct version tag is being published. 5. Now ensure the version tag selected is semantically accurate based on the changes included. 6. Click _"Publish Release"_. - - This will push a new tag and trigger your publishing pipeline on CircleCI. diff --git a/src/@orb.yml b/src/@orb.yml index 9603fc6..b91026e 100755 --- a/src/@orb.yml +++ b/src/@orb.yml @@ -5,9 +5,11 @@ description: > This can be useful in a monorepo setup where one may want to trigger different workflows based on which module(s) in the repo has changed. This orb does not support server at this time. + This is a fork of the original path-filtering orb by CircleCI with shallow clone support. display: home_url: "https://circleci.com/docs/2.0/dynamic-config" - source_url: "https://github.com/CircleCI-Public/path-filtering-orb" + source_url: "https://github.com/affinity/path-filtering-orb" orbs: continuation: circleci/continuation@0.2.0 + git-shallow-clone: guitarrapc/git-shallow-clone@2.8.0 diff --git a/src/commands/set-parameters.yml b/src/commands/set-parameters.yml index 9481431..d41af4f 100644 --- a/src/commands/set-parameters.yml +++ b/src/commands/set-parameters.yml @@ -28,6 +28,10 @@ parameters: description: > The location of the config to continue the pipeline with, please note that this parameter will be ignored if the user passes the config file per mapping in the mapping parameter + enable_merge_queue_support: + type: boolean + description: "Set to true if you use Github merge queue. Requires a context with a CIRCLECI_ACCESS_TOKEN set for API access" + default: false steps: - run: @@ -38,4 +42,5 @@ steps: OUTPUT_PATH: << parameters.output-path >> CONFIG_PATH: << parameters.config-path >> CREATE_PIPELINE_SCRIPT: <> + MERGE_QUEUE_SUPPORT: << parameters.enable_merge_queue_support >> command: <> diff --git a/src/examples/path_filtering.yml b/src/examples/path_filtering.yml index 9cd1c9c..f8aa162 100755 --- a/src/examples/path_filtering.yml +++ b/src/examples/path_filtering.yml @@ -5,12 +5,19 @@ usage: version: 2.1 setup: true orbs: - path-filtering: circleci/path-filtering@0.1.7 + path-filtering: affinity/path-filtering@0.9.3 workflows: generate-config: jobs: - path-filtering/filter: base-revision: main + # use shallow clones to speed up the process + enable_shallow_checkout: true + checkout_shallow_depth: 200 + checkout_shallow_fetch_depth: 200 + # This repo uses Github's merge queue; this will enable support for it + # to find the proper merge base when multiple commits are merged in one push + enable_merge_queue_support: true config-path: .circleci/continue_config.yml mapping: | src/.* build-code true diff --git a/src/executors/default.yml b/src/executors/default.yml index f064d55..6dbb865 100755 --- a/src/executors/default.yml +++ b/src/executors/default.yml @@ -7,7 +7,7 @@ docker: parameters: tag: type: string - default: "3.8" + default: "3.12" description: > Pick a specific cimg/python image variant: https://hub.docker.com/r/cimg/python/tags diff --git a/src/jobs/filter.yml b/src/jobs/filter.yml index 9b22054..aa8c237 100644 --- a/src/jobs/filter.yml +++ b/src/jobs/filter.yml @@ -55,7 +55,7 @@ parameters: default: "small" tag: type: string - default: "3.8" + default: "3.12" description: > Pick a specific cimg/python image variant: https://hub.docker.com/r/cimg/python/tags @@ -63,11 +63,38 @@ parameters: description: Enables jobs to go through a set of well-defined IP address ranges. type: boolean default: false + enable_shallow_checkout: + type: boolean + description: "Flag value for whether shallow checkout is used or not" + default: false + checkout_shallow_depth: + type: integer + description: "shallow checkout depth" + default: 100 + checkout_shallow_fetch_depth: + type: integer + description: "shallow checkout fetch depth" + default: 100 + enable_merge_queue_support: + type: boolean + description: "Set to true if you use Github merge queue. Requires a context with a CIRCLECI_ACCESS_TOKEN set for API access" + default: false circleci_ip_ranges: << parameters.circleci_ip_ranges >> steps: - - checkout + - when: + condition: + equal: [ false, << parameters.enable_shallow_checkout >> ] + steps: + - checkout + - when: + condition: + equal: [ true, << parameters.enable_shallow_checkout >> ] + steps: + - git-shallow-clone/checkout: + depth: << parameters.checkout_shallow_depth >> + fetch_depth: << parameters.checkout_shallow_fetch_depth >> - when: condition: not: @@ -80,6 +107,7 @@ steps: mapping: << parameters.mapping >> output-path: << parameters.output-path >> config-path: << parameters.config-path >> + enable_merge_queue_support: << parameters.enable_merge_queue_support >> - generate-config: config-list-path: /tmp/filtered-config-list generated-config-path: "/tmp/generated-config.yml" diff --git a/src/scripts/create-parameters.py b/src/scripts/create-parameters.py index 7f404a1..ff45f5c 100644 --- a/src/scripts/create-parameters.py +++ b/src/scripts/create-parameters.py @@ -2,6 +2,9 @@ import os import re import subprocess +import sys +import urllib.request +import urllib.error from functools import partial def checkout(revision): @@ -16,6 +19,24 @@ def checkout(revision): check=True ) + +def get_github_org_from_url(git_url): + """ + Extract GitHub organization name from a git URL + + :param git_url: Git URL in the format git@github.com:org/repo.git + :return: Organization name + """ + # Check if the URL is in the expected format + if git_url.startswith('git@github.com:'): + # Split by colon and get the second part + parts = git_url.split(':')[1] + # Split by slash and get the first part + org_name = parts.split('/')[0] + return org_name + + return None + def merge_base(base, head): return subprocess.run( ['git', 'merge-base', base, head], @@ -23,12 +44,90 @@ def merge_base(base, head): capture_output=True ).stdout.decode('utf-8').strip() -def parent_commit(): - return subprocess.run( - ['git', 'rev-parse', 'HEAD~1'], - check=True, - capture_output=True - ).stdout.decode('utf-8').strip() +def extract_vcs_revision(json_data): + """ + Parses JSON data and extracts the 'vcs_revision' from the first element + if the data is a list, or directly if it's a dictionary. + + Args: + json_data (str): A string containing the JSON response. + + Returns: + str: The vcs_revision value, or None if not found or on error. + """ + try: + data = json.loads(json_data) + + # Check if the response is a list (like from the CircleCI builds endpoint) + if isinstance(data, list): + if data: # Check if the list is not empty + first_item = data[0] + if isinstance(first_item, dict) and 'vcs_revision' in first_item: + return first_item['vcs_revision'] + else: + print("Error: First item in the list is not a dictionary or does not contain 'vcs_revision'.", + file=sys.stderr) + return None + else: + print("Error: JSON response is an empty list.", file=sys.stderr) + return None + # Check if the response is a dictionary itself + elif isinstance(data, dict): + if 'vcs_revision' in data: + return data['vcs_revision'] + else: + print("Error: JSON dictionary does not contain 'vcs_revision'.", file=sys.stderr) + return None + else: + print("Error: Unexpected JSON structure.", file=sys.stderr) + return None + + except json.JSONDecodeError as e: + print(f"Error decoding JSON: {e}", file=sys.stderr) + return None + except Exception as e: + print(f"An unexpected error occurred: {e}", file=sys.stderr) + return None + +def parent_commit(merge_queue_support): + branch = os.environ.get('CIRCLE_BRANCH') + repo_name = os.environ.get('CIRCLE_PROJECT_REPONAME') + org_name = get_github_org_from_url(os.environ.get('CIRCLE_REPOSITORY_URL')) + + # If using GitHub's merge queue, several commits can be merged into the main branch + # at once, but only one webhook is sent for the entire merge queue. In this case, we need to + # use the CircleCI API to get the last commit that was built for the branch to ensure + # we are using the correct commit as the base for our diff. + if merge_queue_support and (branch == 'master' or branch == 'main'): + print(f"Merge Queue support is enabled, using CircleCI API to get the previously built commit on {branch}") + # make a request to CircleCI API to get the latest build for the branch, we'll use that as our base + url = f'https://circleci.com/api/v1.1/project/github/{org_name}/{repo_name}/tree/{branch}?filter=successful&limit=1' + + req = urllib.request.Request( + url, + headers={'Accept': 'application/json', 'Circle-Token': os.environ.get('CIRCLECI_ACCESS_TOKEN')} + ) + + try: + with urllib.request.urlopen(req) as response: + data = response.read().decode(response.headers.get_content_charset("utf-8")) + except urllib.error.HTTPError as e: + raise Exception(f"HTTP Error: {e.code} - {e.reason}") + except urllib.error.URLError as e: + raise Exception(f"URL Error: {e.reason}") + + previous_sha = extract_vcs_revision(data) + if previous_sha is None: + raise Exception("Could not find a previous build to use as base.") + + return previous_sha + + else: + return subprocess.run( + ['git', 'rev-parse', 'HEAD~1'], + check=True, + capture_output=True + ).stdout.decode('utf-8').strip() def changed_files(base, head): return subprocess.run( @@ -111,18 +210,20 @@ def is_mapping_line(line: str) -> bool: is_comment_line = (line.strip().startswith("#")) return not (is_comment_line or is_empty_line) -def create_parameters(output_path, config_path, head, base, mapping): +def create_parameters(output_path, config_path, head, base, mapping, merge_queue_support): checkout(base) # Checkout base revision to make sure it is available for comparison checkout(head) # return to head commit base = merge_base(base, head) + print(f"Merge Queue support: {merge_queue_support}") + if head == base: try: # If building on the same branch as BASE_REVISION, we will get the # current commit as merge base. In that case try to go back to the # first parent, i.e. the last state of this branch before the # merge, and use that as the base. - base = parent_commit() + base = parent_commit(merge_queue_support) except: # This can fail if this is the first commit of the repo, so that # HEAD~1 actually doesn't resolve. In this case we can compare @@ -153,5 +254,6 @@ def create_parameters(output_path, config_path, head, base, mapping): os.environ.get('CONFIG_PATH'), os.environ.get('CIRCLE_SHA1'), os.environ.get('BASE_REVISION'), - os.environ.get('MAPPING') + os.environ.get('MAPPING'), + os.environ.get('MERGE_QUEUE_SUPPORT', 'False').lower() in ('true', '1', 't') )