diff --git a/.github/scripts/next_version.sh b/.github/scripts/next_version.sh new file mode 100755 index 000000000..8e4eb57c9 --- /dev/null +++ b/.github/scripts/next_version.sh @@ -0,0 +1,16 @@ +#!/bin/bash +set -e + +GIT_COMMIT=`git rev-parse --short=8 HEAD` + +if [[ "${GITHUB_REF_TYPE-""}" == "tag" ]]; then + PREFIX="stable" +elif [[ "${BRANCH-""}" == "master" ]]; then + PREFIX="stable" +else + PREFIX="develop" +fi + +#create version +NEW_TAG="$PREFIX-$(date '+%Y.%-m.%-d-%H%M')-$GIT_COMMIT" +echo $NEW_TAG diff --git a/.github/workflows/pre-merge.yaml b/.github/workflows/pre-merge.yaml index 307c5db2c..f64bca683 100644 --- a/.github/workflows/pre-merge.yaml +++ b/.github/workflows/pre-merge.yaml @@ -1,105 +1,15 @@ name: Pre-merge CI on: + workflow_call: pull_request: branches: - develop - push: - branches: - - develop - - main concurrency: cancel-in-progress: true group: ${{ github.workflow }}-${{ github.ref }} jobs: - PythonLint: - timeout-minutes: 5 - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - uses: actions/setup-python@v5 - with: - python-version: '3.7' - - shell: bash - run: |- - python -m pip install -r requirements_bundles.txt flake8 --user - - shell: bash - run: |- - export PATH=$PATH:/builder/home/.local/bin - ./bin/flake8_tests.sh - PythonUnitTests: - timeout-minutes: 60 - - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - with: - version: v0.12.1 - - name: Build docker compose - uses: docker/bake-action@v4 - with: - files: docker-compose.yml, docker-compose.ci.yml - load: true - - name: "Prepare test environment" - shell: bash - run: |- - docker compose -f docker-compose.ci.yml up -d postgres redis - - name: "Wait for postgres to start" - shell: bash - run: |- - for i in 1 2 3 4 5; do docker compose -f docker-compose.ci.yml run --rm postgres psql -h postgres -U postgres -c "SELECT 1 FROM pg_database WHERE datname = 'tests'" && break || sleep 2; done - - name: "Create tests database" - shell: bash - run: |- - docker compose -f docker-compose.ci.yml run --rm postgres psql -h postgres -U postgres -c "create database tests;" - - name: "List Enabled Query Runners" - shell: bash - run: |- - docker compose -f docker-compose.ci.yml run --rm server manage ds list_types - - name: "Execute unit tests" - shell: bash - run: |- - docker compose -f docker-compose.ci.yml run --user 0 --name tests server tests --junitxml=junit.xml --cov-report xml --cov=redash --cov-config .coveragerc tests/ - - name: "Extract test results" - shell: bash - if: always() - run: | - docker cp tests:/app/coverage.xml ./coverage.xml - docker cp tests:/app/junit.xml ./results.xml - ls -l - - name: "Publish unit test report" - uses: mikepenz/action-junit-report@v4 - if: always() - with: - fail_on_failure: true - report_paths: results.xml - require_tests: true - include_passed: true - summary: true - job_summary: true - check_name: "Unit Tests Report" - - name: Code Coverage Report - uses: irongut/CodeCoverageSummary@v1.3.0 - with: - filename: coverage.xml - badge: true - fail_below_min: true - format: markdown - hide_branch_rate: false - hide_complexity: true - indicators: true - output: both - thresholds: '60 80' - - name: Add Coverage PR Comment - uses: marocchino/sticky-pull-request-comment@v2 - if: github.event_name == 'pull_request' - with: - recreate: true - path: code-coverage-results.md + unit_tests: + uses: ./.github/workflows/test-unit.yaml + secrets: inherit diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml new file mode 100644 index 000000000..98fb3e1e9 --- /dev/null +++ b/.github/workflows/release.yaml @@ -0,0 +1,54 @@ +name: Release docker Image +on: + push: + branches: + - develop + - main + +concurrency: + cancel-in-progress: true + group: ${{ github.workflow }}-${{ github.ref }} + +jobs: + unit_tests: + uses: ./.github/workflows/test-unit.yaml + secrets: inherit + release_image: + runs-on: ubuntu-latest + needs: + - unit_tests + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - id: get-version + run: | + #!/bin/bash + next_version=$(./.github/scripts/next_version.sh) + echo "Next version: $next_version" + echo "version=$next_version" >> $GITHUB_OUTPUT + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + with: + driver-opts: image=moby/buildkit:buildx-stable-1 + - name: Docker login + uses: docker/login-action@v3 + with: + registry: ${{ secrets.GCR_URL }} + username: ${{ secrets.GCR_USERNAME }} + password: ${{ secrets.GCR_PASSWORD }} + - name: Build and push datareporter + uses: docker/build-push-action@v6 + with: + push: true + tags: europe-west1-docker.pkg.dev/datareporter/datareporter:${{ steps.get-version.outputs.version }} + cache-from: type=gha + cache-to: type=gha,mode=max + - name: Build and push plywood + uses: docker/build-push-action@v6 + with: + push: true + context: plywood/server + tags: europe-west1-docker.pkg.dev/datareporter/plywood:${{ steps.get-version.outputs.version }} + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/.github/workflows/test-unit.yaml b/.github/workflows/test-unit.yaml new file mode 100644 index 000000000..6bb270828 --- /dev/null +++ b/.github/workflows/test-unit.yaml @@ -0,0 +1,96 @@ +name: "Unit tests" +on: + workflow_call: + + +jobs: + PythonLint: + timeout-minutes: 5 + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - uses: actions/setup-python@v5 + with: + python-version: '3.7' + - shell: bash + run: |- + python -m pip install -r requirements_bundles.txt flake8 --user + - shell: bash + run: |- + export PATH=$PATH:/builder/home/.local/bin + ./bin/flake8_tests.sh + PythonUnitTests: + timeout-minutes: 60 + + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + with: + version: v0.12.1 + - name: Build docker compose + uses: docker/bake-action@v4 + with: + files: docker-compose.yml, docker-compose.ci.yml + load: true + - name: "Prepare test environment" + shell: bash + run: |- + docker compose -f docker-compose.ci.yml up -d postgres redis + - name: "Wait for postgres to start" + shell: bash + run: |- + for i in 1 2 3 4 5; do docker compose -f docker-compose.ci.yml run --rm postgres psql -h postgres -U postgres -c "SELECT 1 FROM pg_database WHERE datname = 'tests'" && break || sleep 2; done + - name: "Create tests database" + shell: bash + run: |- + docker compose -f docker-compose.ci.yml run --rm postgres psql -h postgres -U postgres -c "create database tests;" + - name: "List Enabled Query Runners" + shell: bash + run: |- + docker compose -f docker-compose.ci.yml run --rm server manage ds list_types + - name: "Execute unit tests" + shell: bash + run: |- + docker compose -f docker-compose.ci.yml run --user 0 --name tests server tests --junitxml=junit.xml --cov-report xml --cov=redash --cov-config .coveragerc tests/ + - name: "Extract test results" + shell: bash + if: always() + run: | + docker cp tests:/app/coverage.xml ./coverage.xml + docker cp tests:/app/junit.xml ./results.xml + ls -l + - name: "Publish unit test report" + uses: mikepenz/action-junit-report@v4 + if: always() + with: + fail_on_failure: true + report_paths: results.xml + require_tests: true + include_passed: true + summary: true + job_summary: true + check_name: "Unit Tests Report" + - name: Code Coverage Report + uses: irongut/CodeCoverageSummary@v1.3.0 + with: + filename: coverage.xml + badge: true + fail_below_min: true + format: markdown + hide_branch_rate: false + hide_complexity: true + indicators: true + output: both + thresholds: '60 80' + - name: Add Coverage PR Comment + uses: marocchino/sticky-pull-request-comment@v2 + if: github.event_name == 'pull_request' + with: + recreate: true + path: code-coverage-results.md