Merge pull request elementary-data#513 from elementary-data/master

Temporarily added workflow_dispatch.

Author: elongl

.github/workflows/release.yml

@@ -1,5 +1,6 @@
 name: Release package
 on:
+  workflow_dispatch:
   push:
     tags:
       - v[0-9].[0-9]+.[0-9]+
@@ -40,7 +41,6 @@ jobs:
 
   build-and-push-docker-image:
     runs-on: ubuntu-latest
-    needs: publish-to-pypi
     permissions:
       contents: read
       packages: write
@@ -72,14 +72,10 @@ jobs:
 
   merge-to-prod:
     runs-on: ubuntu-latest
-    needs: build-and-push-docker-image
-
     steps:
-      - name: Checkout Elementary
-        uses: actions/checkout@v3
-
-      - name: Merge 'master' to 'prod'
-        run: |
-          git checkout prod
-          git merge master
-          git push
+      - uses: actions/checkout@v3
+      - name: PR master to prod
+        uses: repo-sync/pull-request@v2
+        with:
+          source_branch: master
+          destination_branch: prod

.github/workflows/run-unittest.yml

@@ -52,7 +52,6 @@ jobs:
     with:
       warehouse-type: ${{ matrix.warehouse-type }}
       dbt-data-reliability-ref: ${{ needs.get-latest-release-tags.outputs.dbt }}
-      dbt-version: '1.1.0'
     secrets: inherit
 
   recommendation:

.github/workflows/test-release.yml

@@ -60,11 +60,14 @@ on:
 
 env:
   BRANCH_NAME: ${{ github.head_ref || github.ref_name }}
-  TESTS_DIR: dbt-data-reliability/integration_tests
+  DBT_PKG_INTEG_TESTS_DIR: ${{ github.workspace }}/dbt-data-reliability/integration_tests
 
 jobs:
   test:
     runs-on: ubuntu-20.04
+    defaults:
+      run:
+        working-directory: elementary
     concurrency:
       group: tests-${{ inputs.warehouse-type }}-${{ github.head_ref || github.ref_name }}
       cancel-in-progress: true
@@ -104,13 +107,12 @@ jobs:
         run: pip install dbt-core==${{ inputs.dbt-version }} dbt-${{ inputs.warehouse-type }}==${{ inputs.dbt-version }}
 
       - name: Install Elementary
-        run: pip install "./elementary[${{ inputs.warehouse-type }}]"
+        run: |
+          pip install -r dev-requirements.txt
+          pip install ".[${{ inputs.warehouse-type }}]"
 
-      - name: Upload build artifact
-        uses: actions/upload-artifact@v3
-        with:
-          name: build
-          path: elementary/build
+      - name: Run Python package unit tests
+        run: pytest -vv -k unit
 
       - name: Install dbt package
         run: |
@@ -121,48 +123,60 @@ jobs:
           rm -rf "$DBT_PKGS_PATH/elementary"
           ln -vs "$GITHUB_WORKSPACE/dbt-data-reliability" "$DBT_PKGS_PATH/elementary"
 
-      - name: Run E2E tests
+      - name: Run dbt package integration tests
         if: github.event_name != 'workflow_dispatch' || inputs.should-run-tests
+        working-directory: ${{ env.DBT_PKG_INTEG_TESTS_DIR }}
         run: |
-          dbt deps --project-dir ./dbt-data-reliability/integration_tests
-          python ./dbt-data-reliability/integration_tests/run_e2e_tests.py -t "${{ inputs.warehouse-type }}" --clear-tests "${{ inputs.clear-tests }}"
+          dbt deps
+          python run_e2e_tests.py -t "${{ inputs.warehouse-type }}" --clear-tests "${{ inputs.clear-tests }}"
 
       - name: Run help
         run: edr --help
 
       - name: Run monitor
-        working-directory: ${{ env.TESTS_DIR }}
         env:
           SLACK_WEBHOOK: ${{ secrets.CI_SLACK_WEBHOOK }}
-        run: edr monitor -t "${{ inputs.warehouse-type }}" --slack-webhook "$SLACK_WEBHOOK"
+        run: >
+          edr monitor 
+          --project-dir "${{ env.DBT_PKG_INTEG_TESTS_DIR }}"
+          -t "${{ inputs.warehouse-type }}"
+          --slack-webhook "$SLACK_WEBHOOK"
 
       - name: Run report
-        working-directory: ${{ env.TESTS_DIR }}
-        run: edr monitor report -t "${{ inputs.warehouse-type }}" --file-path "report_${{ inputs.warehouse-type }}_${{ env.BRANCH_NAME }}.html"
+        run: >
+          edr monitor report
+          --project-dir "${{ env.DBT_PKG_INTEG_TESTS_DIR }}"
+          -t "${{ inputs.warehouse-type }}"
 
       - name: Upload report artifact
         uses: actions/upload-artifact@v3
         with:
           name: report_${{ inputs.warehouse-type }}_${{ env.BRANCH_NAME }}.html
-          path: report_${{ inputs.warehouse-type }}_${{ env.BRANCH_NAME }}.html
+          path: elementary_report.html
 
       - name: Write GCS keyfile
         env:
           GCS_KEYFILE: ${{ secrets.GCS_KEYFILE }}
         run: echo "$GCS_KEYFILE" | base64 -d > /tmp/gcs_keyfile.json
 
       - name: Run send report
-        working-directory: ${{ env.TESTS_DIR }}
         env:
           SLACK_TOKEN: ${{ secrets.CI_SLACK_TOKEN }}
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-
         run: >
-          edr monitor send-report -t "${{ inputs.warehouse-type }}" --slack-file-name "report_${{ inputs.warehouse-type }}_${{ env.BRANCH_NAME }}.html"
-          --slack-token "$SLACK_TOKEN" --slack-channel-name data-ops --bucket-file-path "ci_reports/report_${{ inputs.warehouse-type }}_${{ env.BRANCH_NAME }}.html"
-          --aws-access-key-id "$AWS_ACCESS_KEY_ID" --aws-secret-access-key "$AWS_SECRET_ACCESS_KEY" --s3-bucket-name elementary-ci-artifacts
-          --google-service-account-path /tmp/gcs_keyfile.json --gcs-bucket-name elementary_ci_artifacts
+          edr monitor send-report 
+          --project-dir "${{ env.DBT_PKG_INTEG_TESTS_DIR }}"
+          -t "${{ inputs.warehouse-type }}"
+          --slack-file-name "report_${{ inputs.warehouse-type }}_${{ env.BRANCH_NAME }}.html"
+          --slack-token "$SLACK_TOKEN"
+          --slack-channel-name data-ops
+          --bucket-file-path "ci_reports/report_${{ inputs.warehouse-type }}_${{ env.BRANCH_NAME }}.html"
+          --aws-access-key-id "$AWS_ACCESS_KEY_ID"
+          --aws-secret-access-key "$AWS_SECRET_ACCESS_KEY"
+          --s3-bucket-name elementary-ci-artifacts
+          --google-service-account-path /tmp/gcs_keyfile.json
+          --gcs-bucket-name elementary_ci_artifacts
           --update-bucket-website true
 
       - name: Upload edr log
@@ -171,3 +185,7 @@ jobs:
         with:
           name: edr.log
           path: edr.log
+
+      - name: Run Python package integration tests
+        if: github.event_name != 'workflow_dispatch' || inputs.should-run-tests
+        run: pytest -vv -k integration

.github/workflows/test-warehouse.yml

@@ -1,5 +1,7 @@
 FROM python:3.10.7
 
 WORKDIR /app
-RUN pip install "elementary-data[snowflake, bigquery, redshift, databricks]"
+COPY . .
+RUN pip install ".[snowflake, bigquery, redshift, databricks]"
 ENTRYPOINT ["edr"]
+