Merge pull request elementary-data#496 from elementary-data/e2e-tests

E2e tests

Author: elongl

.github/workflows/run-unittest.yml

@@ -104,13 +104,12 @@ jobs:
         run: pip install dbt-core==${{ inputs.dbt-version }} dbt-${{ inputs.warehouse-type }}==${{ inputs.dbt-version }}
 
       - name: Install Elementary
-        run: pip install "./elementary[${{ inputs.warehouse-type }}]"
+        run: |
+          pip install -r ./elementary/dev-requirements.txt
+          pip install "./elementary[${{ inputs.warehouse-type }}]"
 
-      - name: Upload build artifact
-        uses: actions/upload-artifact@v3
-        with:
-          name: build
-          path: elementary/build
+      - name: Run Python package unit tests
+        run: pytest -vv -k unit
 
       - name: Install dbt package
         run: |
@@ -121,7 +120,7 @@ jobs:
           rm -rf "$DBT_PKGS_PATH/elementary"
           ln -vs "$GITHUB_WORKSPACE/dbt-data-reliability" "$DBT_PKGS_PATH/elementary"
 
-      - name: Run E2E tests
+      - name: Run dbt package integration tests
         if: github.event_name != 'workflow_dispatch' || inputs.should-run-tests
         run: |
           dbt deps --project-dir ./dbt-data-reliability/integration_tests
@@ -171,3 +170,6 @@ jobs:
         with:
           name: edr.log
           path: edr.log
+
+      - name: Run Python package integration tests
+        run: pytest -vv -k integration

.github/workflows/test-warehouse.yml

@@ -0,0 +1,3 @@
+[pytest]
+addopts = --ignore=elementary/monitor/dbt_project 
+

pytest.ini

@@ -0,0 +1,5 @@
+## Integration Tests
+
+Those tests should only be run after running
+the [integration tests of the dbt package](https://github.com/elementary-data/dbt-data-reliability/tree/master/integration_tests)
+as the fixtures are based on their output data.

tests/integration/README.md

@@ -0,0 +1,79 @@
+import json
+from pathlib import Path
+from typing import Dict
+
+import pytest
+
+_REPORT_DATA_FILENAME = "elementary_output.json"
+_REPORT_DATA_FIXTURE = Path(__file__).parent / "fixtures" / _REPORT_DATA_FILENAME
+_REPORT_DATA_PATH = Path(_REPORT_DATA_FILENAME)
+
+TotalsEntry = Dict[str, int]
+Totals = Dict[str, TotalsEntry]
+
+report_data = json.loads(_REPORT_DATA_PATH.read_text())
+
+
+def test_report_keys(report_data_fixture):
+    assert report_data.keys() == report_data_fixture.keys()
+
+
+def test_totals(report_data_fixture):
+    for key in report_data_fixture:
+        if key.endswith("_totals"):
+            assert_totals(report_data[key], report_data_fixture[key])
+
+
+def test_sidebar(report_data_fixture):
+    assert (
+        "model.elementary_integration_tests.error_model"
+        in report_data["sidebars"]["dbt"]["elementary_integration_tests"]["models"][
+            "__files__"
+        ]
+    )
+    assert (
+        "model.elementary_integration_tests.nested"
+        in report_data["sidebars"]["dbt"]["elementary_integration_tests"]["models"][
+            "nested"
+        ]["models"]["tree"]["__files__"]
+    )
+    assert (
+        "source.elementary_integration_tests.training.any_type_column_anomalies_training"
+        in report_data["sidebars"]["dbt"]["elementary_integration_tests"]["sources"][
+            "__files__"
+        ]
+    )
+    assert (
+        "model.elementary_integration_tests.any_type_column_anomalies"
+        in report_data["sidebars"]["owners"]["@edr"]
+    )
+    assert (
+        "model.elementary_integration_tests.any_type_column_anomalies"
+        not in report_data["sidebars"]["owners"]["No owners"]
+    )
+    assert (
+        "model.elementary_integration_tests.string_column_anomalies"
+        in report_data["sidebars"]["tags"]["marketing"]
+    )
+    assert (
+        "model.elementary_integration_tests.string_column_anomalies"
+        not in report_data["sidebars"]["tags"]["No tags"]
+    )
+
+
+def assert_totals(data_totals: Totals, fixture_totals: Totals):
+    assert data_totals.keys() == fixture_totals.keys()
+    for total_key in fixture_totals:
+        assert_totals_entry(data_totals[total_key], fixture_totals[total_key])
+
+
+def assert_totals_entry(
+    data_total_entries: TotalsEntry, fixture_total_entries: TotalsEntry
+):
+    for key in fixture_total_entries:
+        assert data_total_entries[key] * fixture_total_entries[key] >= 0
+
+
+@pytest.fixture
+def report_data_fixture():
+    return json.loads(_REPORT_DATA_FIXTURE.read_text())