From a3c6a317d600052bcfbb3bd0367b51bc0e506ba0 Mon Sep 17 00:00:00 2001 From: Colin Date: Mon, 9 Jan 2023 10:44:52 -0800 Subject: [PATCH 01/11] fix whitespace issue with incremental sql --- .../incremental_strategy/merge.sql | 2 +- .../test_incremental_materialization.py | 44 +++++++++++++++++++ 2 files changed, 45 insertions(+), 1 deletion(-) create mode 100644 tests/functional/test_incremental_materialization.py diff --git a/dbt/include/bigquery/macros/materializations/incremental_strategy/merge.sql b/dbt/include/bigquery/macros/materializations/incremental_strategy/merge.sql index 5efb079b6..57c88dbc8 100644 --- a/dbt/include/bigquery/macros/materializations/incremental_strategy/merge.sql +++ b/dbt/include/bigquery/macros/materializations/incremental_strategy/merge.sql @@ -16,7 +16,7 @@ {{ wrap_with_time_ingestion_partitioning_sql(build_partition_time_exp(partition_by), sql, True) }} {%- else -%} {{sql}} - {%- endif -%} + {%- endif %} ) {%- endif -%} {%- endset -%} diff --git a/tests/functional/test_incremental_materialization.py b/tests/functional/test_incremental_materialization.py new file mode 100644 index 000000000..6932363b3 --- /dev/null +++ b/tests/functional/test_incremental_materialization.py @@ -0,0 +1,44 @@ +import pytest +import os +from dbt.tests.util import ( + run_dbt +) + +# This is a short term hack, we need to go back +# and make adapter implementations of: +# https://github.com/dbt-labs/dbt-core/pull/6330 + +_INCREMENTAL_MODEL = """ +{{ + config( + materialized="incremental", + ) +}} + +{% if not is_incremental() %} + + select 10 as id, cast('2020-01-01 01:00:00' as datetime) as date_hour union all + select 30 as id, cast('2020-01-01 02:00:00' as datetime) as date_hour + +{% else %} + + select 20 as id, cast('2020-01-01 01:00:00' as datetime) as date_hour union all + select 40 as id, cast('2020-01-01 02:00:00' as datetime) as date_hour + +{% endif %} +-- Test Comment To Prevent Reccurence of https://github.com/dbt-labs/dbt-core/issues/6485 +""" + +class BaseIncrementalModelConfig: + @pytest.fixture(scope="class") + def models(self): + return { + "test_incremental.sql": _INCREMENTAL_MODEL + } + +class TestIncrementalModel(BaseIncrementalModelConfig): + def test_incremental_model_succeeds(self, project): + results = run_dbt(["run"]) + assert len(results) == 1 + results = run_dbt(["run"]) + assert len(results) == 1 \ No newline at end of file From 546c501e289d5b1f6cb4e3d318e3b705e87a74b0 Mon Sep 17 00:00:00 2001 From: Colin Date: Mon, 9 Jan 2023 10:53:38 -0800 Subject: [PATCH 02/11] add changie --- .changes/unreleased/Fixes-20230109-105325.yaml | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 .changes/unreleased/Fixes-20230109-105325.yaml diff --git a/.changes/unreleased/Fixes-20230109-105325.yaml b/.changes/unreleased/Fixes-20230109-105325.yaml new file mode 100644 index 000000000..2a16883fc --- /dev/null +++ b/.changes/unreleased/Fixes-20230109-105325.yaml @@ -0,0 +1,7 @@ +kind: Fixes +body: stop eliminating trailing whitespace in incremental merge sql +time: 2023-01-09T10:53:25.837837-08:00 +custom: + Author: colin-rogers-dbt + Issue: "457" + PR: "458" From 66e3e77196a769c151a0d12b4fc54097e842fa3a Mon Sep 17 00:00:00 2001 From: Colin Date: Mon, 9 Jan 2023 13:03:56 -0800 Subject: [PATCH 03/11] add setuptools install --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 0c7d51fa8..71c06b6f7 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -186,7 +186,7 @@ jobs: - name: Install python dependencies run: | python -m pip install --user --upgrade pip - python -m pip install --upgrade wheel + python -m pip install --upgrade wheel setuptools python -m pip --version - uses: actions/download-artifact@v2 with: From 3e6f941906945798ed1ea24b8ac152940bcfc3ca Mon Sep 17 00:00:00 2001 From: Colin Date: Mon, 9 Jan 2023 13:25:32 -0800 Subject: [PATCH 04/11] add twine and check-wheel-contents --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 71c06b6f7..6b41ee03e 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -186,7 +186,7 @@ jobs: - name: Install python dependencies run: | python -m pip install --user --upgrade pip - python -m pip install --upgrade wheel setuptools + python -m pip install --upgrade wheel setuptools twine check-wheel-contents python -m pip --version - uses: actions/download-artifact@v2 with: From 73988f7e42531ca6844d01866119f4189186e831 Mon Sep 17 00:00:00 2001 From: Colin Date: Mon, 9 Jan 2023 13:35:01 -0800 Subject: [PATCH 05/11] try installing pkgconfig-lite --- .github/workflows/main.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 6b41ee03e..2d9824708 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -188,6 +188,10 @@ jobs: python -m pip install --user --upgrade pip python -m pip install --upgrade wheel setuptools twine check-wheel-contents python -m pip --version + - name: Windows dependencies + if: ${{matrix.os}} == 'windows-latest' + run: + "choco install pkgconfiglite" - uses: actions/download-artifact@v2 with: name: dist From 5b63ac56907d394056c9f1e6f210e1cda5ed22fc Mon Sep 17 00:00:00 2001 From: Colin Date: Mon, 9 Jan 2023 13:42:36 -0800 Subject: [PATCH 06/11] add --allow-empty-checksums --- .github/workflows/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 2d9824708..78c097d08 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -190,8 +190,8 @@ jobs: python -m pip --version - name: Windows dependencies if: ${{matrix.os}} == 'windows-latest' - run: - "choco install pkgconfiglite" + run: | + choco install pkgconfiglite --allow-empty-checksums - uses: actions/download-artifact@v2 with: name: dist From a4a3ce770c415a83915e8693bde5525fecbec3b5 Mon Sep 17 00:00:00 2001 From: Colin Date: Mon, 9 Jan 2023 13:59:54 -0800 Subject: [PATCH 07/11] check pkg-config installs --- .github/workflows/main.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 78c097d08..901207ec6 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -189,9 +189,10 @@ jobs: python -m pip install --upgrade wheel setuptools twine check-wheel-contents python -m pip --version - name: Windows dependencies - if: ${{matrix.os}} == 'windows-latest' + if: matrix.os == windows-latest run: | choco install pkgconfiglite --allow-empty-checksums + pkg-config --cflags --libs icu-i18n - uses: actions/download-artifact@v2 with: name: dist From d51fe1413c8524ef4cefb87aa2c67e02f95a5cbf Mon Sep 17 00:00:00 2001 From: Colin Date: Mon, 9 Jan 2023 14:13:28 -0800 Subject: [PATCH 08/11] install mac os pyicu dependencies --- .github/workflows/main.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 901207ec6..4d4d013a0 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -193,6 +193,12 @@ jobs: run: | choco install pkgconfiglite --allow-empty-checksums pkg-config --cflags --libs icu-i18n + - name: Mac dependencies + if: matrix.os == macos-latest + run: | + brew install pkg-config icu4c + export PATH="/usr/local/opt/icu4c/bin:/usr/local/opt/icu4c/sbin:$PATH" + export PKG_CONFIG_PATH="$PKG_CONFIG_PATH:/usr/local/opt/icu4c/lib/pkgconfig" - uses: actions/download-artifact@v2 with: name: dist From db8c4f34a4aaf09f29129198a5b62c419f4436ef Mon Sep 17 00:00:00 2001 From: Colin Date: Mon, 9 Jan 2023 14:35:20 -0800 Subject: [PATCH 09/11] fix syntax issue --- .github/workflows/main.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 4d4d013a0..95de8e3d5 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -182,19 +182,18 @@ jobs: uses: actions/setup-python@v4.3.0 with: python-version: ${{ matrix.python-version }} - - name: Install python dependencies run: | python -m pip install --user --upgrade pip python -m pip install --upgrade wheel setuptools twine check-wheel-contents python -m pip --version - name: Windows dependencies - if: matrix.os == windows-latest + if: ${{matrix.os == 'windows-latest'}} run: | choco install pkgconfiglite --allow-empty-checksums pkg-config --cflags --libs icu-i18n - name: Mac dependencies - if: matrix.os == macos-latest + if: ${{matrix.os == 'macos-latest'}} run: | brew install pkg-config icu4c export PATH="/usr/local/opt/icu4c/bin:/usr/local/opt/icu4c/sbin:$PATH" From d27cf24e3bfadc0f1041bea36b6d7fd890e975a8 Mon Sep 17 00:00:00 2001 From: Colin Date: Mon, 9 Jan 2023 15:06:51 -0800 Subject: [PATCH 10/11] force agate 1.6.3 --- .github/workflows/main.yml | 24 +++++++++++++----------- setup.py | 1 + 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 95de8e3d5..ca34a6456 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -187,17 +187,19 @@ jobs: python -m pip install --user --upgrade pip python -m pip install --upgrade wheel setuptools twine check-wheel-contents python -m pip --version - - name: Windows dependencies - if: ${{matrix.os == 'windows-latest'}} - run: | - choco install pkgconfiglite --allow-empty-checksums - pkg-config --cflags --libs icu-i18n - - name: Mac dependencies - if: ${{matrix.os == 'macos-latest'}} - run: | - brew install pkg-config icu4c - export PATH="/usr/local/opt/icu4c/bin:/usr/local/opt/icu4c/sbin:$PATH" - export PKG_CONFIG_PATH="$PKG_CONFIG_PATH:/usr/local/opt/icu4c/lib/pkgconfig" + # - name: Windows dependencies + # if: ${{matrix.os == 'windows-latest'}} + # run: | + # choco install pkgconfiglite --allow-empty-checksums + # pkg-config --cflags --libs icu-i18n + # - name: Mac dependencies + # if: ${{matrix.os == 'macos-latest'}} + # run: | + # brew install pkg-config icu4c + # export PATH="/usr/local/opt/icu4c/bin:/usr/local/opt/icu4c/sbin:$PATH" + # export PKG_CONFIG_PATH="$PKG_CONFIG_PATH:/usr/local/opt/icu4c/lib/pkgconfig" + # echo 'export PATH="/usr/local/opt/icu4c/bin:$PATH"' >> ~/.zshrc + # echo 'export PATH="/usr/local/opt/icu4c/sbin:$PATH"' >> ~/.zshrc - uses: actions/download-artifact@v2 with: name: dist diff --git a/setup.py b/setup.py index 07f501ce6..fbe81c7e8 100644 --- a/setup.py +++ b/setup.py @@ -81,6 +81,7 @@ def _dbt_core_version(plugin_version: str) -> str: "googleapis-common-protos~=1.6", "google-cloud-storage~=2.4", "google-cloud-dataproc~=5.0", + "agate>=1.6.3,<1.7" ], zip_safe=False, classifiers=[ From 52de0f0b5350b755303968bf5abc577df8847962 Mon Sep 17 00:00:00 2001 From: Colin Date: Mon, 9 Jan 2023 15:13:54 -0800 Subject: [PATCH 11/11] add trailing comma --- .github/workflows/main.yml | 13 ------------- setup.py | 2 +- 2 files changed, 1 insertion(+), 14 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index ca34a6456..140557beb 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -187,19 +187,6 @@ jobs: python -m pip install --user --upgrade pip python -m pip install --upgrade wheel setuptools twine check-wheel-contents python -m pip --version - # - name: Windows dependencies - # if: ${{matrix.os == 'windows-latest'}} - # run: | - # choco install pkgconfiglite --allow-empty-checksums - # pkg-config --cflags --libs icu-i18n - # - name: Mac dependencies - # if: ${{matrix.os == 'macos-latest'}} - # run: | - # brew install pkg-config icu4c - # export PATH="/usr/local/opt/icu4c/bin:/usr/local/opt/icu4c/sbin:$PATH" - # export PKG_CONFIG_PATH="$PKG_CONFIG_PATH:/usr/local/opt/icu4c/lib/pkgconfig" - # echo 'export PATH="/usr/local/opt/icu4c/bin:$PATH"' >> ~/.zshrc - # echo 'export PATH="/usr/local/opt/icu4c/sbin:$PATH"' >> ~/.zshrc - uses: actions/download-artifact@v2 with: name: dist diff --git a/setup.py b/setup.py index fbe81c7e8..52defb47d 100644 --- a/setup.py +++ b/setup.py @@ -81,7 +81,7 @@ def _dbt_core_version(plugin_version: str) -> str: "googleapis-common-protos~=1.6", "google-cloud-storage~=2.4", "google-cloud-dataproc~=5.0", - "agate>=1.6.3,<1.7" + "agate>=1.6.3,<1.7", ], zip_safe=False, classifiers=[