From b5c8ad17db3666c305ef6ec869d08cc2f22533ed Mon Sep 17 00:00:00 2001 From: Michelle Ark Date: Tue, 30 May 2023 10:25:08 -0400 Subject: [PATCH 1/4] pass optional sql_header to empty subquery sql rendering --- .../global_project/macros/adapters/columns.sql | 13 ++++++++----- .../models/table/columns_spec_ddl.sql | 2 +- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/core/dbt/include/global_project/macros/adapters/columns.sql b/core/dbt/include/global_project/macros/adapters/columns.sql index 8605ab21d02..0d7e5532b9c 100644 --- a/core/dbt/include/global_project/macros/adapters/columns.sql +++ b/core/dbt/include/global_project/macros/adapters/columns.sql @@ -17,15 +17,18 @@ {% endmacro %} -{% macro get_empty_subquery_sql(select_sql) -%} - {{ return(adapter.dispatch('get_empty_subquery_sql', 'dbt')(select_sql)) }} +{% macro get_empty_subquery_sql(select_sql, select_sql_header=none) -%} + {{ return(adapter.dispatch('get_empty_subquery_sql', 'dbt')(select_sql, select_sql_header)) }} {% endmacro %} {# Builds a query that results in the same schema as the given select_sql statement, without necessitating a data scan. Useful for running a query in a 'pre-flight' context, such as model contract enforcement (assert_columns_equivalent macro). #} -{% macro default__get_empty_subquery_sql(select_sql) %} +{% macro default__get_empty_subquery_sql(select_sql, select_sql_header=none) %} + {%- if select_sql_header is not none -%} + {{ select_sql_header }} + {%- endif -%} select * from ( {{ select_sql }} ) as __dbt_sbq @@ -53,10 +56,10 @@ {%- endif -%} {% endmacro %} -{% macro get_column_schema_from_query(select_sql) -%} +{% macro get_column_schema_from_query(select_sql, select_sql_header=none) -%} {% set columns = [] %} {# -- Using an 'empty subquery' here to get the same schema as the given select_sql statement, without necessitating a data scan.#} - {% set sql = get_empty_subquery_sql(select_sql) %} + {% set sql = get_empty_subquery_sql(select_sql, select_sql_header) %} {% set column_schema = adapter.get_column_schema_from_query(sql) %} {{ return(column_schema) }} {% endmacro %} diff --git a/core/dbt/include/global_project/macros/materializations/models/table/columns_spec_ddl.sql b/core/dbt/include/global_project/macros/materializations/models/table/columns_spec_ddl.sql index 1ce7875868d..410d2d058f9 100644 --- a/core/dbt/include/global_project/macros/materializations/models/table/columns_spec_ddl.sql +++ b/core/dbt/include/global_project/macros/materializations/models/table/columns_spec_ddl.sql @@ -34,7 +34,7 @@ #} {% macro assert_columns_equivalent(sql) %} {#-- Obtain the column schema provided by sql file. #} - {%- set sql_file_provided_columns = get_column_schema_from_query(sql) -%} + {%- set sql_file_provided_columns = get_column_schema_from_query(sql, config.get('sql_header', none)) -%} {#--Obtain the column schema provided by the schema file by generating an 'empty schema' query from the model's columns. #} {%- set schema_file_provided_columns = get_column_schema_from_query(get_empty_schema_sql(model['columns'])) -%} From fb8d6b865340d2dbed050be638dec196512efc51 Mon Sep 17 00:00:00 2001 From: Michelle Ark Date: Wed, 31 May 2023 13:18:37 -0400 Subject: [PATCH 2/4] add tests --- .../dbt/tests/adapter/constraints/fixtures.py | 39 +++++++++++++++++++ .../adapter/constraints/test_constraints.py | 34 ++++++++++++++++ 2 files changed, 73 insertions(+) diff --git a/tests/adapter/dbt/tests/adapter/constraints/fixtures.py b/tests/adapter/dbt/tests/adapter/constraints/fixtures.py index 3310c59f3df..2edc6c41493 100644 --- a/tests/adapter/dbt/tests/adapter/constraints/fixtures.py +++ b/tests/adapter/dbt/tests/adapter/constraints/fixtures.py @@ -133,6 +133,33 @@ {sql_value} as wrong_data_type_column_name """ +my_model_contract_sql_header_sql = """ +{{ + config( + materialized = "table" + ) +}} + +{% call set_sql_header(config) %} +set session time zone 'Asia/Kolkata'; +{%- endcall %} +select current_setting('timezone') as column_name +""" + +my_model_incremental_contract_sql_header_sql = """ +{{ + config( + materialized = "incremental", + on_schema_change="append_new_columns" + ) +}} + +{% call set_sql_header(config) %} +set session time zone 'Asia/Kolkata'; +{%- endcall %} +select current_setting('timezone') as column_name +""" + # model breaking constraints my_model_with_nulls_sql = """ {{ @@ -303,3 +330,15 @@ - name: wrong_data_type_column_name data_type: {data_type} """ + +model_contract_header_schema_yml = """ +version: 2 +models: + - name: my_model_contract_sql_header + config: + contract: + enforced: true + columns: + - name: column_name + data_type: text +""" diff --git a/tests/adapter/dbt/tests/adapter/constraints/test_constraints.py b/tests/adapter/dbt/tests/adapter/constraints/test_constraints.py index 79140f03bbd..b826d41e8da 100644 --- a/tests/adapter/dbt/tests/adapter/constraints/test_constraints.py +++ b/tests/adapter/dbt/tests/adapter/constraints/test_constraints.py @@ -25,6 +25,9 @@ my_model_incremental_with_nulls_sql, model_schema_yml, constrained_model_schema_yml, + my_model_contract_sql_header_sql, + my_model_incremental_contract_sql_header_sql, + model_contract_header_schema_yml, ) @@ -358,6 +361,37 @@ class TestIncrementalConstraintsRollback(BaseIncrementalConstraintsRollback): pass +class BaseContractSqlHeader: + """Tests a contracted model with a sql header dependency.""" + + def test__contract_sql_header(self, project): + run_dbt(["run", "-s", "my_model_contract_sql_header"]) + + manifest = get_manifest(project.project_root) + model_id = "model.test.my_model_contract_sql_header" + model_config = manifest.nodes[model_id].config + + assert model_config.contract.enforced + + +class TestTableContractSqlHeader(BaseContractSqlHeader): + @pytest.fixture(scope="class") + def models(self): + return { + "my_model_contract_sql_header.sql": my_model_contract_sql_header_sql, + "constraints_schema.yml": model_contract_header_schema_yml, + } + + +class TestIncrementalContractSqlHeader(BaseContractSqlHeader): + @pytest.fixture(scope="class") + def models(self): + return { + "my_model_contract_sql_header.sql": my_model_incremental_contract_sql_header_sql, + "constraints_schema.yml": model_contract_header_schema_yml, + } + + class BaseModelConstraintsRuntimeEnforcement: """ These model-level constraints pass muster for dbt's preflight checks. Make sure they're From d5b44901727a8dc164f3d38cc6ea0142812f2ddf Mon Sep 17 00:00:00 2001 From: Michelle Ark Date: Wed, 31 May 2023 13:19:25 -0400 Subject: [PATCH 3/4] changelog entry --- .changes/unreleased/Fixes-20230531-131919.yaml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .changes/unreleased/Fixes-20230531-131919.yaml diff --git a/.changes/unreleased/Fixes-20230531-131919.yaml b/.changes/unreleased/Fixes-20230531-131919.yaml new file mode 100644 index 00000000000..0d5c1aec1b8 --- /dev/null +++ b/.changes/unreleased/Fixes-20230531-131919.yaml @@ -0,0 +1,6 @@ +kind: Fixes +body: send sql header on contract enforcement +time: 2023-05-31T13:19:19.801391-04:00 +custom: + Author: michelleark + Issue: "7714" From 4ae958376b275fc34c6548e1713e67b1c68c3e1b Mon Sep 17 00:00:00 2001 From: Michelle Ark Date: Wed, 31 May 2023 16:50:15 -0400 Subject: [PATCH 4/4] renaming tests for consistency --- .../tests/adapter/constraints/test_constraints.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/tests/adapter/dbt/tests/adapter/constraints/test_constraints.py b/tests/adapter/dbt/tests/adapter/constraints/test_constraints.py index b826d41e8da..9ca4f23a95f 100644 --- a/tests/adapter/dbt/tests/adapter/constraints/test_constraints.py +++ b/tests/adapter/dbt/tests/adapter/constraints/test_constraints.py @@ -374,7 +374,7 @@ def test__contract_sql_header(self, project): assert model_config.contract.enforced -class TestTableContractSqlHeader(BaseContractSqlHeader): +class BaseTableContractSqlHeader(BaseContractSqlHeader): @pytest.fixture(scope="class") def models(self): return { @@ -383,7 +383,7 @@ def models(self): } -class TestIncrementalContractSqlHeader(BaseContractSqlHeader): +class BaseIncrementalContractSqlHeader(BaseContractSqlHeader): @pytest.fixture(scope="class") def models(self): return { @@ -392,6 +392,14 @@ def models(self): } +class TestTableContractSqlHeader(BaseTableContractSqlHeader): + pass + + +class TestIncrementalContractSqlHeader(BaseIncrementalContractSqlHeader): + pass + + class BaseModelConstraintsRuntimeEnforcement: """ These model-level constraints pass muster for dbt's preflight checks. Make sure they're