Skip to content

Commit 0e4a6e6

Browse files
github-actions[bot]github-actions[bot]
authored
add TestLargeEphemeralCompilation (#8376) (#8717)
1 parent 203e331 commit 0e4a6e6

File tree

3 files changed

+291
-43
lines changed

3 files changed

+291
-43
lines changed

‎.changes/unreleased/Under the Hood-20230912-230619.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
kind: Under the Hood
2+
body: add a test for ephemeral cte injection
3+
time: 2023-09-12T23:06:19.938207+01:00
4+
custom:
5+
Author: michelleark
6+
Issue: "8376"

‎tests/functional/materializations/fixtures.py‎

Lines changed: 245 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,245 @@
1+
fct_eph_first_sql = """
2+
-- fct_eph_first.sql
3+
{{ config(materialized='ephemeral') }}
4+
5+
with int_eph_first as(
6+
select * from {{ ref('int_eph_first') }}
7+
)
8+
9+
select * from int_eph_first
10+
"""
11+
12+
int_eph_first_sql = """
13+
-- int_eph_first.sql
14+
{{ config(materialized='ephemeral') }}
15+
16+
select
17+
1 as first_column,
18+
2 as second_column
19+
"""
20+
21+
schema_yml = """
22+
version: 2
23+
24+
models:
25+
- name: int_eph_first
26+
columns:
27+
- name: first_column
28+
tests:
29+
- not_null
30+
- name: second_column
31+
tests:
32+
- not_null
33+
34+
- name: fct_eph_first
35+
columns:
36+
- name: first_column
37+
tests:
38+
- not_null
39+
- name: second_column
40+
tests:
41+
- not_null
42+
43+
"""
44+
45+
bar_sql = """
46+
{{ config(materialized = 'table') }}
47+
48+
WITH foo AS (
49+
50+
SELECT * FROM {{ ref('foo') }}
51+
52+
), foo_1 AS (
53+
54+
SELECT * FROM {{ ref('foo_1') }}
55+
56+
), foo_2 AS (
57+
58+
SELECT * FROM {{ ref('foo_2') }}
59+
60+
)
61+
62+
SELECT * FROM foo
63+
UNION ALL
64+
SELECT * FROM foo_1
65+
UNION ALL
66+
SELECT * FROM foo_2
67+
"""
68+
69+
bar1_sql = """
70+
{{ config(materialized = 'table') }}
71+
72+
WITH foo AS (
73+
74+
SELECT * FROM {{ ref('foo') }}
75+
76+
), foo_1 AS (
77+
78+
SELECT * FROM {{ ref('foo_1') }}
79+
80+
), foo_2 AS (
81+
82+
SELECT * FROM {{ ref('foo_2') }}
83+
84+
)
85+
86+
SELECT * FROM foo
87+
UNION ALL
88+
SELECT * FROM foo_1
89+
UNION ALL
90+
SELECT * FROM foo_2
91+
"""
92+
93+
bar2_sql = """
94+
{{ config(materialized = 'table') }}
95+
96+
WITH foo AS (
97+
98+
SELECT * FROM {{ ref('foo') }}
99+
100+
), foo_1 AS (
101+
102+
SELECT * FROM {{ ref('foo_1') }}
103+
104+
), foo_2 AS (
105+
106+
SELECT * FROM {{ ref('foo_2') }}
107+
108+
)
109+
110+
SELECT * FROM foo
111+
UNION ALL
112+
SELECT * FROM foo_1
113+
UNION ALL
114+
SELECT * FROM foo_2
115+
"""
116+
117+
bar3_sql = """
118+
{{ config(materialized = 'table') }}
119+
120+
WITH foo AS (
121+
122+
SELECT * FROM {{ ref('foo') }}
123+
124+
), foo_1 AS (
125+
126+
SELECT * FROM {{ ref('foo_1') }}
127+
128+
), foo_2 AS (
129+
130+
SELECT * FROM {{ ref('foo_2') }}
131+
132+
)
133+
134+
SELECT * FROM foo
135+
UNION ALL
136+
SELECT * FROM foo_1
137+
UNION ALL
138+
SELECT * FROM foo_2
139+
"""
140+
141+
bar4_sql = """
142+
{{ config(materialized = 'table') }}
143+
144+
WITH foo AS (
145+
146+
SELECT * FROM {{ ref('foo') }}
147+
148+
), foo_1 AS (
149+
150+
SELECT * FROM {{ ref('foo_1') }}
151+
152+
), foo_2 AS (
153+
154+
SELECT * FROM {{ ref('foo_2') }}
155+
156+
)
157+
158+
SELECT * FROM foo
159+
UNION ALL
160+
SELECT * FROM foo_1
161+
UNION ALL
162+
SELECT * FROM foo_2
163+
"""
164+
165+
bar5_sql = """
166+
{{ config(materialized = 'table') }}
167+
168+
WITH foo AS (
169+
170+
SELECT * FROM {{ ref('foo') }}
171+
172+
), foo_1 AS (
173+
174+
SELECT * FROM {{ ref('foo_1') }}
175+
176+
), foo_2 AS (
177+
178+
SELECT * FROM {{ ref('foo_2') }}
179+
180+
)
181+
182+
SELECT * FROM foo
183+
UNION ALL
184+
SELECT * FROM foo_1
185+
UNION ALL
186+
SELECT * FROM foo_2
187+
"""
188+
189+
baz_sql = """
190+
{{ config(materialized = 'table') }}
191+
SELECT * FROM {{ ref('bar') }}
192+
"""
193+
194+
baz1_sql = """
195+
{{ config(materialized = 'table') }}
196+
SELECT * FROM {{ ref('bar_1') }}
197+
"""
198+
199+
foo_sql = """
200+
{{ config(materialized = 'ephemeral') }}
201+
202+
with source as (
203+
204+
select 1 as id
205+
206+
), renamed as (
207+
208+
select id as uid from source
209+
210+
)
211+
212+
select * from renamed
213+
"""
214+
215+
foo1_sql = """
216+
{{ config(materialized = 'ephemeral') }}
217+
218+
WITH source AS (
219+
220+
SELECT 1 AS id
221+
222+
), RENAMED as (
223+
224+
SELECT id as UID FROM source
225+
226+
)
227+
228+
SELECT * FROM renamed
229+
"""
230+
231+
foo2_sql = """
232+
{{ config(materialized = 'ephemeral') }}
233+
234+
WITH source AS (
235+
236+
SELECT 1 AS id
237+
238+
), RENAMED as (
239+
240+
SELECT id as UID FROM source
241+
242+
)
243+
244+
SELECT * FROM renamed
245+
"""

‎tests/functional/materializations/test_ephemeral_compilation.py‎

Lines changed: 40 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -9,51 +9,23 @@
99
# fails fairly regularly if that is broken, but does occasionally work (depending
1010
# on the order in which things are compiled). It requires multi-threading to fail.
1111

12-
13-
fct_eph_first_sql = """
14-
-- fct_eph_first.sql
15-
{{ config(materialized='ephemeral') }}
16-
17-
with int_eph_first as(
18-
select * from {{ ref('int_eph_first') }}
12+
from tests.functional.materializations.fixtures import (
13+
fct_eph_first_sql,
14+
int_eph_first_sql,
15+
schema_yml,
16+
bar_sql,
17+
bar1_sql,
18+
bar2_sql,
19+
bar3_sql,
20+
bar4_sql,
21+
bar5_sql,
22+
baz_sql,
23+
baz1_sql,
24+
foo_sql,
25+
foo1_sql,
26+
foo2_sql,
1927
)
2028

21-
select * from int_eph_first
22-
"""
23-
24-
int_eph_first_sql = """
25-
-- int_eph_first.sql
26-
{{ config(materialized='ephemeral') }}
27-
28-
select
29-
1 as first_column,
30-
2 as second_column
31-
"""
32-
33-
schema_yml = """
34-
version: 2
35-
36-
models:
37-
- name: int_eph_first
38-
columns:
39-
- name: first_column
40-
tests:
41-
- not_null
42-
- name: second_column
43-
tests:
44-
- not_null
45-
46-
- name: fct_eph_first
47-
columns:
48-
- name: first_column
49-
tests:
50-
- not_null
51-
- name: second_column
52-
tests:
53-
- not_null
54-
55-
"""
56-
5729

5830
SUPPRESSED_CTE_EXPECTED_OUTPUT = """-- fct_eph_first.sql
5931
@@ -89,3 +61,28 @@ def test__suppress_injected_ctes(self, project):
8961
node = node_result.node
9062
assert isinstance(node, ModelNode)
9163
assert node.compiled_code == SUPPRESSED_CTE_EXPECTED_OUTPUT
64+
65+
66+
# From: https://github.com/jeremyyeo/ephemeral-invalid-sql-repro/tree/main/models
67+
class TestLargeEphemeralCompilation:
68+
@pytest.fixture(scope="class")
69+
def models(self):
70+
71+
return {
72+
"bar.sql": bar_sql,
73+
"bar_1.sql": bar1_sql,
74+
"bar_2.sql": bar2_sql,
75+
"bar_3.sql": bar3_sql,
76+
"bar_4.sql": bar4_sql,
77+
"bar_5.sql": bar5_sql,
78+
"baz.sql": baz_sql,
79+
"baz_1.sql": baz1_sql,
80+
"foo.sql": foo_sql,
81+
"foo_1.sql": foo1_sql,
82+
"foo_2.sql": foo2_sql,
83+
}
84+
85+
def test_ephemeral_compilation(self, project):
86+
# 8/11 table models are built as expected. no compilation errors
87+
results = run_dbt(["build"])
88+
assert len(results) == 8

0 commit comments

Comments
 (0)