dbt-core/test/rpc/test_deps.py at 30fed8d4218ce79d324ec41741fd41b79f678e9d · dbt-labs/dbt-core · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
import pytest

from .util import (
    get_querier,
    ProjectDefinition,
)


def deps_with_packages(packages, bad_packages, project_dir, profiles_dir, schema):
    project = ProjectDefinition(
        models={
            'my_model.sql': 'select 1 as id',
        },
        packages={'packages': packages},
    )
    querier_ctx = get_querier(
        project_def=project,
        project_dir=project_dir,
        profiles_dir=profiles_dir,
        schema=schema,
        test_kwargs={},
    )

    with querier_ctx as querier:
        # we should be able to run sql queries at startup
        querier.async_wait_for_result(querier.run_sql('select 1 as id'))

        # the status should be something positive
        querier.is_result(querier.status())

        # deps should pass
        querier.async_wait_for_result(querier.deps())

        # queries should work after deps
        tok1 = querier.is_async_result(querier.run())
        tok2 = querier.is_async_result(querier.run_sql('select 1 as id'))

        querier.is_result(querier.async_wait(tok2))
        querier.is_result(querier.async_wait(tok1))

        # now break the project
        project.packages['packages'] = bad_packages
        project.write_packages(project_dir, remove=True)

        # queries should still work because we haven't reloaded
        tok1 = querier.is_async_result(querier.run())
        tok2 = querier.is_async_result(querier.run_sql('select 1 as id'))

        querier.is_result(querier.async_wait(tok2))
        querier.is_result(querier.async_wait(tok1))

        # now run deps again, it should be sad
        querier.async_wait_for_error(querier.deps())
        # it should also not be running.
        result = querier.is_result(querier.ps(active=True, completed=False))
        assert result['rows'] == []

        # fix packages again
        project.packages['packages'] = packages
        project.write_packages(project_dir, remove=True)
        # keep queries broken, we haven't run deps yet
        querier.is_error(querier.run())

        # deps should pass now
        querier.async_wait_for_result(querier.deps())
        querier.is_result(querier.status())

        tok1 = querier.is_async_result(querier.run())
        tok2 = querier.is_async_result(querier.run_sql('select 1 as id'))

        querier.is_result(querier.async_wait(tok2))
        querier.is_result(querier.async_wait(tok1))


@pytest.mark.parametrize(
    "packages, bad_packages",
    # from dbt hub
    [(
        [{
            'package': 'fishtown-analytics/dbt_utils',
            'version': '0.5.0',
        }],
        # wrong package name
        [{
            'package': 'fishtown-analytics/dbt_util',
            'version': '0.5.0',
        }],
    ),
    # from git release/tag/branch
    (
        [{
            'git': 'https://github.com/fishtown-analytics/dbt-utils.git',
            'revision': '0.5.0',
        }],
        # if you use a bad URL, git thinks it's a private repo and prompts for auth
        [{
            'git': 'https://github.com/fishtown-analytics/dbt-utils.git',
            'revision': 'not-a-real-revision',
        }],
    ),
    # from git commit
    (
        [{
            'git': 'https://github.com/fishtown-analytics/dbt-utils.git',
            'revision': 'b736cf6acdbf80d2de69b511a51c8d7fe214ee79',
        }],
        # don't use short commits
        [{
            'git': 'https://github.com/fishtown-analytics/dbt-utils.git',
            'revision': 'b736cf6',
        }],
    ),
    # from git release and subdirectory
    (
        [{
            'git': 'https://github.com/fishtown-analytics/dbt-labs-experimental-features.git',
            'revision': '0.0.1',
            'subdirectory': 'materialized-views',
        }],
        [{
            'git': 'https://github.com/fishtown-analytics/dbt-labs-experimental-features.git',
            'revision': '0.0.1',
            'subdirectory': 'path/to/nonexistent/dir',
        }],
    ),
    # from git commit and subdirectory
    (
        [{
            'git': 'https://github.com/fishtown-analytics/dbt-utils.git',
            'revision': 'f4f84e9110db26aba22f756abbae9f1f8dbb15da',
            'subdirectory': 'dbt_projects/dbt_utils',
        }],
        [{
            'git': 'https://github.com/fishtown-analytics/dbt-utils.git',
            'revision': 'f4f84e9110db26aba22f756abbae9f1f8dbb15da',
            'subdirectory': 'path/to/nonexistent/dir',
        }],
    )],
    ids=[
        "from dbt hub",
        "from git release/tag/branch",
        "from git commit",
        "from git release and subdirectory",
        "from git commit and subdirectory",
    ],
)
@pytest.mark.supported('postgres')
def test_rpc_deps_packages(project_root, profiles_root, dbt_profile, unique_schema, packages, bad_packages):
    deps_with_packages(packages, bad_packages, project_root, profiles_root, unique_schema)