From 9179070416b21e7fb07d7a6a5929f01d4576175a Mon Sep 17 00:00:00 2001 From: Ezra Date: Mon, 24 Apr 2023 16:17:49 -0400 Subject: [PATCH 1/4] Use profile specified in --profile with dbt init --- .../unreleased/Fixes-20230424-161642.yaml | 7 + core/dbt/task/init.py | 27 +++- tests/functional/init/test_init.py | 149 ++++++++++++++++++ 3 files changed, 181 insertions(+), 2 deletions(-) create mode 100644 .changes/unreleased/Fixes-20230424-161642.yaml diff --git a/.changes/unreleased/Fixes-20230424-161642.yaml b/.changes/unreleased/Fixes-20230424-161642.yaml new file mode 100644 index 00000000000..aeeb8e1ccab --- /dev/null +++ b/.changes/unreleased/Fixes-20230424-161642.yaml @@ -0,0 +1,7 @@ +kind: Fixes +body: If --profile specified with dbt-init, create the project with the specified + profile +time: 2023-04-24T16:16:42.994547-04:00 +custom: + Author: ezraerb + Issue: CT-1418 diff --git a/core/dbt/task/init.py b/core/dbt/task/init.py index dcc37d859bf..b9b6468f714 100644 --- a/core/dbt/task/init.py +++ b/core/dbt/task/init.py @@ -11,6 +11,7 @@ import dbt.config import dbt.clients.system +from dbt.config.profile import read_profile from dbt.flags import get_flags from dbt.version import _get_adapter_plugin_names from dbt.adapters.factory import load_plugin, get_include_paths @@ -256,6 +257,14 @@ def run(self): in_project = False if in_project: + # If --profile was specified, it means use an existing profile, which is not + # applicable to this case + if getattr(get_flags(), "PROFILE", None): + print( + "Can not init existing project with specified profile, edit dbt_project.yml instead" + ) + sys.exit(1) + # When dbt init is run inside an existing project, # just setup the user's profile. fire_event(SettingUpProfile()) @@ -289,16 +298,30 @@ def run(self): fire_event(ProjectNameAlreadyExists(name=project_name)) return + # If the user specified an existing profile to use, use it instead of generating a new one + user_profile_name = getattr(get_flags(), "PROFILE", None) + if user_profile_name: + # Verify it exists. Can't use the regular profile validation routine because it assumes + # the project file exists + raw_profiles = read_profile(profiles_dir) + if user_profile_name not in raw_profiles: + print("Could not find profile named '{}'".format(user_profile_name)) + sys.exit(1) + self.copy_starter_repo(project_name) os.chdir(project_name) with open("dbt_project.yml", "r+") as f: - content = f"{f.read()}".format(project_name=project_name, profile_name=project_name) + content = f"{f.read()}".format( + project_name=project_name, + profile_name=user_profile_name if user_profile_name else project_name, + ) f.seek(0) f.write(content) f.truncate() + # If an existing profile to use was not provided, generate a profile # Ask for adapter only if skip_profile_setup flag is not provided. - if not self.args.skip_profile_setup: + if not user_profile_name and not self.args.skip_profile_setup: if not self.check_if_can_write_profile(profile_name=project_name): return adapter = self.ask_for_adapter_choice() diff --git a/tests/functional/init/test_init.py b/tests/functional/init/test_init.py index 4e48eeeffb5..cc95552ef56 100644 --- a/tests/functional/init/test_init.py +++ b/tests/functional/init/test_init.py @@ -1,5 +1,6 @@ import click import os +import yaml import pytest from pathlib import Path from unittest import mock @@ -84,6 +85,12 @@ def test_init_task_in_project_with_existing_profiles_yml( """ ) + def test_init_task_in_project_specifying_profile_errors(self): + # This triggers a call to sys.exit(), requring the following to test it + with pytest.raises(SystemExit) as error: + run_dbt(["init", "--profile", "test"]) + assert error.value.code == 1 + class TestInitProjectWithoutExistingProfilesYml: @mock.patch("dbt.task.init._get_adapter_plugin_names") @@ -159,6 +166,21 @@ def exists_side_effect(path): """ ) + @mock.patch.object(Path, "exists", autospec=True) + def test_init_task_in_project_without_profile_yml_specifying_profile_errors(self, exists): + def exists_side_effect(path): + # Override responses on specific files, default to 'real world' if not overriden + return {"profiles.yml": False}.get(path.name, os.path.exists(path)) + + exists.side_effect = exists_side_effect + + # Even through no profiles.yml file exists, the init will not modify project.yml, + # so this errors + # This triggers a call to sys.exit(), requring the following to test it + with pytest.raises(SystemExit) as error: + run_dbt(["init", "--profile", "test"]) + assert error.value.code == 1 + class TestInitProjectWithoutExistingProfilesYmlOrTemplate: @mock.patch("dbt.task.init._get_adapter_plugin_names") @@ -684,3 +706,130 @@ def test_init_provided_project_name_and_skip_profile_setup( +materialized: view """ ) + + +class TestInitOutsideOfProjectWithSpecifiedProfile(TestInitOutsideOfProjectBase): + @mock.patch("dbt.task.init._get_adapter_plugin_names") + @mock.patch("click.prompt") + def test_init_task_outside_of_project_with_specified_profile( + self, mock_prompt, mock_get_adapter, project, project_name, unique_schema, dbt_profile_data + ): + manager = Mock() + manager.attach_mock(mock_prompt, "prompt") + manager.prompt.side_effect = [ + project_name, + ] + mock_get_adapter.return_value = [project.adapter.type()] + run_dbt(["init", "--profile", "test"]) + + manager.assert_has_calls( + [ + call.prompt("Enter a name for your project (letters, digits, underscore)"), + ] + ) + + # profiles.yml is NOT overwritten, so assert that the text matches that of the + # original fixture + with open(os.path.join(project.profiles_dir, "profiles.yml"), "r") as f: + assert f.read() == yaml.safe_dump(dbt_profile_data) + + with open(os.path.join(project.project_root, project_name, "dbt_project.yml"), "r") as f: + assert ( + f.read() + == f""" +# Name your project! Project names should contain only lowercase characters +# and underscores. A good package name should reflect your organization's +# name or the intended use of these models +name: '{project_name}' +version: '1.0.0' +config-version: 2 + +# This setting configures which "profile" dbt uses for this project. +profile: 'test' + +# These configurations specify where dbt should look for different types of files. +# The `model-paths` config, for example, states that models in this project can be +# found in the "models/" directory. You probably won't need to change these! +model-paths: ["models"] +analysis-paths: ["analyses"] +test-paths: ["tests"] +seed-paths: ["seeds"] +macro-paths: ["macros"] +snapshot-paths: ["snapshots"] + +clean-targets: # directories to be removed by `dbt clean` + - "target" + - "dbt_packages" + + +# Configuring models +# Full documentation: https://docs.getdbt.com/docs/configuring-models + +# In this example config, we tell dbt to build all models in the example/ +# directory as views. These settings can be overridden in the individual model +# files using the `{{{{ config(...) }}}}` macro. +models: + {project_name}: + # Config indicated by + and applies to all files under models/example/ + example: + +materialized: view +""" + ) + + +class TestInitOutsideOfProjectSpecifyingInvalidProfile(TestInitOutsideOfProjectBase): + @mock.patch("dbt.task.init._get_adapter_plugin_names") + @mock.patch("click.prompt") + def test_init_task_outside_project_specifying_invalid_profile_errors( + self, mock_prompt, mock_get_adapter, project, project_name + ): + manager = Mock() + manager.attach_mock(mock_prompt, "prompt") + manager.prompt.side_effect = [ + project_name, + ] + mock_get_adapter.return_value = [project.adapter.type()] + + # This triggers a call to sys.exit(), requring the following to test it + with pytest.raises(SystemExit) as error: + run_dbt(["init", "--profile", "invalid"]) + assert error.value.code == 1 + + manager.assert_has_calls( + [ + call.prompt("Enter a name for your project (letters, digits, underscore)"), + ] + ) + + +class TestInitOutsideOfProjectSpecifyingProfileNoProfilesYml(TestInitOutsideOfProjectBase): + @mock.patch("dbt.task.init._get_adapter_plugin_names") + @mock.patch("click.prompt") + def test_init_task_outside_project_specifying_profile_no_profiles_yml_errors( + self, mock_prompt, mock_get_adapter, project, project_name + ): + manager = Mock() + manager.attach_mock(mock_prompt, "prompt") + manager.prompt.side_effect = [ + project_name, + ] + mock_get_adapter.return_value = [project.adapter.type()] + + # Override responses on specific files, default to 'real world' if not overriden + original_isfile = os.path.isfile + with mock.patch( + "os.path.isfile", + new=lambda path: {"profiles.yml": False}.get( + os.path.basename(path), original_isfile(path) + ), + ): + # This triggers a call to sys.exit(), requring the following to test it + with pytest.raises(SystemExit) as error: + run_dbt(["init", "--profile", "test"]) + assert error.value.code == 1 + + manager.assert_has_calls( + [ + call.prompt("Enter a name for your project (letters, digits, underscore)"), + ] + ) From 2755820e043d4e5e1b605682b052f269c1990859 Mon Sep 17 00:00:00 2001 From: ezraerb Date: Mon, 22 May 2023 20:55:04 -0400 Subject: [PATCH 2/4] Update .changes/unreleased/Fixes-20230424-161642.yaml Co-authored-by: Doug Beatty <44704949+dbeatty10@users.noreply.github.com> --- .changes/unreleased/Fixes-20230424-161642.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changes/unreleased/Fixes-20230424-161642.yaml b/.changes/unreleased/Fixes-20230424-161642.yaml index aeeb8e1ccab..1eae7a7bd32 100644 --- a/.changes/unreleased/Fixes-20230424-161642.yaml +++ b/.changes/unreleased/Fixes-20230424-161642.yaml @@ -4,4 +4,4 @@ body: If --profile specified with dbt-init, create the project with the specifie time: 2023-04-24T16:16:42.994547-04:00 custom: Author: ezraerb - Issue: CT-1418 + Issue: "6154" From 7e921cea2a5ac0a7df6c349193ac970648af08de Mon Sep 17 00:00:00 2001 From: Ezra Date: Wed, 7 Jun 2023 17:08:49 -0400 Subject: [PATCH 3/4] Refactor run() method into functions, replace exit() calls with exceptions --- core/dbt/task/init.py | 83 +++++++++++++++++------------- tests/functional/init/test_init.py | 30 +++++------ 2 files changed, 61 insertions(+), 52 deletions(-) diff --git a/core/dbt/task/init.py b/core/dbt/task/init.py index 63ca074ea3c..39d18b8dcb2 100644 --- a/core/dbt/task/init.py +++ b/core/dbt/task/init.py @@ -3,7 +3,6 @@ from pathlib import Path import re import shutil -import sys from typing import Optional import yaml @@ -12,6 +11,7 @@ import dbt.config import dbt.clients.system from dbt.config.profile import read_profile +from dbt.exceptions import DbtRuntimeError from dbt.flags import get_flags from dbt.version import _get_adapter_plugin_names from dbt.adapters.factory import load_plugin, get_include_paths @@ -190,6 +190,15 @@ def create_profile_from_target(self, adapter: str, profile_name: str): # sample_profiles.yml self.create_profile_from_sample(adapter, profile_name) + def check_if_profile_exists(self, profile_name: str) -> bool: + """ + Validate that the specified profile exists. Can't use the regular profile validation + routine because it assumes the project file exists + """ + profiles_dir = get_flags().PROFILES_DIR + raw_profiles = read_profile(profiles_dir) + return profile_name in raw_profiles + def check_if_can_write_profile(self, profile_name: Optional[str] = None) -> bool: """Using either a provided profile name or that specified in dbt_project.yml, check if the profile already exists in profiles.yml, and if so ask the @@ -235,6 +244,25 @@ def ask_for_adapter_choice(self) -> str: numeric_choice = click.prompt(prompt_msg, type=click.INT) return available_adapters[numeric_choice - 1] + def setup_profile(self, profile_name: str) -> None: + """Set up a new profile for a project""" + fire_event(SettingUpProfile()) + if not self.check_if_can_write_profile(profile_name=profile_name): + return + # If a profile_template.yml exists in the project root, that effectively + # overrides the profile_template.yml for the given target. + profile_template_path = Path("profile_template.yml") + if profile_template_path.exists(): + try: + # This relies on a valid profile_template.yml from the user, + # so use a try: except to fall back to the default on failure + self.create_profile_using_project_profile_template(profile_name) + return + except Exception: + fire_event(InvalidProfileTemplateYAML()) + adapter = self.ask_for_adapter_choice() + self.create_profile_from_target(adapter, profile_name=profile_name) + def get_valid_project_name(self) -> str: """Returns a valid project name, either from CLI arg or user prompt.""" name = self.args.project_name @@ -278,16 +306,16 @@ def run(self): if in_project: # If --profile was specified, it means use an existing profile, which is not # applicable to this case - if getattr(get_flags(), "PROFILE", None): - print( - "Can not init existing project with specified profile, edit dbt_project.yml instead" + if self.args.profile: + raise DbtRuntimeError( + msg="Can not init existing project with specified profile, edit dbt_project.yml instead" ) - sys.exit(1) # When dbt init is run inside an existing project, # just setup the user's profile. - profile_name = self.get_profile_name_from_current_project() - profile_specified = False + if not self.args.skip_profile_setup: + profile_name = self.get_profile_name_from_current_project() + self.setup_profile(profile_name) else: # When dbt init is run outside of an existing project, # create a new project and set up the user's profile. @@ -298,36 +326,19 @@ def run(self): return # If the user specified an existing profile to use, use it instead of generating a new one - user_profile_name = getattr(get_flags(), "PROFILE", None) + user_profile_name = self.args.profile if user_profile_name: - # Verify it exists. Can't use the regular profile validation routine because it assumes - # the project file exists - raw_profiles = read_profile(profiles_dir) - if user_profile_name not in raw_profiles: - print("Could not find profile named '{}'".format(user_profile_name)) - sys.exit(1) - profile_name = user_profile_name - profile_specified = True + if not self.check_if_profile_exists(user_profile_name): + raise DbtRuntimeError( + msg="Could not find profile named '{}'".format(user_profile_name) + ) + self.create_new_project(project_name, user_profile_name) else: profile_name = project_name - profile_specified = False - self.create_new_project(project_name, profile_name) + # Create the profile after creating the project to avoid leaving a random profile + # if the former fails. + self.create_new_project(project_name, profile_name) - # Ask for adapter only if skip_profile_setup flag is not provided and no profile to use was specified. - if not self.args.skip_profile_setup and not profile_specified: - fire_event(SettingUpProfile()) - if not self.check_if_can_write_profile(profile_name=profile_name): - return - # If a profile_template.yml exists in the project root, that effectively - # overrides the profile_template.yml for the given target. - profile_template_path = Path("profile_template.yml") - if profile_template_path.exists(): - try: - # This relies on a valid profile_template.yml from the user, - # so use a try: except to fall back to the default on failure - self.create_profile_using_project_profile_template(profile_name) - return - except Exception: - fire_event(InvalidProfileTemplateYAML()) - adapter = self.ask_for_adapter_choice() - self.create_profile_from_target(adapter, profile_name=profile_name) + # Ask for adapter only if skip_profile_setup flag is not provided + if not self.args.skip_profile_setup: + self.setup_profile(profile_name) diff --git a/tests/functional/init/test_init.py b/tests/functional/init/test_init.py index dfcfdc379ab..9ac821d7c26 100644 --- a/tests/functional/init/test_init.py +++ b/tests/functional/init/test_init.py @@ -6,6 +6,8 @@ from unittest import mock from unittest.mock import Mock, call +from dbt.exceptions import DbtRuntimeError + from dbt.tests.util import run_dbt @@ -86,10 +88,9 @@ def test_init_task_in_project_with_existing_profiles_yml( ) def test_init_task_in_project_specifying_profile_errors(self): - # This triggers a call to sys.exit(), requring the following to test it - with pytest.raises(SystemExit) as error: - run_dbt(["init", "--profile", "test"]) - assert error.value.code == 1 + with pytest.raises(DbtRuntimeError) as error: + run_dbt(["init", "--profile", "test"], expect_pass=False) + assert "Can not init existing project with specified profile" in str(error) class TestInitProjectWithoutExistingProfilesYml: @@ -176,10 +177,9 @@ def exists_side_effect(path): # Even through no profiles.yml file exists, the init will not modify project.yml, # so this errors - # This triggers a call to sys.exit(), requring the following to test it - with pytest.raises(SystemExit) as error: - run_dbt(["init", "--profile", "test"]) - assert error.value.code == 1 + with pytest.raises(DbtRuntimeError) as error: + run_dbt(["init", "--profile", "test"], expect_pass=False) + assert "Could not find profile named test" in str(error) class TestInitProjectWithoutExistingProfilesYmlOrTemplate: @@ -814,10 +814,9 @@ def test_init_task_outside_project_specifying_invalid_profile_errors( ] mock_get_adapter.return_value = [project.adapter.type()] - # This triggers a call to sys.exit(), requring the following to test it - with pytest.raises(SystemExit) as error: - run_dbt(["init", "--profile", "invalid"]) - assert error.value.code == 1 + with pytest.raises(DbtRuntimeError) as error: + run_dbt(["init", "--profile", "invalid"], expect_pass=False) + assert "Could not find profile named invalid" in str(error) manager.assert_has_calls( [ @@ -847,10 +846,9 @@ def test_init_task_outside_project_specifying_profile_no_profiles_yml_errors( os.path.basename(path), original_isfile(path) ), ): - # This triggers a call to sys.exit(), requring the following to test it - with pytest.raises(SystemExit) as error: - run_dbt(["init", "--profile", "test"]) - assert error.value.code == 1 + with pytest.raises(DbtRuntimeError) as error: + run_dbt(["init", "--profile", "test"], expect_pass=False) + assert "Could not find profile named invalid" in str(error) manager.assert_has_calls( [ From 9a58a15ad0d9f6b473fe0161c0a76fdb395a092f Mon Sep 17 00:00:00 2001 From: Ezra Date: Thu, 14 Sep 2023 22:27:10 -0400 Subject: [PATCH 4/4] Update help text for profile option --- core/dbt/cli/params.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/dbt/cli/params.py b/core/dbt/cli/params.py index b638fc539dc..e59756105cb 100644 --- a/core/dbt/cli/params.py +++ b/core/dbt/cli/params.py @@ -281,7 +281,7 @@ profile = click.option( "--profile", envvar=None, - help="Which profile to load. Overrides setting in dbt_project.yml.", + help="Which existing profile to load. Overrides setting in dbt_project.yml.", ) profiles_dir = click.option(