diff --git a/core/dbt/adapters/cache.py b/core/dbt/adapters/cache.py index 24a0e469df1..dc8ff14e67e 100644 --- a/core/dbt/adapters/cache.py +++ b/core/dbt/adapters/cache.py @@ -17,7 +17,7 @@ ) from dbt.events.functions import fire_event, fire_event_if from dbt.events.types import CacheAction, CacheDumpGraph -import dbt.flags as flags +from dbt.flags import get_flags from dbt.utils import lowercase @@ -319,6 +319,7 @@ def add(self, relation): :param BaseRelation relation: The underlying relation. """ + flags = get_flags() cached = _CachedRelation(relation) fire_event_if( flags.LOG_CACHE_EVENTS, @@ -456,7 +457,7 @@ def rename(self, old, new): ref_key_2=_make_msg_from_ref_key(new), ) ) - + flags = get_flags() fire_event_if( flags.LOG_CACHE_EVENTS, lambda: CacheDumpGraph(before_after="before", action="rename", dump=self.dump_graph()), diff --git a/core/dbt/cli/flags.py b/core/dbt/cli/flags.py index dcfb59507c5..e6d15ced5ab 100644 --- a/core/dbt/cli/flags.py +++ b/core/dbt/cli/flags.py @@ -12,30 +12,85 @@ from dbt.config.profile import read_user_config from dbt.contracts.project import UserConfig +from dbt.helper_types import WarnErrorOptions +from dbt.config.project import PartialProject +from dbt.exceptions import DbtProjectError if os.name != "nt": # https://bugs.python.org/issue41567 import multiprocessing.popen_spawn_posix # type: ignore # noqa: F401 +# TODO anything that has a default in params should be removed here? +# Or maybe only the ones that's in the root click group +FLAGS_DEFAULTS = { + "INDIRECT_SELECTION": "eager", + "TARGET_PATH": None, + # cli args without user_config or env var option + "FULL_REFRESH": False, + "STRICT_MODE": False, + "STORE_FAILURES": False, +} + + +# For backwards compatability, some params are defined across multiple levels, +# Top-level value should take precedence. +# e.g. dbt --target-path test2 run --target-path test2 +EXPECTED_DUPLICATE_PARAMS = [ + "full_refresh", + "target_path", + "version_check", + "fail_fast", + "indirect_selection", + "store_failures", +] + + +def convert_config(config_name, config_value): + # This function should take care of converting the values from config and original + # set_from_args to the correct type + ret = config_value + if config_name.lower() == "warn_error_options": + ret = WarnErrorOptions( + include=config_value.get("include", []), exclude=config_value.get("exclude", []) + ) + return ret + @dataclass(frozen=True) class Flags: def __init__(self, ctx: Context = None, user_config: UserConfig = None) -> None: + # set the default flags + for key, value in FLAGS_DEFAULTS.items(): + object.__setattr__(self, key, value) + if ctx is None: ctx = get_current_context() def assign_params(ctx, params_assigned_from_default): """Recursively adds all click params to flag object""" for param_name, param_value in ctx.params.items(): + # TODO: this is to avoid duplicate params being defined in two places (version_check in run and cli) + # However this is a bit of a hack and we should find a better way to do this + # N.B. You have to use the base MRO method (object.__setattr__) to set attributes # when using frozen dataclasses. # https://docs.python.org/3/library/dataclasses.html#frozen-instances - if hasattr(self, param_name): - raise Exception(f"Duplicate flag names found in click command: {param_name}") - object.__setattr__(self, param_name.upper(), param_value) - if ctx.get_parameter_source(param_name) == ParameterSource.DEFAULT: - params_assigned_from_default.add(param_name) + if hasattr(self, param_name.upper()): + if param_name not in EXPECTED_DUPLICATE_PARAMS: + raise Exception( + f"Duplicate flag names found in click command: {param_name}" + ) + else: + # Expected duplicate param from multi-level click command (ex: dbt --full_refresh run --full_refresh) + # Overwrite user-configured param with value from parent context + if ctx.get_parameter_source(param_name) != ParameterSource.DEFAULT: + object.__setattr__(self, param_name.upper(), param_value) + else: + object.__setattr__(self, param_name.upper(), param_value) + if ctx.get_parameter_source(param_name) == ParameterSource.DEFAULT: + params_assigned_from_default.add(param_name) + if ctx.parent: assign_params(ctx.parent, params_assigned_from_default) @@ -64,7 +119,9 @@ def assign_params(ctx, params_assigned_from_default): user_config_param_value = getattr(user_config, param_assigned_from_default, None) if user_config_param_value is not None: object.__setattr__( - self, param_assigned_from_default.upper(), user_config_param_value + self, + param_assigned_from_default.upper(), + convert_config(param_assigned_from_default, user_config_param_value), ) param_assigned_from_default_copy.remove(param_assigned_from_default) params_assigned_from_default = param_assigned_from_default_copy @@ -73,6 +130,26 @@ def assign_params(ctx, params_assigned_from_default): object.__setattr__(self, "WHICH", invoked_subcommand_name or ctx.info_name) object.__setattr__(self, "MP_CONTEXT", get_context("spawn")) + # Default LOG_PATH from PROJECT_DIR, if available. + if getattr(self, "LOG_PATH", None) is None: + log_path = "logs" + project_dir = getattr(self, "PROJECT_DIR", None) + # If available, set LOG_PATH from log-path in dbt_project.yml + # Known limitations: + # 1. Using PartialProject here, so no jinja rendering of log-path. + # 2. Programmatic invocations of the cli via dbtRunner may pass a Project object directly, + # which is not being used here to extract log-path. + if project_dir: + try: + partial = PartialProject.from_project_root( + project_dir, verify_version=getattr(self, "VERSION_CHECK", True) + ) + log_path = str(partial.project_dict.get("log-path", log_path)) + except DbtProjectError: + pass + + object.__setattr__(self, "LOG_PATH", log_path) + # Support console DO NOT TRACK initiave object.__setattr__( self, diff --git a/core/dbt/cli/main.py b/core/dbt/cli/main.py index 956f6bde81a..109bd839cd1 100644 --- a/core/dbt/cli/main.py +++ b/core/dbt/cli/main.py @@ -490,7 +490,6 @@ def seed(ctx, **kwargs): ctx.obj["runtime_config"], ctx.obj["manifest"], ) - results = task.run() success = task.interpret_results(results) return results, success diff --git a/core/dbt/cli/option_types.py b/core/dbt/cli/option_types.py index 6149dd7d5ee..b5eea995624 100644 --- a/core/dbt/cli/option_types.py +++ b/core/dbt/cli/option_types.py @@ -27,6 +27,7 @@ class WarnErrorOptionsType(YAML): name = "WarnErrorOptionsType" def convert(self, value, param, ctx): + # this function is being used by param in click include_exclude = super().convert(value, param, ctx) return WarnErrorOptions( diff --git a/core/dbt/cli/params.py b/core/dbt/cli/params.py index c741ece1de0..afe128bb480 100644 --- a/core/dbt/cli/params.py +++ b/core/dbt/cli/params.py @@ -132,7 +132,7 @@ "--log-path", envvar="DBT_LOG_PATH", help="Configure the 'log-path'. Only applies this setting for the current run. Overrides the 'DBT_LOG_PATH' if it is set.", - default=lambda: Path.cwd() / "logs", + default=None, type=click.Path(resolve_path=True, path_type=Path), ) @@ -415,7 +415,7 @@ def _version_callback(ctx, _param, value): warn_error_options = click.option( "--warn-error-options", envvar="DBT_WARN_ERROR_OPTIONS", - default=None, + default="{}", help="""If dbt would normally warn, instead raise an exception based on include/exclude configuration. Examples include --select that selects nothing, deprecations, configurations with no associated models, invalid test configurations, and missing sources/refs in tests. This argument should be a YAML string, with keys 'include' or 'exclude'. eg. '{"include": "all", "exclude": ["NoNodesForSelectionCriteria"]}'""", type=WarnErrorOptionsType(), diff --git a/core/dbt/cli/requires.py b/core/dbt/cli/requires.py index 74ec80c986a..eb45374db35 100644 --- a/core/dbt/cli/requires.py +++ b/core/dbt/cli/requires.py @@ -1,4 +1,5 @@ from dbt.adapters.factory import adapter_management, register_adapter +from dbt.flags import set_flags from dbt.cli.flags import Flags from dbt.config import RuntimeConfig from dbt.config.runtime import load_project, load_profile @@ -21,6 +22,7 @@ def wrapper(*args, **kwargs): # Flags flags = Flags(ctx) ctx.obj["flags"] = flags + set_flags(flags) # Tracking initialize_from_flags(flags.ANONYMOUS_USAGE_STATS, flags.PROFILES_DIR) diff --git a/core/dbt/clients/jinja.py b/core/dbt/clients/jinja.py index e9dcb45017b..ecf668c11e5 100644 --- a/core/dbt/clients/jinja.py +++ b/core/dbt/clients/jinja.py @@ -40,7 +40,7 @@ UndefinedCompilationError, UndefinedMacroError, ) -from dbt import flags +from dbt.flags import get_flags from dbt.node_types import ModelLanguage @@ -99,8 +99,9 @@ def _compile(self, source, filename): If the value is 'write', also write the files to disk. WARNING: This can write a ton of data if you aren't careful. """ - if filename == "