From 33d9437dcb6eebed21dec02a1173805cc342d60e Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Wed, 26 Jul 2023 10:45:17 -0400 Subject: [PATCH 1/2] Initialize sqlparse lexer and tweak order of setting compilation fields --- .changes/unreleased/Fixes-20230726-104448.yaml | 6 ++++++ core/dbt/compilation.py | 18 ++++++++++++------ 2 files changed, 18 insertions(+), 6 deletions(-) create mode 100644 .changes/unreleased/Fixes-20230726-104448.yaml diff --git a/.changes/unreleased/Fixes-20230726-104448.yaml b/.changes/unreleased/Fixes-20230726-104448.yaml new file mode 100644 index 00000000000..0c8e55dd6b9 --- /dev/null +++ b/.changes/unreleased/Fixes-20230726-104448.yaml @@ -0,0 +1,6 @@ +kind: Fixes +body: Improve handling of CTE injection with ephemeral models +time: 2023-07-26T10:44:48.888451-04:00 +custom: + Author: gshank + Issue: "8213" diff --git a/core/dbt/compilation.py b/core/dbt/compilation.py index 41fd3e0da22..2854b1ddecc 100644 --- a/core/dbt/compilation.py +++ b/core/dbt/compilation.py @@ -4,7 +4,6 @@ import networkx as nx # type: ignore import os import pickle -import sqlparse from collections import defaultdict from typing import List, Dict, Any, Tuple, Optional @@ -36,6 +35,7 @@ from dbt.events.format import pluralize import dbt.tracking import dbt.task.list as list_task +import sqlparse graph_file_name = "graph.gpickle" @@ -378,16 +378,16 @@ def _recursively_prepend_ctes( _add_prepended_cte(prepended_ctes, InjectedCTE(id=cte.id, sql=sql)) - injected_sql = inject_ctes_into_sql( - model.compiled_code, - prepended_ctes, - ) # Check again before updating for multi-threading if not model.extra_ctes_injected: + injected_sql = inject_ctes_into_sql( + model.compiled_code, + prepended_ctes, + ) + model.extra_ctes_injected = True model._pre_injected_sql = model.compiled_code model.compiled_code = injected_sql model.extra_ctes = prepended_ctes - model.extra_ctes_injected = True # if model.extra_ctes is not set to prepended ctes, something went wrong return model, model.extra_ctes @@ -563,6 +563,12 @@ def inject_ctes_into_sql(sql: str, ctes: List[InjectedCTE]) -> str: if len(ctes) == 0: return sql + # Make sure Lexer for sqlparse 0.4.4 is initialized + from sqlparse.lexer import Lexer # type: ignore + + if hasattr(Lexer, "get_default_instance"): + Lexer.get_default_instance() + parsed_stmts = sqlparse.parse(sql) parsed = parsed_stmts[0] From 393cd906888c2879dcd29a2c49be09479ac18427 Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Wed, 26 Jul 2023 11:55:48 -0400 Subject: [PATCH 2/2] Move initialization of sqlparse lexer --- core/dbt/compilation.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/core/dbt/compilation.py b/core/dbt/compilation.py index 2854b1ddecc..959ad2516d1 100644 --- a/core/dbt/compilation.py +++ b/core/dbt/compilation.py @@ -523,6 +523,12 @@ def compile_node( the node's raw_code into compiled_code, and then calls the recursive method to "prepend" the ctes. """ + # Make sure Lexer for sqlparse 0.4.4 is initialized + from sqlparse.lexer import Lexer # type: ignore + + if hasattr(Lexer, "get_default_instance"): + Lexer.get_default_instance() + node = self._compile_code(node, manifest, extra_context) node, _ = self._recursively_prepend_ctes(node, manifest, extra_context) @@ -563,12 +569,6 @@ def inject_ctes_into_sql(sql: str, ctes: List[InjectedCTE]) -> str: if len(ctes) == 0: return sql - # Make sure Lexer for sqlparse 0.4.4 is initialized - from sqlparse.lexer import Lexer # type: ignore - - if hasattr(Lexer, "get_default_instance"): - Lexer.get_default_instance() - parsed_stmts = sqlparse.parse(sql) parsed = parsed_stmts[0]