Model Deprecation

core/dbt/config/renderer.py

@@ -1,6 +1,7 @@
 from typing import Dict, Any, Tuple, Optional, Union, Callable
 import re
 import os
+from datetime import date
 
 from dbt.clients.jinja import get_rendered, catch_jinja
 from dbt.constants import SECRET_ENV_PREFIX
@@ -33,10 +34,10 @@ def render_entry(self, value: Any, keypath: Keypath) -> Any:
         return self.render_value(value, keypath)
 
     def render_value(self, value: Any, keypath: Optional[Keypath] = None) -> Any:
-        # keypath is ignored.
-        # if it wasn't read as a string, ignore it
+        # keypath is ignored (and someone who knows should explain why here)
         if not isinstance(value, str):
-            return value
+            return value if not isinstance(value, date) else value.isoformat()
+
         try:
             with catch_jinja():
                 return get_rendered(value, self.context, native=True)

core/dbt/contracts/graph/nodes.py

@@ -1,4 +1,5 @@
 import os
+from datetime import datetime
 import time
 from dataclasses import dataclass, field
 from enum import Enum
@@ -685,6 +686,7 @@ class ModelNode(CompiledNode):
     constraints: List[ModelLevelConstraint] = field(default_factory=list)
     version: Optional[NodeVersion] = None
     latest_version: Optional[NodeVersion] = None
+    deprecation_date: Optional[datetime] = None
 
     @property
     def is_latest_version(self) -> bool:

core/dbt/contracts/graph/unparsed.py

@@ -1,3 +1,4 @@
+import datetime
 import re
 
 from dbt import deprecations
@@ -210,6 +211,7 @@ class UnparsedModelUpdate(UnparsedNodeUpdate):
     access: Optional[str] = None
     latest_version: Optional[NodeVersion] = None
     versions: Sequence[UnparsedVersion] = field(default_factory=list)
+    deprecation_date: Optional[datetime.datetime] = None
 
     def __post_init__(self):
         if self.latest_version:

core/dbt/events/types.proto

@@ -1174,6 +1174,49 @@ message UnpinnedRefNewVersionAvailableMsg {
     UnpinnedRefNewVersionAvailable data = 2;
 }
 
+// I065
+message DeprecatedModel {
+    string model_name = 1;
+    string model_version = 2;
+    string deprecation_date = 3;
+}
+
+message DeprecatedModelMsg {
+    EventInfo info = 1;
+    DeprecatedModel data = 2;
+}
+
+// I066
+message UpcomingReferenceDeprecation {
+    string model_name = 1;
+    string ref_model_package = 2;
+    string ref_model_name = 3;
+    string ref_model_version = 4;
+    string ref_model_latest_version = 5;
+    string ref_model_deprecation_date = 6;
+}
+
+message UpcomingReferenceDeprecationMsg {
+    EventInfo info = 1;
+    UpcomingReferenceDeprecation data = 2;
+}
+
+// I067
+message DeprecatedReference {
+    string model_name = 1;
+    string ref_model_package = 2;
+    string ref_model_name = 3;
+    string ref_model_version = 4;
+    string ref_model_latest_version = 5;
+    string ref_model_deprecation_date = 6;
+}
+
+message DeprecatedReferenceMsg {
+    EventInfo info = 1;
+    DeprecatedReference data = 2;
+}
+
+
 // M - Deps generation
 
 // M001

core/dbt/events/types.py

@@ -1146,6 +1146,30 @@ def message(self) -> str:
         return msg
 
 
+class DeprecatedModel(WarnLevel):
+    def code(self):
+        return "I065"
+
+    def message(self) -> str:
+        pass
+
+
+class UpcomingReferenceDeprecation(WarnLevel):
+    def code(self):
+        return "I066"
+
+    def message(self) -> str:
+        pass
+
+
+class DeprecatedReference(WarnLevel):
+    def code(self):
+        return "I067"
+
+    def message(self) -> str:
+        pass
+
+
 # =======================================================
 # M - Deps generation
 # =======================================================

core/dbt/parser/manifest.py

@@ -1,7 +1,7 @@
 from copy import deepcopy
 from dataclasses import dataclass
 from dataclasses import field
-from datetime import datetime
+import datetime
 import os
 import traceback
 from typing import (
@@ -22,6 +22,7 @@
 from dbt.events.base_types import EventLevel
 import json
 import pprint
+import msgpack
 
 import dbt.exceptions
 import dbt.tracking
@@ -125,6 +126,36 @@
 PERF_INFO_FILE_NAME = "perf_info.json"
 
 
+def extended_mashumaro_encoder(data):
+    return msgpack.packb(data, default=extended_msgpack_encoder, use_bin_type=True)
+
+
+def extended_msgpack_encoder(obj):
+    if type(obj) is datetime.date:
+        date_bytes = msgpack.ExtType(1, obj.isoformat().encode())
+        return date_bytes
+    elif type(obj) is datetime.datetime:
+        datetime_bytes = msgpack.ExtType(2, obj.isoformat().encode())
+        return datetime_bytes
+
+    return obj
+
+
+def extended_mashumuro_decoder(data):
+    return msgpack.unpackb(data, ext_hook=extended_msgpack_decoder, raw=False)
+
+
+def extended_msgpack_decoder(code, data):
+    if code == 1:
+        d = datetime.date.fromisoformat(data.decode())
+        return d
+    elif code == 2:
+        dt = datetime.datetiem.fromisoformat(data.decode())
+        return dt
+    else:
+        return msgpack.ExtType(code, data)
+
+
 class ReparseReason(StrEnum):
     version_mismatch = "01_version_mismatch"
     file_not_found = "02_file_not_found"
@@ -644,7 +675,7 @@ def write_manifest_for_partial_parse(self):
                     UnableToPartialParse(reason="saved manifest contained the wrong version")
                 )
                 self.manifest.metadata.dbt_version = __version__
-            manifest_msgpack = self.manifest.to_msgpack()
+            manifest_msgpack = self.manifest.to_msgpack(extended_mashumaro_encoder)
             make_directory(os.path.dirname(path))
             with open(path, "wb") as fp:
                 fp.write(manifest_msgpack)
@@ -867,14 +898,14 @@ def read_manifest_for_partial_parse(self) -> Optional[Manifest]:
             try:
                 with open(path, "rb") as fp:
                     manifest_mp = fp.read()
-                manifest: Manifest = Manifest.from_msgpack(manifest_mp)  # type: ignore
+                manifest: Manifest = Manifest.from_msgpack(manifest_mp, decoder=extended_mashumuro_decoder)  # type: ignore
                 # keep this check inside the try/except in case something about
                 # the file has changed in weird ways, perhaps due to being a
                 # different version of dbt
                 is_partial_parsable, reparse_reason = self.is_partial_parsable(manifest)
                 if is_partial_parsable:
                     # We don't want to have stale generated_at dates
-                    manifest.metadata.generated_at = datetime.utcnow()
+                    manifest.metadata.generated_at = datetime.datetime.utcnow()
                     # or invocation_ids
                     manifest.metadata.invocation_id = get_invocation_id()
                     return manifest

core/dbt/utils.py

@@ -185,7 +185,7 @@ def _deep_map_render(
     value: Any,
     keypath: Tuple[Union[str, int], ...],
 ) -> Any:
-    atomic_types: Tuple[Type[Any], ...] = (int, float, str, type(None), bool)
+    atomic_types: Tuple[Type[Any], ...] = (int, float, str, type(None), bool, datetime.date)
 
     ret: Any
 

schemas/dbt/manifest/v10.json

@@ -1977,6 +1977,13 @@
               "type": "null"
             }
           ]
+        },
+        "deprecation_date": {
+          "oneOf": [
+            {
+              "type": "string"
+            }
+          ]
         }
       },
       "additionalProperties": false,

third-party-stubs/msgpack/__init__.pyi

@@ -0,0 +1,99 @@
+from __future__ import annotations
+from typing import Any, Callable, Dict, List, Optional, Tuple
+from msgpack.exceptions import (
+    BufferFull,
+    ExtraData,
+    FormatError,
+    OutOfData,
+    PackException,
+    PackOverflowError,
+    PackValueError,
+    StackError,
+    UnpackException,
+    UnpackValueError,
+)
+from typing_extensions import Protocol
+from msgpack.fallback import Packer, Unpacker, unpackb
+from msgpack import exceptions
+from msgpack.ext import ExtType
+from msgpack import ext
+
+class _Stream(Protocol):
+    def read(self) -> bytes: ...
+
+class _FileLike(Protocol):
+    def read(self, n: int) -> bytes: ...
+
+def pack(
+    o: Any,
+    stream: _Stream,
+    default: Optional[Callable[[Any], Any]] = ...,
+    use_single_float: bool = ...,
+    autoreset: bool = ...,
+    use_bin_type: bool = ...,
+    strict_types: bool = ...,
+    datetime: bool = ...,
+    unicode_errors: Optional[str] = ...,
+) -> None: ...
+def packb(
+    o: Any,
+    default: Optional[Callable[[Any], Any]] = ...,
+    use_single_float: bool = ...,
+    autoreset: bool = ...,
+    use_bin_type: bool = ...,
+    strict_types: bool = ...,
+    datetime: bool = ...,
+    unicode_errors: Optional[str] = ...,
+) -> bytes: ...
+def unpack(
+    stream: _Stream,
+    file_like: Optional[_FileLike] = ...,
+    read_size: int = ...,
+    use_list: bool = ...,
+    raw: bool = ...,
+    timestamp: int = ...,
+    strict_map_key: bool = ...,
+    object_hook: Optional[Callable[[Dict[Any, Any]], Any]] = ...,
+    object_pairs_hook: Optional[Callable[[List[Tuple[Any, Any]]], Any]] = ...,
+    list_hook: Optional[Callable[[List[Any]], Any]] = ...,
+    unicode_errors: Optional[str] = ...,
+    max_buffer_size: int = ...,
+    ext_hook: Callable[[int, bytes], Any] = ...,
+    max_str_len: int = ...,
+    max_bin_len: int = ...,
+    max_array_len: int = ...,
+    max_map_len: int = ...,
+    max_ext_len: int = ...,
+) -> Any: ...
+
+load = unpack
+loads = unpackb
+
+dump = pack
+dumps = packb
+
+__all__ = [
+    "BufferFull",
+    "ExtType",
+    "ExtraData",
+    "FormatError",
+    "OutOfData",
+    "PackException",
+    "PackOverflowError",
+    "PackValueError",
+    "Packer",
+    "StackError",
+    "UnpackException",
+    "UnpackValueError",
+    "Unpacker",
+    "dump",
+    "dumps",
+    "exceptions",
+    "ext",
+    "load",
+    "loads",
+    "pack",
+    "packb",
+    "unpack",
+    "unpackb",
+]

third-party-stubs/msgpack/_version.pyi

@@ -0,0 +1,3 @@
+from typing import Tuple
+
+version: Tuple[int, int, int]

third-party-stubs/msgpack/exceptions.pyi

@@ -0,0 +1,16 @@
+from typing import Any
+
+class UnpackException(Exception): ...
+class BufferFull(UnpackException): ...
+class OutOfData(UnpackException): ...
+class FormatError(ValueError, UnpackException): ...
+class StackError(ValueError, UnpackException): ...
+
+UnpackValueError = ValueError
+
+class ExtraData(UnpackValueError):
+    def __init__(self, unpacked: Any, exta: Any) -> None: ...
+
+PackException = Exception
+PackValueError = ValueError
+PackOverflowError = OverflowError

third-party-stubs/msgpack/ext.pyi

@@ -0,0 +1,28 @@
+from __future__ import annotations
+from typing import NamedTuple
+import datetime
+
+class _ExtType(NamedTuple):
+    code: int
+    data: bytes
+
+class ExtType(_ExtType): ...
+
+class TimeStamp:
+    def __init__(self, seconds: int, nanoseconds: int = ...) -> None: ...
+    def __eq__(self, o: object) -> bool: ...
+    def __ne__(self, o: object) -> bool: ...
+    @staticmethod
+    def from_bytes(b: bytes) -> TimeStamp: ...
+    @staticmethod
+    def to_bytes(self) -> bytes: ...
+    @staticmethod
+    def from_unix(self, unix_sec: float) -> TimeStamp: ...
+    def to_unix(self) -> float: ...
+    @staticmethod
+    def from_unix_nano(unix_ns: int) -> TimeStamp: ...
+    @staticmethod
+    def to_unix_nano(self) -> int: ...
+    def to_datetime(self) -> datetime.datetime: ...
+    @staticmethod
+    def from_datetime(dt: datetime.datetime) -> TimeStamp: ...