first pass: external nodes

core/dbt/cli/main.py

@@ -76,7 +76,6 @@ def invoke(self, args: List[str], **kwargs) -> dbtRunnerResult:
             dbt_ctx.obj = {
                 "manifest": self.manifest,
                 "callbacks": self.callbacks,
-                "_publications": kwargs.get("publications"),
             }
 
             for key, value in kwargs.items():

core/dbt/cli/requires.py

@@ -23,6 +23,7 @@
 from dbt.profiler import profiler
 from dbt.tracking import active_user, initialize_from_flags, track_run
 from dbt.utils import cast_dict_to_dict_of_strings
+from dbt.plugins import setup_plugin_manager, get_plugin_manager
 
 from click import Context
 from functools import update_wrapper
@@ -68,6 +69,9 @@ def wrapper(*args, **kwargs):
         # Adapter management
         ctx.with_resource(adapter_management())
 
+        # Plugins
+        setup_plugin_manager()
+
         return func(*args, **kwargs)
 
     return update_wrapper(wrapper, func)
@@ -242,12 +246,15 @@ def wrapper(*args, **kwargs):
                 manifest = ManifestLoader.get_full_manifest(
                     runtime_config,
                     write_perf_info=write_perf_info,
-                    publications=ctx.obj.get("_publications"),
                 )
 
                 ctx.obj["manifest"] = manifest
                 if write and ctx.obj["flags"].write_json:
                     write_manifest(manifest, ctx.obj["runtime_config"].project_target_path)
+                    pm = get_plugin_manager()
+                    artifacts = pm.get_external_artifacts(manifest, runtime_config)
+                    for path, artifact in artifacts.items():
+                        artifact.write(path)
 
             return func(*args, **kwargs)
 

core/dbt/contracts/graph/manifest.py

@@ -22,8 +22,6 @@
 from typing_extensions import Protocol
 from uuid import UUID
 
-from dbt.contracts.publication import PublicationConfig
-
 from dbt.contracts.graph.nodes import (
     BaseNode,
     Documentation,
@@ -700,7 +698,6 @@ class Manifest(MacroMethods, DataClassMessagePackMixin, dbtClassMixin):
     source_patches: MutableMapping[SourceKey, SourcePatch] = field(default_factory=dict)
     disabled: MutableMapping[str, List[GraphMemberNode]] = field(default_factory=dict)
     env_vars: MutableMapping[str, str] = field(default_factory=dict)
-    publications: MutableMapping[str, PublicationConfig] = field(default_factory=dict)
     semantic_models: MutableMapping[str, SemanticModel] = field(default_factory=dict)
 
     _doc_lookup: Optional[DocLookup] = field(
@@ -851,7 +848,6 @@ def deepcopy(self):
             disabled={k: _deepcopy(v) for k, v in self.disabled.items()},
             files={k: _deepcopy(v) for k, v in self.files.items()},
             state_check=_deepcopy(self.state_check),
-            publications={k: _deepcopy(v) for k, v in self.publications.items()},
             semantic_models={k: _deepcopy(v) for k, v in self.semantic_models.items()},
         )
         copy.build_flat_graph()
@@ -1297,7 +1293,6 @@ def __reduce_ex__(self, protocol):
             self.source_patches,
             self.disabled,
             self.env_vars,
-            self.publications,
             self.semantic_models,
             self._doc_lookup,
             self._source_lookup,

core/dbt/contracts/publication.py

@@ -1,16 +1,11 @@
-from typing import Any, Dict, List, Optional
-from datetime import datetime
+from typing import Any, Dict, List
 
 
 from dataclasses import dataclass, field
 
 from dbt.contracts.util import (
     AdditionalPropertiesMixin,
-    ArtifactMixin,
-    BaseArtifactMetadata,
-    schema_version,
 )
-from dbt.contracts.graph.unparsed import NodeVersion
 from dbt.dataclass_schema import dbtClassMixin, ExtensibleDbtClassMixin
 
 
@@ -23,66 +18,3 @@ class ProjectDependency(AdditionalPropertiesMixin, ExtensibleDbtClassMixin):
 @dataclass
 class ProjectDependencies(dbtClassMixin):
     projects: List[ProjectDependency] = field(default_factory=list)
-
-
-@dataclass
-class PublicationMetadata(BaseArtifactMetadata):
-    dbt_schema_version: str = field(
-        default_factory=lambda: str(PublicationArtifact.dbt_schema_version)
-    )
-    adapter_type: Optional[str] = None
-    quoting: Dict[str, Any] = field(default_factory=dict)
-
-
-@dataclass
-class PublicModel(dbtClassMixin):
-    """Used to represent cross-project models"""
-
-    name: str
-    package_name: str
-    unique_id: str
-    relation_name: str
-    identifier: str
-    schema: str
-    database: Optional[str] = None
-    version: Optional[NodeVersion] = None
-    latest_version: Optional[NodeVersion] = None
-    # list of model unique_ids
-    public_node_dependencies: List[str] = field(default_factory=list)
-    generated_at: datetime = field(default_factory=datetime.utcnow)
-    deprecation_date: Optional[datetime] = None
-
-
-@dataclass
-class PublicationMandatory:
-    project_name: str
-
-
-@dataclass
-@schema_version("publication", 1)
-class PublicationArtifact(ArtifactMixin, PublicationMandatory):
-    public_models: Dict[str, PublicModel] = field(default_factory=dict)
-    metadata: PublicationMetadata = field(default_factory=PublicationMetadata)
-    # list of project name strings
-    dependencies: List[str] = field(default_factory=list)
-
-
-@dataclass
-class PublicationConfig(ArtifactMixin, PublicationMandatory):
-    """This is for the part of the publication artifact which is stored in
-    the internal manifest. The public_nodes are stored separately in the manifest,
-    and just the unique_ids of the public models are stored here."""
-
-    metadata: PublicationMetadata = field(default_factory=PublicationMetadata)
-    # list of project name strings
-    dependencies: List[str] = field(default_factory=list)
-    public_node_ids: List[str] = field(default_factory=list)
-
-    @classmethod
-    def from_publication(cls, publication: PublicationArtifact):
-        return cls(
-            project_name=publication.project_name,
-            metadata=publication.metadata,
-            dependencies=publication.dependencies,
-            public_node_ids=list(publication.public_models.keys()),
-        )

core/dbt/events/types.proto

@@ -1517,18 +1517,6 @@ message NoNodesForSelectionCriteriaMsg {
     NoNodesForSelectionCriteria data = 2;
 }
 
-// P - Artifacts
-
-// P001
-message PublicationArtifactAvailable {
-    google.protobuf.Struct pub_artifact = 1;
-}
-
-message PublicationArtifactAvailableMsg {
-    EventInfo info = 1;
-    PublicationArtifactAvailable data = 2;
-}
-
 // Q - Node execution
 
 // Q001

core/dbt/events/types.py

@@ -1468,19 +1468,6 @@ def message(self) -> str:
 # =======================================================
 
 
-class PublicationArtifactAvailable(DebugLevel):
-    def code(self):
-        return "P001"
-
-    def message(self) -> str:
-        return "Publication artifact available"
-
-
-# =======================================================
-# Q - Node execution
-# =======================================================
-
-
 class RunningOperationCaughtError(ErrorLevel):
     def code(self):
         return "Q001"

core/dbt/exceptions.py

@@ -297,6 +297,11 @@ def type(self):
         return "Parsing"
 
 
+class dbtPluginError(DbtRuntimeError):
+    CODE = 10020
+    MESSAGE = "Plugin Error"
+
+
 # TODO: this isn't raised in the core codebase.  Is it raised elsewhere?
 class JSONValidationError(DbtValidationError):
     def __init__(self, typename, errors):
@@ -405,19 +410,6 @@ class DbtProfileError(DbtConfigError):
     pass
 
 
-class PublicationConfigNotFound(DbtConfigError):
-    def __init__(self, project=None, file_name=None):
-        self.project = project
-        msg = self.message()
-        super().__init__(msg, project=project)
-
-    def message(self):
-        return (
-            f"A dependency on project {self.project} was specified, "
-            f"but a publication for {self.project} was not found."
-        )
-
-
 class SemverError(Exception):
     def __init__(self, msg: Optional[str] = None):
         self.msg = msg
@@ -910,19 +902,6 @@ def get_message(self) -> str:
         return msg
 
 
-class ProjectDependencyCycleError(ParsingError):
-    def __init__(self, pub_project_name, project_name):
-        self.pub_project_name = pub_project_name
-        self.project_name = project_name
-        super().__init__(msg=self.get_message())
-
-    def get_message(self) -> str:
-        return (
-            f"A project dependency cycle has been detected. The current project {self.project_name} "
-            f"depends on {self.pub_project_name} which also depends on the current project."
-        )
-
-
 class MacroArgTypeError(CompilationError):
     def __init__(self, method_name: str, arg_name: str, got_value: Any, expected_type):
         self.method_name = method_name