ENG-415 bulk upsert_content functions for sync (#202)

Author: maparent

packages/database/doc/upsert_content.md

@@ -0,0 +1,60 @@
+# A few examples of using upsert_content / upsert_document etc.
+
+In general, for external references, you can either embed it inline, reference it by `X_local_id` (where local means the platform) or use the db_ids as in the tables.
+
+```typescript
+import type { LocalDocumentDataInput, LocalContentDataInput } from '@repo/database/input_types.ts';
+
+const docs: LocalDocumentDataInput[] = [{ "source_local_id": "page1_uid", "created": "2000/01/01", "last_modified": "2001/01/02", author_local_id:"sR22zZ470dNPkIf9PpjQXXdTBjG2"}];
+const contents: LocalContentDataInput[] = [
+  {
+    "author_local_id": "sR22zZ470dNPkIf9PpjQXXdTBjG2",
+    "author_inline": {
+      "account_local_id": "sR22zZ470dNPkIf9PpjQXXdTBjG2",
+      "name": "maparent"
+    },
+    "document_inline": {
+      "source_local_id": "page1_uid",
+      "created": "2000/01/01",
+      "last_modified": "2001/01/02",
+      "author_local_id": "sR22zZ470dNPkIf9PpjQXXdTBjG2"
+    },
+    "source_local_id": "a_roam_uid",
+    "scale": "document",
+    "created": "2000/01/01",
+    "last_modified": "2001/01/02",
+    "text": "Some text"
+  },
+  { "author_local_id": "sR22zZ470dNPkIf9PpjQXXdTBjG2", "document_local_id":"page1_uid", "source_local_id": "a_roam_uid2", "scale":"document", "created": "2000/01/02", "last_modified": "2001/01/03", "part_of_local_id":"a_roam_uid", "text": "Some subtext" },
+  {
+    "author_local_id": "sR22zZ470dNPkIf9PpjQXXdTBjG2", "document_inline": docs[0], "source_local_id": "a_roam_uid3", "scale": "document", "created": "2000/01/02", "last_modified": "2001/01/03", "part_of_local_id": "a_roam_uid2", "text": "Some subsubtext", "embedding_inline": {
+    "model":"openai_text_embedding_3_small_1536", "vector":[1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
+  } }
+];
+
+// optional preliminary step: upsert documents.
+{
+  const { data, error } = await client.rpc("upsert_documents", { v_space_id: 12, data: docs as Json });
+  if (error) console.error(error);
+  console.log(data);
+}
+
+// upsert content
+{
+  const { data, error } = await client.rpc("upsert_content", { v_space_id: 12, data: contents as Json, v_creator_id: 63 });
+  if (error) console.error(error);
+  console.log(data);
+}
+
+// more compact version if all content is known to also be a page
+const page_contents: LocalContentDataInput[] = [
+  { "author_local_id": "sR22zZ470dNPkIf9PpjQXXdTBjG2", "source_local_id": "a_page_uid2", "created": "2000/01/02", "last_modified": "2001/01/03", "text": "Some other page" },
+];
+
+{
+  const { data, error } = await client.rpc("upsert_content", { v_space_id: 12, data: page_contents as Json, v_creator_id: 63, content_as_document: true });
+  if (error) console.error(error);
+  console.log(data);
+}
+
+```

packages/database/input_types.ts

@@ -0,0 +1,13 @@
+import { Database, TablesInsert } from "./types.gen.ts";
+
+export type LocalDocumentDataInput = Partial<
+  Omit<Database['public']["CompositeTypes"]["document_local_input"], "author_inline">
+  & { author_inline: Partial<TablesInsert<"PlatformAccount">> }>;
+export type LocalContentDataInput = Partial<
+  Omit<Database['public']["CompositeTypes"]["content_local_input"], "document_inline" | "author_inline" | "creator_inline">
+  & {
+    document_inline: LocalDocumentDataInput,
+    author_inline: Partial<TablesInsert<"PlatformAccount">>,
+    creator_inline: Partial<TablesInsert<"PlatformAccount">>
+  }>;
+export type LocalConceptDataInput = Partial<Database['public']["CompositeTypes"]["concept_local_input"]>;

packages/database/schema.puml

@@ -74,12 +74,18 @@ class "AgentIdentifier" [[{An identifying attribute associated with an agent, ca
 "Concept" --> "0..*" "PlatformAccount" : "contributors"
 "Content" --> "1" "PlatformAccount" : "creator"
 "Content" --> "1" "Document" : "document"
-class "ContentEmbedding" [[{An embedding for a piece of content.}]] {
+abstract "ContentEmbedding" [[{An embedding for a piece of content.}]] {
     {field} model : EmbeddingName  
     {field} vector : vector  
     {field} obsolete : boolean  
 }
+class "ContentEmbedding_openai_text_embedding_3_small_1536" [[{The table for the openai text_embedding_3_small model (1536 dimensions)}]] {
+    {field} model(i) : EmbeddingName  
+    {field} vector(i) : vector  
+    {field} obsolete(i) : boolean  
+}
 "ContentEmbedding" --> "1" "Content" : "target"
+"ContentEmbedding" ^-- "ContentEmbedding_openai_text_embedding_3_small_1536"
 "Content" --> "0..1" "Content" : "part_of"
 "Concept" --> "0..1" "Content" : "represented_by"
 class "ConceptSchema" [[{A Concept that describes a schema (type) for other concepts}]] {

packages/database/schema.svg

@@ -280,7 +280,7 @@ classes:
   #     - abstract
   ContentEmbedding:
     description: An embedding for a piece of content.
-    # abstract: true
+    abstract: true
     attributes:
       target:
         range: Content
@@ -299,6 +299,9 @@ classes:
         description: Whether this embedding is obsolete (becauses the Content was modified)
         range: boolean
         ifabsent: 'false'
+  ContentEmbedding_openai_text_embedding_3_small_1536:
+    is_a: ContentEmbedding
+    description: The table for the openai text_embedding_3_small model (1536 dimensions)
   Concept:
     description: An abstract concept, claim or relation
     slots: