Skip to content

Commit 4012930

Browse files
authored
ENG-415 bulk upsert_content functions for sync (#202)
1 parent 767484c commit 4012930

File tree

9 files changed

+923
-32
lines changed

9 files changed

+923
-32
lines changed
Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
# A few examples of using upsert_content / upsert_document etc.
2+
3+
In general, for external references, you can either embed it inline, reference it by `X_local_id` (where local means the platform) or use the db_ids as in the tables.
4+
5+
```typescript
6+
import type { LocalDocumentDataInput, LocalContentDataInput } from '@repo/database/input_types.ts';
7+
8+
const docs: LocalDocumentDataInput[] = [{ "source_local_id": "page1_uid", "created": "2000/01/01", "last_modified": "2001/01/02", author_local_id:"sR22zZ470dNPkIf9PpjQXXdTBjG2"}];
9+
const contents: LocalContentDataInput[] = [
10+
{
11+
"author_local_id": "sR22zZ470dNPkIf9PpjQXXdTBjG2",
12+
"author_inline": {
13+
"account_local_id": "sR22zZ470dNPkIf9PpjQXXdTBjG2",
14+
"name": "maparent"
15+
},
16+
"document_inline": {
17+
"source_local_id": "page1_uid",
18+
"created": "2000/01/01",
19+
"last_modified": "2001/01/02",
20+
"author_local_id": "sR22zZ470dNPkIf9PpjQXXdTBjG2"
21+
},
22+
"source_local_id": "a_roam_uid",
23+
"scale": "document",
24+
"created": "2000/01/01",
25+
"last_modified": "2001/01/02",
26+
"text": "Some text"
27+
},
28+
{ "author_local_id": "sR22zZ470dNPkIf9PpjQXXdTBjG2", "document_local_id":"page1_uid", "source_local_id": "a_roam_uid2", "scale":"document", "created": "2000/01/02", "last_modified": "2001/01/03", "part_of_local_id":"a_roam_uid", "text": "Some subtext" },
29+
{
30+
"author_local_id": "sR22zZ470dNPkIf9PpjQXXdTBjG2", "document_inline": docs[0], "source_local_id": "a_roam_uid3", "scale": "document", "created": "2000/01/02", "last_modified": "2001/01/03", "part_of_local_id": "a_roam_uid2", "text": "Some subsubtext", "embedding_inline": {
31+
"model":"openai_text_embedding_3_small_1536", "vector":[1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
32+
} }
33+
];
34+
35+
// optional preliminary step: upsert documents.
36+
{
37+
const { data, error } = await client.rpc("upsert_documents", { v_space_id: 12, data: docs as Json });
38+
if (error) console.error(error);
39+
console.log(data);
40+
}
41+
42+
// upsert content
43+
{
44+
const { data, error } = await client.rpc("upsert_content", { v_space_id: 12, data: contents as Json, v_creator_id: 63 });
45+
if (error) console.error(error);
46+
console.log(data);
47+
}
48+
49+
// more compact version if all content is known to also be a page
50+
const page_contents: LocalContentDataInput[] = [
51+
{ "author_local_id": "sR22zZ470dNPkIf9PpjQXXdTBjG2", "source_local_id": "a_page_uid2", "created": "2000/01/02", "last_modified": "2001/01/03", "text": "Some other page" },
52+
];
53+
54+
{
55+
const { data, error } = await client.rpc("upsert_content", { v_space_id: 12, data: page_contents as Json, v_creator_id: 63, content_as_document: true });
56+
if (error) console.error(error);
57+
console.log(data);
58+
}
59+
60+
```

packages/database/input_types.ts

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
import { Database, TablesInsert } from "./types.gen.ts";
2+
3+
export type LocalDocumentDataInput = Partial<
4+
Omit<Database['public']["CompositeTypes"]["document_local_input"], "author_inline">
5+
& { author_inline: Partial<TablesInsert<"PlatformAccount">> }>;
6+
export type LocalContentDataInput = Partial<
7+
Omit<Database['public']["CompositeTypes"]["content_local_input"], "document_inline" | "author_inline" | "creator_inline">
8+
& {
9+
document_inline: LocalDocumentDataInput,
10+
author_inline: Partial<TablesInsert<"PlatformAccount">>,
11+
creator_inline: Partial<TablesInsert<"PlatformAccount">>
12+
}>;
13+
export type LocalConceptDataInput = Partial<Database['public']["CompositeTypes"]["concept_local_input"]>;

packages/database/schema.puml

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -74,12 +74,18 @@ class "AgentIdentifier" [[{An identifying attribute associated with an agent, ca
7474
"Concept" --> "0..*" "PlatformAccount" : "contributors"
7575
"Content" --> "1" "PlatformAccount" : "creator"
7676
"Content" --> "1" "Document" : "document"
77-
class "ContentEmbedding" [[{An embedding for a piece of content.}]] {
77+
abstract "ContentEmbedding" [[{An embedding for a piece of content.}]] {
7878
{field} model : EmbeddingName
7979
{field} vector : vector
8080
{field} obsolete : boolean
8181
}
82+
class "ContentEmbedding_openai_text_embedding_3_small_1536" [[{The table for the openai text_embedding_3_small model (1536 dimensions)}]] {
83+
{field} model(i) : EmbeddingName
84+
{field} vector(i) : vector
85+
{field} obsolete(i) : boolean
86+
}
8287
"ContentEmbedding" --> "1" "Content" : "target"
88+
"ContentEmbedding" ^-- "ContentEmbedding_openai_text_embedding_3_small_1536"
8389
"Content" --> "0..1" "Content" : "part_of"
8490
"Concept" --> "0..1" "Content" : "represented_by"
8591
class "ConceptSchema" [[{A Concept that describes a schema (type) for other concepts}]] {

packages/database/schema.svg

Lines changed: 1 addition & 1 deletion
Loading

packages/database/schema.yaml

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -280,7 +280,7 @@ classes:
280280
# - abstract
281281
ContentEmbedding:
282282
description: An embedding for a piece of content.
283-
# abstract: true
283+
abstract: true
284284
attributes:
285285
target:
286286
range: Content
@@ -299,6 +299,9 @@ classes:
299299
description: Whether this embedding is obsolete (becauses the Content was modified)
300300
range: boolean
301301
ifabsent: 'false'
302+
ContentEmbedding_openai_text_embedding_3_small_1536:
303+
is_a: ContentEmbedding
304+
description: The table for the openai text_embedding_3_small model (1536 dimensions)
302305
Concept:
303306
description: An abstract concept, claim or relation
304307
slots:

0 commit comments

Comments
 (0)