Loro source of truth

Author: joepio

AGENTS.md

@@ -84,15 +84,28 @@ Atomic Server is a graph database with real-time sync, built on **Loro CRDT** fo
 1. Commit arrives at `/commit`
 2. `apply_changes()` imports `loroUpdate` into resource's LoroDoc
 3. `import_update_with_diff()` computes add/remove atoms for search indexing
-4. `loro_value_to_atomic_value()` materializes Loro values to Atomic `Value` types
+4. `loro_value_to_atomic_value_tagged()` materializes Loro values to Atomic `Value` types, using the `datatypes` map
 5. Loro snapshot stored alongside PropVals for future merges
 
 ### Loro value serialization in the Map
 
-- Strings, numbers, booleans → stored directly
-- `ResourceArray` → JSON string `["url1", "url2"]`
-- `AtomicUrl` → plain string
-- `loro_value_to_atomic_value()` parses back: strings starting with `[` → ResourceArray, `{` → NestedResource
+The LoroDoc has two sibling root maps:
+
+- **`properties`** — `property URL → value`. Loro primitives stored directly
+  (strings, numbers, booleans); arrays as native `LoroList`s; objects as JSON strings.
+- **`datatypes`** — sparse `property URL → tag`, recording the datatype only
+  where a bare primitive is ambiguous in a load-bearing way. Tags: `atomicUrl`,
+  `resourceArray`, `jsonArray`, `json`, `resource`. Scalars and plain/cosmetic
+  strings carry no entry. Written by `set_property` (Rust) and
+  `Resource.writeDatatypeTags` at sign time (TS).
+
+Materialization prefers the tag: `loro_value_to_atomic_value_tagged()` recovers
+the exact `Value` variant from it. Untagged values fall back to the
+`loro_value_to_atomic_value()` heuristic (URL-shaped strings → `AtomicUrl`,
+`{...}` → `NestedResource`), kept for legacy / not-yet-tagged docs. Cosmetic
+datatypes (`markdown`/`slug`/`date`/`uri`, `timestamp`) are deliberately not
+tagged — they collapse to `string`/`integer`; the Property's `datatype` stays
+authoritative. See `planning/loro-source-of-truth.md`.
 
 ### Critical: always build on existing state
 

browser/lib/src/datatypes.test.ts

@@ -1,6 +1,6 @@
 import { describe, it } from 'vitest';
 
-import { Datatype, urls, validateDatatype } from './index.js';
+import { Datatype, datatypeTag, urls, validateDatatype } from './index.js';
 
 describe('Datatypes', () => {
   it('throws errors when datatypes dont match values', async ({ expect }) => {
@@ -65,3 +65,29 @@ describe('Datatypes', () => {
     expect(() => validateDatatype(int, Datatype.RESOURCEARRAY)).to.throw();
   });
 });
+
+describe('datatypeTag', () => {
+  it('tags load-bearing datatypes and collapses the rest', ({ expect }) => {
+    // Load-bearing: references and arrays get a tag.
+    expect(datatypeTag(Datatype.ATOMIC_URL, 'https://example.com/x')).toBe(
+      'atomicUrl',
+    );
+    expect(datatypeTag(Datatype.RESOURCEARRAY, [])).toBe('resourceArray');
+    expect(datatypeTag(Datatype.RESOURCEARRAY, ['https://example.com/x'])).toBe(
+      'resourceArray',
+    );
+    expect(datatypeTag(Datatype.JSON, '{"a":1}')).toBe('json');
+
+    // A nested resource (object stored as a JSON string under an atomicURL
+    // property) stays untagged — the server heuristic handles `{...}`.
+    expect(datatypeTag(Datatype.ATOMIC_URL, '{"a":1}')).toBeUndefined();
+
+    // Cosmetic / scalar datatypes collapse — no tag.
+    expect(datatypeTag(Datatype.STRING, 'hello')).toBeUndefined();
+    expect(datatypeTag(Datatype.MARKDOWN, '# heading')).toBeUndefined();
+    expect(datatypeTag(Datatype.SLUG, 'a-slug')).toBeUndefined();
+    expect(datatypeTag(Datatype.DATE, '2026-05-21')).toBeUndefined();
+    expect(datatypeTag(Datatype.INTEGER, 5)).toBeUndefined();
+    expect(datatypeTag(Datatype.BOOLEAN, true)).toBeUndefined();
+  });
+});

browser/lib/src/datatypes.ts

@@ -42,6 +42,37 @@ export const datatypeFromUrl = (url: string): Datatype => {
   return Datatype.UNKNOWN;
 };
 
+/**
+ * The sibling `datatypes` Loro-map tag for a property, mirroring the Rust
+ * `datatype_tag` (`lib/src/loro.rs`). Lets the server materialize a value to
+ * the exact `Value` variant instead of guessing from the primitive.
+ *
+ * Only the load-bearing reference / array distinctions are tagged; cosmetic
+ * datatypes (string/markdown/slug/date/uri) and scalars collapse and carry no
+ * tag. A nested resource (an object stored as a JSON string under an
+ * `atomicURL` property) is left untagged for the server's heuristic — see
+ * `planning/loro-source-of-truth.md`.
+ *
+ * `loroValue` is the value as stored in the Loro `properties` map.
+ */
+export const datatypeTag = (
+  datatype: string,
+  loroValue: unknown,
+): string | undefined => {
+  switch (datatype) {
+    case Datatype.ATOMIC_URL:
+      return typeof loroValue === 'string' && !loroValue.startsWith('{')
+        ? 'atomicUrl'
+        : undefined;
+    case Datatype.RESOURCEARRAY:
+      return 'resourceArray';
+    case Datatype.JSON:
+      return 'json';
+    default:
+      return undefined;
+  }
+};
+
 const slug_regex = /^[a-z0-9]+(?:-[a-z0-9]+)*$/;
 // https://stackoverflow.com/a/22061879/2502163
 const dateStringRegex = /^\d{4}-(0[1-9]|1[012])-(0[1-9]|[12][0-9]|3[01])$/;

browser/lib/src/resource.ts

@@ -12,7 +12,7 @@ import {
   commitIdOf,
   commitToJsonADObject,
 } from './commit.js';
-import { validateDatatype } from './datatypes.js';
+import { validateDatatype, datatypeTag } from './datatypes.js';
 import { isUnauthorized } from './error.js';
 import { commits } from './ontologies/commits.js';
 import { core } from './ontologies/core.js';
@@ -537,6 +537,45 @@ export class Resource<C extends OptionalClass = any> {
     this._cache = nextCache;
   }
 
+  /**
+   * Phase 1 (loro-source-of-truth): populate the sibling `datatypes` Loro map
+   * so the server recovers reference / array `Value` variants exactly instead
+   * of guessing. The map is sparse — only load-bearing datatypes get a tag;
+   * see {@link datatypeTag}. Idempotent: re-signing rewrites nothing.
+   *
+   * Cache-only — never triggers a fetch. A property whose definition is not
+   * already cached is left untagged; the server then falls back to its
+   * materialization heuristic, exactly as before this map existed. Properties
+   * edited via `set()` with validation are always cached by the time we sign.
+   */
+  private writeDatatypeTags(): void {
+    const doc = this._loroDoc;
+
+    if (!doc) {
+      return;
+    }
+
+    const props = doc.getMap('properties').toJSON() as Record<string, unknown>;
+    const datatypesMap = doc.getMap('datatypes');
+
+    for (const [prop, loroValue] of Object.entries(props)) {
+      const datatype = this.store?.resources
+        .get(prop)
+        ?.get(core.properties.datatype)
+        ?.toString();
+
+      if (datatype === undefined) {
+        continue;
+      }
+
+      const tag = datatypeTag(datatype, loroValue);
+
+      if (tag !== undefined && datatypesMap.get(prop) !== tag) {
+        datatypesMap.set(prop, tag);
+      }
+    }
+  }
+
   private resetLoroState(): void {
     this._loroDoc = undefined;
     this._loroMap = undefined;
@@ -1396,11 +1435,7 @@ export class Resource<C extends OptionalClass = any> {
       list = map.setContainer(propUrl, new LoroList());
     }
 
-    if (
-      item !== null &&
-      typeof item === 'object' &&
-      !Array.isArray(item)
-    ) {
+    if (item !== null && typeof item === 'object' && !Array.isArray(item)) {
       const itemMap = list.pushContainer(new LoroMap());
       this.writeJsonToLoroMap(itemMap, item as JSONObject);
     } else {
@@ -1438,10 +1473,7 @@ export class Resource<C extends OptionalClass = any> {
     }
   }
 
-  private writeJsonToLoroList(
-    list: LoroList,
-    arr: JSONValue[],
-  ): void {
+  private writeJsonToLoroList(list: LoroList, arr: JSONValue[]): void {
     const { LoroList, LoroMap } = LoroLoader.Loro;
 
     for (const item of arr) {
@@ -1501,6 +1533,12 @@ export class Resource<C extends OptionalClass = any> {
     this.rebuildCacheFromLoro();
     this._cacheDirty = false;
 
+    // Phase 1 (loro-source-of-truth): stamp the sibling `datatypes` map so
+    // the server materializes references/arrays exactly. Runs here — after
+    // every property is in the doc, before the snapshot export below — so it
+    // covers props set via `set()` and via cache hydration alike.
+    this.writeDatatypeTags();
+
     // Chain: use last locally-signed commit, or the server-known lastCommit.
     if (this._lastLocalSignature) {
       // Construct the full commit URL that the server will use.  This ensures

lib/src/db.rs

@@ -1483,16 +1483,28 @@ impl Db {
         Ok(())
     }
 
-    /// Recursively removes a resource and its children from the database
+    /// Recursively removes a resource and its children from the database.
+    /// `removed` collects the `pure_id()` of every deleted subject so the
+    /// caller can tombstone them after the transaction is applied.
     async fn recursive_remove(
         &self,
         subject: &Subject,
         transaction: &mut Transaction,
+        removed: &mut Vec<String>,
     ) -> AtomicResult<()> {
-        let subject_str = subject.to_string();
+        // Key by `pure_id()` — that is how resources and Loro snapshots are
+        // stored (`add_resource_tx`, `apply_commit`). Looking up by the raw
+        // `to_string()` (which may carry `?drive=` params) would miss the
+        // row entirely for DID subjects with a drive hint.
+        let subject_str = subject.pure_id();
         if let Ok(found) = self.get_propvals(&subject_str) {
             let resource = Resource::from_propvals(found, subject.clone());
             transaction.push(Operation::remove_resource(&subject_str));
+            // Remove the Loro snapshot in the same transaction. Without this
+            // the snapshot is orphaned in `Tree::LoroSnapshots` and leaks
+            // forever — only the WS/Iroh DESTROY path cleaned it before.
+            transaction.push(Operation::remove_loro_snapshot(&subject_str));
+            removed.push(subject_str.clone());
             let mut children = resource.get_children(self).await?;
             for child in children.iter_mut() {
                 // Notify subscribers so clients evict the cascade-deleted
@@ -1505,7 +1517,8 @@ impl Db {
                     source_id: None,
                 });
                 // Because the function is async we need to box it to use recursion.
-                Box::pin(self.recursive_remove(child.get_subject(), transaction)).await?;
+                Box::pin(self.recursive_remove(child.get_subject(), transaction, removed))
+                    .await?;
             }
             for (prop, val) in resource.get_propvals() {
                 let remove_atom = crate::Atom::new(subject.clone(), prop.clone(), val.clone());
@@ -2319,8 +2332,15 @@ impl Storelike for Db {
     #[instrument(skip_all)]
     async fn remove_resource(&self, subject: &Subject) -> AtomicResult<()> {
         let mut transaction = Transaction::new();
-        self.recursive_remove(subject, &mut transaction).await?;
+        let mut removed = Vec::new();
+        self.recursive_remove(subject, &mut transaction, &mut removed)
+            .await?;
         self.apply_transaction(&mut transaction)?;
+        // Tombstone every removed subject so bulk sync (Iroh / WS `SYNC`)
+        // does not resurrect them from a peer that still holds a stale copy.
+        for s in &removed {
+            crate::sync::tombstones::record_tombstone(self, s);
+        }
         // TODO: deletion sync — should create a signed destroy commit
         // and push it through the normal commit pipeline, not a raw DESTROY frame.
         Ok(())

lib/src/db/test.rs

@@ -1342,3 +1342,96 @@ async fn loro_non_property_container_survives_commit_roundtrip() {
          (this is what the WS GET handler serves to a second viewer)",
     );
 }
+
+/// A deleted resource must not leave its Loro snapshot orphaned in
+/// `Tree::LoroSnapshots`, and the subject must be tombstoned so bulk sync
+/// does not resurrect it. Regression test for Phase 0 of the
+/// `loro-source-of-truth` plan.
+#[tokio::test]
+#[timeout(30000)]
+async fn remove_resource_deletes_loro_snapshot() {
+    let store = Db::init_temp("orphan_snapshot").await.unwrap();
+    let drive = store.create_drive("test-drive").await.unwrap();
+    let did = store
+        .create_resource(
+            "https://atomicdata.dev/classes/Property",
+            &drive,
+            "age",
+            None,
+        )
+        .await
+        .unwrap();
+    let subject = Subject::from_raw(&did, store.get_base_domain().as_deref());
+    let pure_id = subject.pure_id();
+
+    assert!(
+        store
+            .kv
+            .get(Tree::LoroSnapshots, pure_id.as_bytes())
+            .unwrap()
+            .is_some(),
+        "a Loro snapshot should be persisted for a freshly created resource"
+    );
+
+    store.remove_resource(&subject).await.unwrap();
+
+    assert!(
+        store
+            .kv
+            .get(Tree::LoroSnapshots, pure_id.as_bytes())
+            .unwrap()
+            .is_none(),
+        "Loro snapshot was orphaned after remove_resource"
+    );
+    assert!(
+        crate::sync::tombstones::is_tombstoned(&store, &pure_id),
+        "removed subject should be tombstoned to prevent sync resurrection"
+    );
+}
+
+/// Deleting via a subject that carries a `?drive=` hint must still remove the
+/// snapshot — it is keyed by `pure_id()`. Regression test for the mis-keyed
+/// `apply_destroy` snapshot removal.
+#[tokio::test]
+#[timeout(30000)]
+async fn remove_resource_with_drive_hint_subject_deletes_snapshot() {
+    let store = Db::init_temp("orphan_snapshot_hint").await.unwrap();
+    let drive = store.create_drive("test-drive").await.unwrap();
+    let did = store
+        .create_resource(
+            "https://atomicdata.dev/classes/Property",
+            &drive,
+            "age",
+            None,
+        )
+        .await
+        .unwrap();
+    let subject = Subject::from_raw(&did, store.get_base_domain().as_deref());
+    let pure_id = subject.pure_id();
+    assert!(
+        store
+            .kv
+            .get(Tree::LoroSnapshots, pure_id.as_bytes())
+            .unwrap()
+            .is_some()
+    );
+
+    // Delete via a subject carrying a `?drive=` hint. The snapshot is keyed by
+    // pure_id(); the old raw-subject key would miss it.
+    let hinted = subject.clone().set_drive_hint(drive.clone());
+    assert_ne!(
+        hinted.to_string(),
+        pure_id,
+        "drive hint should make to_string() differ from pure_id()"
+    );
+    store.remove_resource(&hinted).await.unwrap();
+
+    assert!(
+        store
+            .kv
+            .get(Tree::LoroSnapshots, pure_id.as_bytes())
+            .unwrap()
+            .is_none(),
+        "snapshot orphaned when deleting via a drive-hinted subject"
+    );
+}

lib/src/db/trees.rs

@@ -121,6 +121,18 @@ impl Operation {
             val: None,
         }
     }
+
+    /// Remove a resource's Loro snapshot. `pure_id` must be the
+    /// [`crate::Subject::pure_id`] form — that is the key snapshots are
+    /// written under (see `apply_commit` / `add_resource_opts`).
+    pub fn remove_loro_snapshot(pure_id: &str) -> Self {
+        Operation {
+            tree: Tree::LoroSnapshots,
+            method: Method::Delete,
+            key: pure_id.as_bytes().to_vec(),
+            val: None,
+        }
+    }
 }
 
 /// A set of [Operation]s that should be executed atomically by the database.