Skip to content
Merged
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Merge remote-tracking branch 'origin/main' into HEAD
# Conflicts:
#	crates/gitlawb-node/src/db/mod.rs
  • Loading branch information
kevincodex1 committed Aug 31, 2026
commit 0bae56f5cac8f6defe0300d79616da1c3ed52c8b
223 changes: 223 additions & 0 deletions crates/gitlawb-node/src/db/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -150,6 +150,21 @@ fn validate_mirror_upstream_url(raw: &str) -> Result<reqwest::Url> {
Ok(url)
}

/// One row of a keyset page from [`Db::list_repos_page_for_scan`].
///
/// Carries the row's quarantine flag inline (so the IPFS scan needs no separate
/// whole-node quarantine query) and the RAW stored `created_at` text, which is
/// the first half of the keyset cursor. The raw text is kept because the keyset
/// comparison is a text comparison and re-serializing the parsed `DateTime` is
/// not guaranteed to reproduce the stored bytes — a cursor that differs from the
/// stored value by one character skips or repeats rows.
#[derive(Debug, Clone)]
pub struct ScanRepoRow {
pub repo: RepoRecord,
pub quarantined: bool,
pub created_at_key: String,
}

/// Per-rule replication mode for a visibility rule.
/// `A` hides existence entirely (only valid at whole-repo scope `/`).
/// `B` keeps object SHAs and the path visible but withholds content
Expand Down Expand Up @@ -1027,6 +1042,214 @@ const MIGRATIONS: &[Migration] = &[
"ALTER TABLE sync_queue ADD COLUMN IF NOT EXISTS attempted_at TEXT",
],
},
// The six pin-provenance migrations below were numbered 11-16 while this work was
// in flight and moved to 18-23 on merge, because 11 and 16 were claimed elsewhere.
// A database that ran an earlier commit of this branch therefore has schema_migrations
// rows for the old numbers. Those rows are orphans: the runner skips on `version`
// alone and never reads `name`, so nothing detects them, and the DDL below re-runs
// as a no-op against objects that already exist. Recreate any such database rather
// than upgrading it in place.
Migration {
version: 18,
name: "pinned_cids_cid_index",
stmts: &[
// GET /ipfs/{cid} resolves an incoming CID -> git oid via pinned_cids.cid
// (#173); index it so the per-request lookup is not a table scan. This is
// a NEW versioned migration (not appended to the applied v1 bundle) so a
// node already past v1 actually gets the index. Non-unique on purpose: cid
// is a function of raw content, so a UNIQUE index could reject a legitimate
// record_pinned_cid insert, and colliding rows serve byte-identical content.
"CREATE INDEX IF NOT EXISTS idx_pinned_cids_cid ON pinned_cids(cid)",
],
},
Migration {
version: 19,
name: "pinned_cids_repo_provenance",
stmts: &[
// Record the repository a pin came from so GET /ipfs/{cid} resolves a
// provenanced pin straight to its ONE source repo instead of scanning every
// repo (#173, jatmn round 2 — bounds the anonymous fan-out and removes the
// updated_at-ordering false-404). NEW versioned migration (never appended to
// the applied v1 pinned_cids table) so a node past v1 gets the column.
// Nullable: pins recorded before this migration have no provenance and fall
// back to the legacy repo scan; new pins carry repo_id and resolve to one
// repo. Indexed for the resolver's oid -> repo_id lookup.
"ALTER TABLE pinned_cids ADD COLUMN IF NOT EXISTS repo_id TEXT",
"CREATE INDEX IF NOT EXISTS idx_pinned_cids_repo_id ON pinned_cids(repo_id)",
],
},
Migration {
version: 20,
name: "pin_repo_sources",
stmts: &[
// F1 (#173, jatmn round 8): a shared object (a blob/tree/commit common to
// forks and mirrors) can be pinned from more than one repo. `pinned_cids`
// keeps only the FIRST pinner's `repo_id`, so a shared object first pinned
// from a private/quarantined repo 404s by CID even when a later PUBLIC repo
// also pinned it. Record EVERY pin-path source so `GET /ipfs/{cid}` can try
// each. NEW versioned migration (never appended to an applied block, INV-7).
// Bounded per object at insert time (MAX_PIN_SOURCES) so an adversary pushing
// one object from N repos cannot make resolution O(repos) (R2, INV-10).
"CREATE TABLE IF NOT EXISTS pin_repo_sources (
sha256_hex TEXT NOT NULL,
repo_id TEXT NOT NULL,
PRIMARY KEY (sha256_hex, repo_id)
)",
"CREATE INDEX IF NOT EXISTS idx_pin_repo_sources_sha ON pin_repo_sources(sha256_hex)",
],
},
Migration {
version: 21,
name: "pinned_cids_legacy_provider_cid",
stmts: &[
// R8 (#173, jatmn round 10): the opportunistic legacy provider-CID repair
// rewrites `pinned_cids.cid` from a stored PROVIDER CID (Kubo dag-pb /
// Pinata CIDv0) to the raw-content resolver key and stashes the OLD value
// here, so the rewrite is auditable and the row's legacy origin survives.
// Distinct from `pinata_cid` on purpose: `has_pinata_cid` gates the Pinata
// pin-skip, so parking a Kubo-legacy CID there would make Pinata forever
// skip re-pinning that object. NEW versioned migration (never appended to an
// applied block, INV-7) so a node past v13 actually gets the column.
// Nullable: only a repaired row sets it.
"ALTER TABLE pinned_cids ADD COLUMN IF NOT EXISTS legacy_provider_cid TEXT",
],
},
Migration {
version: 22,
name: "pinned_cids_sources_incomplete",
stmts: &[
// U3 (#173): `record_pin_source` is BEST EFFORT at every call site, so a
// non-empty, below-cap source set is not proof that every source was
// recorded. An object first pinned from a private repo and later pushed
// from a PUBLIC repo whose record failed keeps a set naming only the
// private source, and the resolver used to call that set complete and 404
// an object the public repo would serve. Record the miss DURABLY here so
// `GET /ipfs/{cid}` keeps the bounded scan fallback for exactly those
// objects. Not inferable from row counts or timestamps: neither can tell
// "no other source exists" from "a source failed to record", which is the
// whole distinction. NEW versioned migration (never appended to an applied
// block, INV-7). NOT NULL DEFAULT FALSE so every pre-existing row reads as
// complete and ordinary denials stay off the O(repos) path (INV-10).
"ALTER TABLE pinned_cids ADD COLUMN IF NOT EXISTS pin_sources_incomplete BOOLEAN NOT NULL DEFAULT FALSE",
],
},
Migration {
version: 23,
name: "pin_repair_sweep_cursor",
stmts: &[
// U4 (#173): the legacy provider-CID repair sweep walks `pinned_cids` in
// bounded batches over an ordered `sha256_hex` cursor. The cursor has to be
// DURABLE, or a restart rewinds the walk to the start of the table and an
// upgraded node with a large pin set never finishes repairing it. One row
// (`id = 1`, enforced by the CHECK) rather than a key-value table: there is
// exactly one sweep and no second consumer, and a real constraint beats a
// convention nobody can enforce. NEW versioned migration (never appended to
// an applied block, INV-7). No default row is inserted: an absent row is the
// "never swept" state, which the empty-string cursor start already means, so
// there is no first-run special case to get wrong.
"CREATE TABLE IF NOT EXISTS pin_repair_sweep (
id INTEGER NOT NULL PRIMARY KEY CHECK (id = 1),
cursor TEXT NOT NULL
)",
],
},
Migration {
version: 24,
name: "pin_source_failures",
stmts: &[
// #173 round 12 (jatmn): v22's `pin_sources_incomplete` is one boolean per
// OBJECT, so any successful source record cleared it, including one from a
// repo unrelated to the failure. The resolver then read the set as fully
// enumerated, dropped the scan fallback, and 404'd an anonymous caller whose
// only servable copy was the unrecorded public one. The missing source is a
// property of an (object, repo) PAIR, so it is stored as one.
//
// NEW versioned migration (never appended to an applied block, INV-7). A new
// table rather than a column on `pinned_cids`: the relation is many-per-object
// and `CREATE TABLE` takes no lock on the pin table a live node is reading.
"CREATE TABLE IF NOT EXISTS pin_source_failures (
sha256_hex TEXT NOT NULL,
repo_id TEXT NOT NULL,
PRIMARY KEY (sha256_hex, repo_id)
)",
// Carry the pre-upgrade markers over. Which repo failed was never recorded,
// so they get the empty sentinel, which no real `repo_id` equals: those
// objects keep the scan fallback until something repairs them, rather than
// being cleared by the next unrelated record the way they would have been
// before. Strictly safer than the behavior being replaced, and bounded by how
// rare an exhausted record is.
"INSERT INTO pin_source_failures (sha256_hex, repo_id)
SELECT sha256_hex, '' FROM pinned_cids WHERE pin_sources_incomplete
ON CONFLICT DO NOTHING",
// `pinned_cids.pin_sources_incomplete` is deliberately NOT dropped. Nothing
// reads it after this migration, and leaving it costs one unused boolean,
// whereas dropping it makes a rollback to the previous release lose the
// markers it still reads.
],
},
Migration {
version: 25,
name: "repos_created_at_id_index",
stmts: &[
// #173 (jatmn): backs the keyset order of the paged legacy CID scan
// (`list_repos_page_for_scan`, `ORDER BY created_at ASC, id ASC` with a
// `(created_at, id) > (...)` cursor). The scan replaced a whole-table
// preload precisely to stop one anonymous `GET /ipfs/{cid}` from costing
// work proportional to the node's repo inventory (INV-10), and without this
// index that bound is only half real: `repos` carries no index on
// `(created_at, id)`, so Postgres seq-scans the whole table and top-N sorts
// it to return EVERY page, while the scarce IPFS walk admission is held.
// Measured on a 50k-row fixture: 954 shared buffers and ~47ms per page
// without it, versus an Index Only Scan at 4-5 buffers, ~0.08ms, and
// `Heap Fetches: 0` with it — and the keyset predicate is pushed down as an
// `Index Cond` instead of filtering after a scan.
//
// Column order and direction are load-bearing and must match the query
// exactly; `idx_repos_updated_at` (the order the scan used to use) cannot
// serve this one. NOTHING NAMES THIS INDEX IN ANY QUERY TEXT, so a
// grep-driven "unused index" cleanup will not see its consumer: it is
// reachable from an unauthenticated route and dropping it reopens the
// amplification, so treat it as part of the resolver, not as tuning.
//
// NEW versioned migration (never appended to an applied block, INV-7).
"CREATE INDEX IF NOT EXISTS idx_repos_created_at_id ON repos (created_at ASC, id ASC)",
],
},
Migration {
version: 26,
name: "pin_repair_sweep_discovery_cursor",
stmts: &[
// #173 round 13 (F5): discovery probes at most
// `MAX_LEGACY_DISCOVERY_PROBES` warm candidates per source-less row, taken
// from a list ordered `(created_at, id)`. That order is stable, so without a
// continuation every traversal probed the same oldest sixteen and a holder
// at position seventeen was never reached by anything, on any node, ever.
// These two columns are the boundary the next traversal's window starts
// after, so coverage becomes a bounded number of traversals rather than
// unreachable.
//
// STEERABILITY is why this is a keyset KEY and not an offset into the list.
// `repo_id` derives from a grindable owner DID, so the one thing an attacker
// must not be able to do is move the window off the true holder. Candidates
// enter and leave the warm list between traversals (a cold repo warming on a
// Tigris-backed node, a fresh registration, a deletion), and every such
// change silently renumbers an offset while leaving a key's boundary exactly
// where it was. Fresh registrations sort LAST under `created_at` and cannot
// be backdated, so they can only ever be appended behind the window.
//
// RESIDUAL, stated rather than implied: an operator who can insert repos
// with an arbitrary `created_at` can still place candidates between the
// continuation and the holder and delay it by a traversal per sixteen rows
// inserted. That is a privileged write, it costs a real repo row each, and
// it delays rather than prevents, since the window keeps advancing.
//
// NEW versioned migration (never appended to an applied block, INV-7). NOT
// NULL DEFAULT '' so an existing `pin_repair_sweep` row reads as "start at
// the head of the list", which is the same thing a never-swept node reads.
"ALTER TABLE pin_repair_sweep ADD COLUMN IF NOT EXISTS discovery_cursor_created_at TEXT NOT NULL DEFAULT ''",
"ALTER TABLE pin_repair_sweep ADD COLUMN IF NOT EXISTS discovery_cursor_id TEXT NOT NULL DEFAULT ''",
],
},
// Reservation: open branches currently claim migration versions through
// v29. Gaps are valid, while reusing a version can silently skip one
// branch's schema when both eventually land. v30 and v31 are intentionally
Expand Down
Loading
You are viewing a condensed version of this merge commit. You can view the full changes here.