Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
75 commits
Select commit Hold shift + click to select a range
34df1fd
feat(node): add per-IP write-surface rate brake and apply to write_ro…
beardthelion Jul 13, 2026
74e05be
feat(node): extend the per-IP write brake to the remaining write sinks
beardthelion Jul 13, 2026
6fded6e
test(node): adversarial TrustedProxy verification through the write b…
beardthelion Jul 13, 2026
e6324d2
feat(node): purge-spam admin subcommand for the empty-burst cleanup
beardthelion Jul 13, 2026
285d7c7
fix(review): make purge-spam empty-check reject git-discovery reads
beardthelion Jul 13, 2026
090019d
fix(review): normalization-consistent purge, brake helper, CLI + cove…
beardthelion Jul 13, 2026
dfd4f6c
test(review): close the reasoned-not-run gaps by execution
beardthelion Jul 13, 2026
0135419
fix(node): reject path-traversal repo names in purge-spam before remo…
beardthelion Jul 15, 2026
a0fcbb0
fix(node): delete child rows transactionally in delete_repo_by_id
beardthelion Jul 15, 2026
fde9cfe
fix(node): reap write_rate_limiter in the periodic cleanup loop
beardthelion Jul 15, 2026
15522d5
fix(node): hold the per-repo advisory lock across purge recheck+delete
beardthelion Jul 15, 2026
1fcca0d
fix(node): count purge-spam disk-removal failures separately from del…
beardthelion Jul 15, 2026
d9eebd2
fix(node): warn on unparseable rate-limit env vars instead of silentl…
beardthelion Jul 15, 2026
6325402
docs(node): document GITLAWB_IPFS_API/TIGRIS_BUCKET/METRICS_ADDR/SHUT…
beardthelion Jul 15, 2026
bfbe502
test(node): exercise the dry-run guard with a real on-disk candidate …
beardthelion Jul 15, 2026
cdb102e
style(node): cargo fmt for the #196 fix set
beardthelion Jul 15, 2026
4df79f2
fix(node): pin the writer's advisory-lock connection in RepoWriteGuard
beardthelion Jul 15, 2026
68c1f5c
refactor(node): introduce ObjectStore trait seam over TigrisClient
beardthelion Jul 15, 2026
3c1ceb8
fix(node): make purge-spam Tigris-authoritative and archive-deleting
beardthelion Jul 15, 2026
cf669bb
fix(review): don't hold/leak the pool connection in acquire_write
beardthelion Jul 15, 2026
dde5ea9
fix(node): make advisory-lock guards drop-safe and reorder acquire_write
beardthelion Jul 16, 2026
b94d440
fix(node): complete a fully-received push after client disconnect
beardthelion Jul 16, 2026
e4c4161
fix(node): serialize archive uploads under the per-repo advisory lock
beardthelion Jul 16, 2026
6ba698c
fix(node): let purge-spam reach repos that exist only as archives
beardthelion Jul 16, 2026
9f7757c
refactor(node): dedup limiter setup + write-brake test driver; fix .e…
beardthelion Jul 16, 2026
2a78524
docs(node): drop the .env.example shutdown-grace entry #196 added (#196)
beardthelion Jul 19, 2026
c4b337b
fix(node): give advisory-lock guards a dedicated pool and bound the r…
beardthelion Jul 19, 2026
4934646
fix(node): advertise a window-derived Retry-After on rate-limit 429s …
beardthelion Jul 19, 2026
0c279c9
fix(node): skip slug-scoped purge cascade when a mirror row shares th…
beardthelion Jul 19, 2026
86a8228
fix(node): fail closed classifying the purge target directory (#196)
beardthelion Jul 20, 2026
bf2a54e
fix(node): run the push completion tail in the disconnect-surviving t…
beardthelion Jul 20, 2026
c24b541
fix(node): tombstone bounties when their repo is purged (#196)
beardthelion Jul 20, 2026
3baa9ed
fix(node): serialize create_repo against the purge advisory lock (#196)
beardthelion Jul 20, 2026
9916bca
test(node): prove purge deletes at MAX_CONNECTIONS=1 with the split p…
beardthelion Jul 20, 2026
8f3d643
fix(node): fail the push when the durable upload times out, don't 200…
beardthelion Jul 20, 2026
d1e596e
fix(node): tombstone bounties under both owner forms on purge (#196)
beardthelion Jul 20, 2026
ce5dcc3
fix(node): fail closed on a symlink at the purge target (#196)
beardthelion Jul 20, 2026
7dc7be2
fix(node): retry-after on the push-advert 429, and create_repo locks …
beardthelion Jul 20, 2026
d2ba14e
fix(node): correct the create-pool and bounty-tombstone remediations …
beardthelion Jul 20, 2026
5a46ef6
fix(node): close the durable-upload data-loss class on every write pa…
beardthelion Jul 20, 2026
2fa9fa3
chore: merge main (iCaptcha PoW + gl retry tests) into feat/write-sin…
beardthelion Jul 21, 2026
16517f8
refactor(node): route the per-DID 429 through the shared too_many_req…
beardthelion Jul 21, 2026
ba572b3
fix(node): init's background upload waits out the creator's advisory …
beardthelion Jul 21, 2026
859240b
fix(node): roll back the local issue ref before unlock when the durab…
beardthelion Jul 21, 2026
c07435d
fix(node): fork publishes only after a durable upload, serialized on …
beardthelion Jul 21, 2026
2ad1c79
fix(node): read-path downloads publish under the purge lock, without …
beardthelion Jul 21, 2026
b2ca5a7
fix(node): bound every under-lock object-store call and dedup the upl…
beardthelion Jul 21, 2026
fa4322c
fix(node): harden the read-path download against cancellation, stale …
beardthelion Jul 21, 2026
c3d6e73
fix(node): keep the fork's unbounded work out of the target-lock span…
beardthelion Jul 21, 2026
03ef9b5
refactor(node): move read-path download coordination into a repo_stor…
beardthelion Jul 21, 2026
4408d27
fix(node): bound acquire_write's download and free the download-map e…
beardthelion Jul 21, 2026
a12f21a
fix(node): roll back the fork mirror and archive on late clone/insert…
beardthelion Jul 21, 2026
556aa31
fix(node): serialize repo purge cascade with mirror ingest via adviso…
beardthelion Jul 22, 2026
1d8c281
fix(node): roll back local ref on failed durable upload (close/merge/…
beardthelion Jul 22, 2026
1d0d3d2
fix(node): tie cancelled cold-download cleanup to blocking extraction
beardthelion Jul 22, 2026
03cd81e
chore(node): fmt + clippy cleanup for the #196 fixes
beardthelion Jul 22, 2026
9152178
fix(node): never let a failed ref snapshot become a delete-all rollba…
beardthelion Jul 22, 2026
296704f
fix(node): slug-lock every peer-ingest writer and drive the real inge…
beardthelion Jul 22, 2026
1e79ae5
fix(node): own cold-download cleanup in a spawned task so handler can…
beardthelion Jul 22, 2026
ce25bba
refactor(node): delegate issue ref helpers to the store ref primitives
beardthelion Jul 22, 2026
4329b2a
Merge origin/main into feat/write-sink-brake-coverage (reintegrate cu…
beardthelion Jul 22, 2026
d755180
fix(node): slug-lock arweave anchor writes; fail closed when the pre-…
beardthelion Jul 22, 2026
e6f4b2f
fix(node): close review residuals: coalesce cold downloads to one in-…
beardthelion Jul 22, 2026
1459cf2
Merge remote-tracking branch 'origin/main' into feat/write-sink-brake…
beardthelion Nov 11, 2026
da876e8
fix(node): re-apply create/fork purge serialization and restore brake…
beardthelion Nov 11, 2026
edfa6bb
fix(node): close review gaps in purge, locks, and fork cleanup
beardthelion Nov 11, 2026
96ed2b2
fix(node): subscriptions-only ws schema and unsettled-work purge guard
beardthelion Nov 11, 2026
082a37f
Merge remote-tracking branch 'origin/main' into feat/write-sink-brake…
beardthelion Nov 11, 2026
e9e86a1
fix(node): wire close_issue rollback, close purge symlink gap, correc…
beardthelion Nov 11, 2026
1187dca
fix(node): close purge/upload lifecycle gaps from review
beardthelion Nov 11, 2026
261f7ab
fix(node): merge rollback, honest create rollback, purge row recheck
beardthelion Nov 12, 2026
662b9a3
fix(node): probe purge slug liveness, gate issue write brake behind t…
beardthelion Nov 12, 2026
43d4f59
fix(node): close-issue rollback uses resolved ref, drop router brake …
beardthelion Nov 12, 2026
4b483a3
fix(node): close_issue spends neither bucket when either cap refuses,…
beardthelion Nov 12, 2026
11cadb9
fix(node): refund the write slot when the close cap refuses, send Ret…
beardthelion Nov 12, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
fix(node): bound acquire_write's download and free the download-map e…
…ntry on cancel (#196)

Two gaps a second-model adversarial pass caught that the in-family
review missed. acquire_write's under-lock exists()/download() ran with
no timeout, so a stalled Tigris GET pinned the write lock and its
lock-pool connection forever (the local-copy fallback only fires on an
error, never a hang) - both are now wrapped in
timeout(release_upload_timeout), a timeout mapping to the existing
download-error fallback, which also makes the prior 'bound every
under-lock call' claim true. The download_locks map entry was removed
only on normal return, not on a cancelled cold read, so a disconnecting
caller could grow the map across public repos; a DownloadEntryGuard now
prunes it in Drop (outer map moved to std::sync::Mutex, held only for
map ops, never across an await). RED observed on both new tests,
including a real mid-download cancellation.
  • Loading branch information
beardthelion committed Jul 21, 2026
commit 4408d2705865ffcb9ab422a8ea725b64c8895e58
167 changes: 152 additions & 15 deletions crates/gitlawb-node/src/git/repo_store.rs
Original file line number Diff line number Diff line change
Expand Up @@ -51,9 +51,14 @@ pub struct RepoStore {
/// same repo serialize here so only one runs the network download while
/// the rest await it and serve what the winner published. Entries are
/// created only after a caller has confirmed the archive exists (reached
/// the download branch) and are removed when the holder finishes, so
/// permissionless requests for arbitrary names cannot grow the map.
download_locks: Arc<Mutex<HashMap<String, Arc<Mutex<()>>>>>,
/// the download branch) and are removed when the holder finishes (on every
/// exit, including handler cancellation, via `DownloadEntryGuard`), so
/// permissionless requests for arbitrary names cannot grow the map. The
/// OUTER map is a std::sync::Mutex (only ever held briefly for a get/insert/
/// remove, never across an await), so the RAII entry guard's synchronous
/// `Drop` can prune it. The INNER per-repo `Arc<Mutex<()>>` stays a
/// tokio::sync::Mutex — it is the async serialization primitive readers await.
download_locks: Arc<std::sync::Mutex<HashMap<String, Arc<Mutex<()>>>>>,
}

impl RepoStore {
Expand All @@ -65,7 +70,7 @@ impl RepoStore {
lock_pool,
release_upload_timeout: DEFAULT_RELEASE_UPLOAD_TIMEOUT,
migrated: Arc::new(tokio::sync::Mutex::new(std::collections::HashSet::new())),
download_locks: Arc::new(Mutex::new(HashMap::new())),
download_locks: Arc::new(std::sync::Mutex::new(HashMap::new())),
}
}

Expand All @@ -80,7 +85,7 @@ impl RepoStore {
lock_pool,
release_upload_timeout: DEFAULT_RELEASE_UPLOAD_TIMEOUT,
migrated: Arc::new(Mutex::new(HashSet::new())),
download_locks: Arc::new(Mutex::new(HashMap::new())),
download_locks: Arc::new(std::sync::Mutex::new(HashMap::new())),
}
}

Expand Down Expand Up @@ -298,20 +303,45 @@ impl RepoStore {

// Always download the latest from the object store before writing.
// Local disk may be stale if another machine pushed since our last access.
//
// Both the exists() re-check and the download() are bounded by
// release_upload_timeout (INV-22): they run while the write advisory lock
// is held on a pinned lock-pool connection, so a stalled (never-erroring)
// GET would pin the lock and its connection indefinitely, and the local-
// copy fallback below (which only fires on Err) would never trigger on a
// hang. On timeout, exists() collapses to "not present" (skip the
// download, matching download_and_publish's bounded exists re-check), and
// download() takes the SAME arm as a download error so a hang bails
// instead of hanging. The guard was wrapped before this await (KTD-2), so
// every bail here still frees the lock via the guard's Drop.
if let Some(ref store) = self.object_store {
if store
.exists(&guard.owner_slug, &guard.repo_name)
.await
.unwrap_or(false)
{
let archive_present = matches!(
tokio::time::timeout(
self.release_upload_timeout,
store.exists(&guard.owner_slug, &guard.repo_name),
)
.await,
Ok(Ok(true))
);
if archive_present {
debug!(repo = %guard.repo_name, "write acquire: downloading latest from object store");
if let Err(e) = store
.download(&guard.owner_slug, &guard.repo_name, &guard.local_path)
.await
let download_result = match tokio::time::timeout(
self.release_upload_timeout,
store.download(&guard.owner_slug, &guard.repo_name, &guard.local_path),
)
.await
{
Ok(res) => res,
Err(_) => Err(anyhow::anyhow!(
"object-store download timed out after {}s under the write lock",
self.release_upload_timeout.as_secs()
)),
};
if let Err(e) = download_result {
// Same self-healing fallback as acquire_fresh: a corrupt/unreadable
// archive must not block a write when a valid local copy
// exists — release(success) will re-upload a good archive.
// archive OR a stalled-then-timed-out GET must not block a write
// when a valid local copy exists — release(success) will re-upload
// a good archive.
if guard.local_path.exists() {
warn!(repo = %guard.repo_name, err = %e,
"write acquire: object-store download failed — falling back to local copy");
Expand Down Expand Up @@ -1899,6 +1929,61 @@ mod tests {
);
}

// FINDING 3 / R2 / INV-22: acquire_write's under-lock freshness DOWNLOAD must
// be bounded, not just release()'s upload. acquire_write holds the write
// advisory lock on a pinned lock-pool connection, then does store.exists() +
// store.download() before returning the guard. A stalled (never-erroring) GET
// would pin that lock and its connection forever, and the local-copy fallback
// (which only fires on Err) never triggers on a hang. With a short bound and a
// download that never completes, acquire_write returns within the bound and
// the lock is freed. Pre-fix (untimed download) acquire_write hangs on the
// stalled GET and never returns -> the outer 5s timeout fires -> RED.
#[sqlx::test]
async fn acquire_write_download_stall_is_bounded(
pool_opts: sqlx::postgres::PgPoolOptions,
connect_opts: sqlx::postgres::PgConnectOptions,
) {
use sqlx::ConnectOptions;
let pool = no_reap_pool(pool_opts, &connect_opts).await;
let tmp = tempfile::TempDir::new().unwrap();
let mut ts = GatedStore::new(true);
// The download parks forever; the acquire bound is what must unblock it.
ts.download_gate = Some(std::sync::Arc::new(tokio::sync::Notify::new()));
let store = RepoStore::new(
tmp.path().to_path_buf(),
Some(std::sync::Arc::new(ts)),
pool,
)
.with_release_upload_timeout(std::time::Duration::from_millis(300));
let owner = "did:key:z6MkAcquireWriteStallAAAAAAAAAAAAAAAAAAAAAA";
let name = "acqwritestall";
let key = lock_key_for(owner, name);
let mut observer = connect_opts.connect().await.unwrap();

// No local copy exists, so the timed-out download has nothing to fall back
// to and bails; the guard drops and its Drop frees the advisory lock. The
// call must RETURN within the bound rather than hang on the stalled GET.
let start = std::time::Instant::now();
let res = tokio::time::timeout(
std::time::Duration::from_secs(5),
store.acquire_write(owner, name),
)
.await
.expect("acquire_write must return within the download bound, not hang on the stalled GET");
assert!(
res.is_err(),
"with no local copy to fall back to, a timed-out download must bail"
);
assert!(
start.elapsed() < std::time::Duration::from_secs(5),
"acquire_write must return within the download bound"
);
assert!(
poll_until_free(&mut observer, key).await,
"the advisory lock must be freed after the bounded (timed-out) download"
);
}

// R2/INV-22: the OTHER under-lock upload path — upload_locked's PUT (init's
// background upload, wait=true) — must be bounded exactly like release()'s.
// With a stalled PUT, upload_locked acquires the advisory lock, parks on the
Expand Down Expand Up @@ -2543,4 +2628,56 @@ mod tests {
.execute(&mut observer)
.await;
}

// ── Finding 4: the download-coordination map entry is freed on cancellation ─

// A cold-read holder parks mid-download, then its handler future is dropped
// (axum cancels it on client disconnect). The per-repo map entry must be
// removed anyway — the pre-fix code removed it only at explicit return points,
// which a cancelled future skips, so a permissionless caller could fan cold
// GETs across many public repos and disconnect to grow the map. The RAII
// DownloadEntryGuard's Drop closes that gap. This reads the private
// download_locks map directly (the test module is a child of RepoStore's
// module). Pre-fix: after abort the entry LEAKS -> still present -> RED.
#[sqlx::test]
async fn cancelled_cold_read_frees_the_map_entry(pool: sqlx::PgPool) {
let tmp = tempfile::TempDir::new().unwrap();
let mut ts = GatedStore::new(true);
let gate = std::sync::Arc::new(tokio::sync::Notify::new());
ts.download_gate = Some(gate.clone());
let downloads = ts.downloads.clone();
let store = RepoStore::new(
tmp.path().to_path_buf(),
Some(std::sync::Arc::new(ts)),
pool,
);
let owner = "did:key:z6MkCancelMapEntryAAAAAAAAAAAAAAAAAAAAAAAA";
let name = "cancelmapentry";
let map_key = format!("{}/{name}", owner.replace([':', '/'], "_"));

// Cache miss: the holder enters download_published, registers the map
// entry, and parks mid-download on the gate.
let store2 = store.clone();
let (o2, n2) = (owner.to_string(), name.to_string());
let h = tokio::spawn(async move { store2.acquire(&o2, &n2).await });
assert!(
poll_until_true(|| downloads.lock().unwrap().len() == 1).await,
"the reader's download must be in flight"
);
// Precondition: the map entry is registered while the holder runs.
assert!(
store.download_locks.lock().unwrap().contains_key(&map_key),
"the coordination entry must be registered while the holder downloads"
);

// Cancel the handler future mid-download (the client-disconnect hazard).
h.abort();

// The RAII guard's Drop must remove the entry despite the cancellation.
let freed = poll_until_true(|| store.download_locks.lock().unwrap().is_empty()).await;
assert!(
freed,
"a cold-read future cancelled mid-download must still free its map entry"
);
}
}
60 changes: 45 additions & 15 deletions crates/gitlawb-node/src/git/repo_store/download.rs
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,35 @@ impl Drop for TempDownloadDir {
}
}

/// RAII removal of a download-coordination map entry (finding 4). Its `Drop`
/// removes the entry from `download_locks` whenever the map still holds THIS
/// Arc (ptr_eq-guarded so a newer entry inserted after an earlier removal is
/// never clobbered). Removing on every exit — normal return, `?`, or a handler
/// future cancelled mid cold-read (axum drops it on client disconnect) — is
/// what keeps the map from growing per arbitrary requested name: the pre-fix
/// explicit removals ran only at the labelled return points, which a cancelled
/// future skipped, leaking the entry. The outer map is a std::sync::Mutex so
/// this Drop can prune it synchronously (Drop cannot be async); it is only ever
/// held briefly for a get/insert/remove, never across an await.
struct DownloadEntryGuard {
locks: Arc<std::sync::Mutex<HashMap<String, Arc<Mutex<()>>>>>,
key: String,
entry: Arc<Mutex<()>>,
}

impl Drop for DownloadEntryGuard {
fn drop(&mut self) {
// A poisoned map is still safe to prune — recover the guard and remove.
let mut map = self.locks.lock().unwrap_or_else(|p| p.into_inner());
if map
.get(&self.key)
.is_some_and(|cur| Arc::ptr_eq(cur, &self.entry))
{
map.remove(&self.key);
}
}
}

impl RepoStore {
/// Coordinated read-path download (KTD-3). Serializes concurrent readers
/// of the same repo on a per-repo async mutex: the first one in becomes
Expand All @@ -76,20 +105,30 @@ impl RepoStore {
) -> Result<DownloadOutcome> {
let map_key = format!("{owner_slug}/{repo_name}");
let entry = {
let mut map = self.download_locks.lock().await;
let mut map = self.download_locks.lock().unwrap();
Arc::clone(
map.entry(map_key.clone())
.or_insert_with(|| Arc::new(Mutex::new(()))),
)
};
// RAII: remove the map entry on EVERY exit of this function — normal
// return, `?`, or handler cancellation (finding 4). Its Drop is ptr_eq-
// guarded, so it never clobbers a newer entry, and it subsumes the
// pre-fix explicit removals (which a cancelled future skipped, leaking
// the entry). Declared here so it outlives `held` below and prunes the
// map only after the inner per-repo lock has been released.
let _entry_guard = DownloadEntryGuard {
locks: Arc::clone(&self.download_locks),
key: map_key.clone(),
entry: Arc::clone(&entry),
};
let held = match entry.try_lock() {
Ok(g) => g,
Err(_) => {
let g = entry.lock().await;
if local_path.exists() {
// A concurrent reader published while we waited — serve it.
drop(g);
self.remove_download_entry(&map_key, &entry).await;
// `_entry_guard`'s Drop removes the map entry on return.
return Ok(DownloadOutcome::Published);
}
// The holder degraded without publishing; this reader takes over.
Expand All @@ -99,7 +138,7 @@ impl RepoStore {
// (finding 3). Narrows the duplicate-download window; publishes
// stay serialized by the advisory lock regardless.
{
let mut map = self.download_locks.lock().await;
let mut map = self.download_locks.lock().unwrap();
map.entry(map_key.clone())
.or_insert_with(|| Arc::clone(&entry));
}
Expand All @@ -116,19 +155,10 @@ impl RepoStore {
local_existed_at_entry,
)
.await;
self.remove_download_entry(&map_key, &entry).await;
drop(held);
outcome
}

/// Remove a completed download-coordination entry, but only when the map
/// still holds THIS entry — a newer entry inserted after an earlier
/// removal must not be clobbered.
async fn remove_download_entry(&self, key: &str, entry: &Arc<Mutex<()>>) {
let mut map = self.download_locks.lock().await;
if map.get(key).is_some_and(|cur| Arc::ptr_eq(cur, entry)) {
map.remove(key);
}
// `_entry_guard`'s Drop removes the map entry here (after `held` and on
// every early return above), including when this future is cancelled.
}

/// The holder's half of the coordinated download: fetch and extract the
Expand Down