Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
75 commits
Select commit Hold shift + click to select a range
34df1fd
feat(node): add per-IP write-surface rate brake and apply to write_ro…
beardthelion Jul 13, 2026
74e05be
feat(node): extend the per-IP write brake to the remaining write sinks
beardthelion Jul 13, 2026
6fded6e
test(node): adversarial TrustedProxy verification through the write b…
beardthelion Jul 13, 2026
e6324d2
feat(node): purge-spam admin subcommand for the empty-burst cleanup
beardthelion Jul 13, 2026
285d7c7
fix(review): make purge-spam empty-check reject git-discovery reads
beardthelion Jul 13, 2026
090019d
fix(review): normalization-consistent purge, brake helper, CLI + cove…
beardthelion Jul 13, 2026
dfd4f6c
test(review): close the reasoned-not-run gaps by execution
beardthelion Jul 13, 2026
0135419
fix(node): reject path-traversal repo names in purge-spam before remo…
beardthelion Jul 15, 2026
a0fcbb0
fix(node): delete child rows transactionally in delete_repo_by_id
beardthelion Jul 15, 2026
fde9cfe
fix(node): reap write_rate_limiter in the periodic cleanup loop
beardthelion Jul 15, 2026
15522d5
fix(node): hold the per-repo advisory lock across purge recheck+delete
beardthelion Jul 15, 2026
1fcca0d
fix(node): count purge-spam disk-removal failures separately from del…
beardthelion Jul 15, 2026
d9eebd2
fix(node): warn on unparseable rate-limit env vars instead of silentl…
beardthelion Jul 15, 2026
6325402
docs(node): document GITLAWB_IPFS_API/TIGRIS_BUCKET/METRICS_ADDR/SHUT…
beardthelion Jul 15, 2026
bfbe502
test(node): exercise the dry-run guard with a real on-disk candidate …
beardthelion Jul 15, 2026
cdb102e
style(node): cargo fmt for the #196 fix set
beardthelion Jul 15, 2026
4df79f2
fix(node): pin the writer's advisory-lock connection in RepoWriteGuard
beardthelion Jul 15, 2026
68c1f5c
refactor(node): introduce ObjectStore trait seam over TigrisClient
beardthelion Jul 15, 2026
3c1ceb8
fix(node): make purge-spam Tigris-authoritative and archive-deleting
beardthelion Jul 15, 2026
cf669bb
fix(review): don't hold/leak the pool connection in acquire_write
beardthelion Jul 15, 2026
dde5ea9
fix(node): make advisory-lock guards drop-safe and reorder acquire_write
beardthelion Jul 16, 2026
b94d440
fix(node): complete a fully-received push after client disconnect
beardthelion Jul 16, 2026
e4c4161
fix(node): serialize archive uploads under the per-repo advisory lock
beardthelion Jul 16, 2026
6ba698c
fix(node): let purge-spam reach repos that exist only as archives
beardthelion Jul 16, 2026
9f7757c
refactor(node): dedup limiter setup + write-brake test driver; fix .e…
beardthelion Jul 16, 2026
2a78524
docs(node): drop the .env.example shutdown-grace entry #196 added (#196)
beardthelion Jul 19, 2026
c4b337b
fix(node): give advisory-lock guards a dedicated pool and bound the r…
beardthelion Jul 19, 2026
4934646
fix(node): advertise a window-derived Retry-After on rate-limit 429s …
beardthelion Jul 19, 2026
0c279c9
fix(node): skip slug-scoped purge cascade when a mirror row shares th…
beardthelion Jul 19, 2026
86a8228
fix(node): fail closed classifying the purge target directory (#196)
beardthelion Jul 20, 2026
bf2a54e
fix(node): run the push completion tail in the disconnect-surviving t…
beardthelion Jul 20, 2026
c24b541
fix(node): tombstone bounties when their repo is purged (#196)
beardthelion Jul 20, 2026
3baa9ed
fix(node): serialize create_repo against the purge advisory lock (#196)
beardthelion Jul 20, 2026
9916bca
test(node): prove purge deletes at MAX_CONNECTIONS=1 with the split p…
beardthelion Jul 20, 2026
8f3d643
fix(node): fail the push when the durable upload times out, don't 200…
beardthelion Jul 20, 2026
d1e596e
fix(node): tombstone bounties under both owner forms on purge (#196)
beardthelion Jul 20, 2026
ce5dcc3
fix(node): fail closed on a symlink at the purge target (#196)
beardthelion Jul 20, 2026
7dc7be2
fix(node): retry-after on the push-advert 429, and create_repo locks …
beardthelion Jul 20, 2026
d2ba14e
fix(node): correct the create-pool and bounty-tombstone remediations …
beardthelion Jul 20, 2026
5a46ef6
fix(node): close the durable-upload data-loss class on every write pa…
beardthelion Jul 20, 2026
2fa9fa3
chore: merge main (iCaptcha PoW + gl retry tests) into feat/write-sin…
beardthelion Jul 21, 2026
16517f8
refactor(node): route the per-DID 429 through the shared too_many_req…
beardthelion Jul 21, 2026
ba572b3
fix(node): init's background upload waits out the creator's advisory …
beardthelion Jul 21, 2026
859240b
fix(node): roll back the local issue ref before unlock when the durab…
beardthelion Jul 21, 2026
c07435d
fix(node): fork publishes only after a durable upload, serialized on …
beardthelion Jul 21, 2026
2ad1c79
fix(node): read-path downloads publish under the purge lock, without …
beardthelion Jul 21, 2026
b2ca5a7
fix(node): bound every under-lock object-store call and dedup the upl…
beardthelion Jul 21, 2026
fa4322c
fix(node): harden the read-path download against cancellation, stale …
beardthelion Jul 21, 2026
c3d6e73
fix(node): keep the fork's unbounded work out of the target-lock span…
beardthelion Jul 21, 2026
03ef9b5
refactor(node): move read-path download coordination into a repo_stor…
beardthelion Jul 21, 2026
4408d27
fix(node): bound acquire_write's download and free the download-map e…
beardthelion Jul 21, 2026
a12f21a
fix(node): roll back the fork mirror and archive on late clone/insert…
beardthelion Jul 21, 2026
556aa31
fix(node): serialize repo purge cascade with mirror ingest via adviso…
beardthelion Jul 22, 2026
1d8c281
fix(node): roll back local ref on failed durable upload (close/merge/…
beardthelion Jul 22, 2026
1d0d3d2
fix(node): tie cancelled cold-download cleanup to blocking extraction
beardthelion Jul 22, 2026
03cd81e
chore(node): fmt + clippy cleanup for the #196 fixes
beardthelion Jul 22, 2026
9152178
fix(node): never let a failed ref snapshot become a delete-all rollba…
beardthelion Jul 22, 2026
296704f
fix(node): slug-lock every peer-ingest writer and drive the real inge…
beardthelion Jul 22, 2026
1e79ae5
fix(node): own cold-download cleanup in a spawned task so handler can…
beardthelion Jul 22, 2026
ce25bba
refactor(node): delegate issue ref helpers to the store ref primitives
beardthelion Jul 22, 2026
4329b2a
Merge origin/main into feat/write-sink-brake-coverage (reintegrate cu…
beardthelion Jul 22, 2026
d755180
fix(node): slug-lock arweave anchor writes; fail closed when the pre-…
beardthelion Jul 22, 2026
e6f4b2f
fix(node): close review residuals: coalesce cold downloads to one in-…
beardthelion Jul 22, 2026
1459cf2
Merge remote-tracking branch 'origin/main' into feat/write-sink-brake…
beardthelion Nov 11, 2026
da876e8
fix(node): re-apply create/fork purge serialization and restore brake…
beardthelion Nov 11, 2026
edfa6bb
fix(node): close review gaps in purge, locks, and fork cleanup
beardthelion Nov 11, 2026
96ed2b2
fix(node): subscriptions-only ws schema and unsettled-work purge guard
beardthelion Nov 11, 2026
082a37f
Merge remote-tracking branch 'origin/main' into feat/write-sink-brake…
beardthelion Nov 11, 2026
e9e86a1
fix(node): wire close_issue rollback, close purge symlink gap, correc…
beardthelion Nov 11, 2026
1187dca
fix(node): close purge/upload lifecycle gaps from review
beardthelion Nov 11, 2026
261f7ab
fix(node): merge rollback, honest create rollback, purge row recheck
beardthelion Nov 12, 2026
662b9a3
fix(node): probe purge slug liveness, gate issue write brake behind t…
beardthelion Nov 12, 2026
43d4f59
fix(node): close-issue rollback uses resolved ref, drop router brake …
beardthelion Nov 12, 2026
4b483a3
fix(node): close_issue spends neither bucket when either cap refuses,…
beardthelion Nov 12, 2026
11cadb9
fix(node): refund the write slot when the close cap refuses, send Ret…
beardthelion Nov 12, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
fix(node): fork publishes only after a durable upload, serialized on …
…the target-namespace lock (#196)

fork_repo now takes the per-repo advisory lock before its conflict check
and holds it through clone, upload, and row insert, 503ing on contention
exactly as create_repo does. The upload runs under the held guard via
upload_under_guard, PUT bounded by release_upload_timeout, returning
false only for an attempted upload that failed or timed out (no-store
nodes fork unchanged); on failure the mirror is removed and the fork
500s before any row is built. release_after_write is retired; its tests
migrated onto the new method. RED observed on the serialization,
durability, and contention tests; defect injection (row insert reordered
above the upload) turned the durability test RED and was reverted.
  • Loading branch information
beardthelion committed Jul 21, 2026
commit c07435d2453331df229c044b8aeafd7e4feed97e
361 changes: 357 additions & 4 deletions crates/gitlawb-node/src/api/repos.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1658,6 +1658,24 @@ pub async fn fork_repo(
));
}

// Serialize against a concurrent same-key purge or creation on the FORK
// TARGET namespace, mirroring create_repo's lock above (same rationale,
// same 503 mapping). Held across the conflict check, clone, durable upload,
// and row insert, so none of it can interleave with a purge's delete-row +
// remove-dir window, and so the row is only published after the archive
// durably landed. Released explicitly on both exits below; the guard's
// Drop frees the lock on every intermediate error path.
let repo_lock = state
.repo_store
.lock_repo_blocking(&forker_did, &fork_name)
.await
.map_err(|e| AppError::Unavailable(e.to_string()))?
.ok_or_else(|| {
AppError::Unavailable(format!(
"could not acquire repo lock for {forker_did}/{fork_name}: held by a live writer or purge"
))
})?;

// Check no name conflict under the forker's ownership
let forker_short = crate::db::normalize_owner_key(&forker_did);
if state.db.get_repo(forker_short, &fork_name).await?.is_some() {
Expand Down Expand Up @@ -1696,11 +1714,26 @@ pub async fn fork_repo(
)));
}

// Upload fork to Tigris
state
// Upload the fork to durable storage under the held target lock, bounded by
// the release-upload timeout. Publish-after-durability: an attempted upload
// that failed or timed out fails the fork (the sibling write paths' 5xx)
// BEFORE any RepoRecord exists, and removes the cloned mirror so no later
// acquire serves a fork whose archive never landed. Store-less nodes take
// the success arm (nothing to upload is not a failure).
if !state
.repo_store
.release_after_write(&forker_did, &fork_name)
.await;
.upload_under_guard(&forker_did, &fork_name, &repo_lock)
.await
{
if let Err(e) = std::fs::remove_dir_all(&disk_path) {
tracing::warn!(fork = %fork_name, err = %e,
"failed to remove fork mirror after failed durable upload");
}
repo_lock.release().await;
return Err(AppError::Git(format!(
"durable storage upload failed for {fork_name}"
)));
}

let now = Utc::now();
let record = crate::db::RepoRecord {
Expand All @@ -1719,6 +1752,10 @@ pub async fn fork_repo(

state.db.create_repo(&record).await?;

// Row, archive, and on-disk dir now all exist consistently; the race window
// is closed, so the target lock can be released before the best-effort tail.
repo_lock.release().await;

// Persist the proof so the fork carries it when it propagates to peers.
if let Some(p) = verified_proof {
if let Err(e) = p.record_for_repo(&state.db, &record.id).await {
Expand Down Expand Up @@ -3908,6 +3945,322 @@ mod tests {
);
}

// ── U3/R2/R4: fork tail, guarded span, publish only after durability ─────

const FORK_SRC_OWNER: &str = "z6forksrcowner";
const FORK_SRC_NAME: &str = "forksrc";

/// Build a state whose repo_store AND config.repos_dir point at `repos_dir`
/// (fork_repo computes its clone target from config.repos_dir, so the two
/// must agree), seeded with a bare public source repo any caller may fork.
async fn fork_test_state(
repos_dir: &std::path::Path,
repo_store: crate::git::repo_store::RepoStore,
pool: sqlx::PgPool,
) -> AppState {
let mut state = crate::test_support::test_state(pool).await;
state.repo_store = repo_store;
let mut cfg = (*state.config).clone();
cfg.repos_dir = repos_dir.to_path_buf();
state.config = std::sync::Arc::new(cfg);

let bare = repos_dir
.join(FORK_SRC_OWNER) // slug == owner: no ':' or '/' to replace
.join(format!("{FORK_SRC_NAME}.git"));
std::fs::create_dir_all(bare.parent().unwrap()).unwrap();
let out = std::process::Command::new("git")
.args(["init", "--bare", "-q", &bare.to_string_lossy()])
.output()
.unwrap();
assert!(out.status.success(), "git init --bare failed");
state
.db
.upsert_mirror_repo(
FORK_SRC_OWNER,
FORK_SRC_NAME,
&bare.to_string_lossy(),
None,
false,
)
.await
.unwrap();
state
}

async fn call_fork(
state: &AppState,
forker: &str,
fork_name: &str,
) -> Result<axum::response::Response> {
use axum::extract::{Path as AxPath, State};
use axum::response::IntoResponse;
use axum::Extension;
super::fork_repo(
State(state.clone()),
Extension(crate::auth::AuthenticatedDid(forker.to_string())),
AxPath((FORK_SRC_OWNER.to_string(), FORK_SRC_NAME.to_string())),
axum::http::HeaderMap::new(),
Json(ForkRepoRequest {
name: Some(fork_name.to_string()),
}),
)
.await
.map(|r| r.into_response())
}

// R2: fork must take the SAME per-owner/name advisory lock on its TARGET
// namespace that purge and create hold, so it cannot interleave with a
// purge's delete-row + remove-dir (or another creator) on that key.
// Deterministic form: hold the target lock, prove the fork BLOCKS (no clone
// dir, no repos row) rather than proceeding, then completes cleanly once the
// lock frees. RED on base: fork takes no lock and completes in the window.
#[sqlx::test]
async fn fork_serializes_against_target_namespace_lock(
pool_opts: sqlx::postgres::PgPoolOptions,
connect_opts: sqlx::postgres::PgConnectOptions,
) {
use std::time::Duration;

// Multi-connection pool: the held guard pins one connection, so the
// fork's own lock attempt must get a DIFFERENT one and observe the key
// held (Ok(None) -> retry), not merely block on pool exhaustion.
let pool = pool_opts
.max_connections(5)
.connect_with(connect_opts)
.await
.unwrap();
let repos_dir = tempfile::TempDir::new().unwrap();
let repo_store = crate::git::repo_store::RepoStore::for_testing(
repos_dir.path().to_path_buf(),
pool.clone(),
);
let state = fork_test_state(repos_dir.path(), repo_store, pool).await;

let forker = "did:key:z6MkForkSerializeAAAAAAAAAAAAAAAAAAAAAAAAAA";
let fork_name = "serialfork";

// A purge (or another creator) holds the fork target's advisory lock.
let held = state
.repo_store
.try_lock_repo(forker, fork_name)
.await
.unwrap()
.expect("target lock free before the fork");

let state2 = state.clone();
let forker2 = forker.to_string();
let fork_name2 = fork_name.to_string();
let handle = tokio::spawn(async move { call_fork(&state2, &forker2, &fork_name2).await });

// While the target lock is held the fork must make no progress past it.
tokio::time::sleep(Duration::from_millis(500)).await;
let clone_dir = store::repo_disk_path(&state.config.repos_dir, forker, fork_name);
assert!(
state
.db
.get_repo(forker, fork_name)
.await
.unwrap()
.is_none(),
"fork must not insert a repos row while the target-namespace lock is \
held (RED on base: fork takes no lock and the row lands in the window)"
);
assert!(
!clone_dir.exists(),
"fork must not clone the mirror while the target-namespace lock is held"
);
assert!(
!handle.is_finished(),
"fork must be blocked on the target lock, not have completed"
);

held.release().await;
let resp = tokio::time::timeout(Duration::from_secs(8), handle)
.await
.expect("fork should finish once the lock frees")
.expect("fork task join")
.expect("fork_repo returns Ok once it wins the lock");
assert_eq!(resp.status(), StatusCode::CREATED);
assert!(
state
.db
.get_repo(forker, fork_name)
.await
.unwrap()
.is_some(),
"repos row present after the fork wins the lock"
);
assert!(clone_dir.exists(), "cloned mirror present after the fork");
}

// R4: publish-after-durability. When the fork's durable upload stalls, the
// fork must fail (5xx) within the release-upload bound, insert NO repos row,
// and remove the half-created mirror from disk. RED on base: the foreground
// upload's unbounded PUT hangs the handler on a stall (and an erroring
// upload would return 201 + insert the row despite the failed upload).
#[sqlx::test]
async fn fork_durable_upload_failure_fails_before_row_insert(pool: sqlx::PgPool) {
use axum::response::IntoResponse;
use std::path::Path as StdPath;
use std::time::Duration;

// Object store whose upload() parks forever: the fork-tail upload bound
// is the only thing that unblocks it. `exists()` is false so acquire
// never downloads over the local source repo.
struct StallStore;
#[async_trait::async_trait]
impl crate::git::tigris::ObjectStore for StallStore {
async fn exists(&self, _o: &str, _r: &str) -> anyhow::Result<bool> {
Ok(false)
}
async fn upload(&self, _o: &str, _r: &str, _p: &StdPath) -> anyhow::Result<()> {
std::future::pending::<()>().await;
Ok(())
}
async fn download(&self, _o: &str, _r: &str, _p: &StdPath) -> anyhow::Result<()> {
Ok(())
}
async fn delete(&self, _o: &str, _r: &str) -> anyhow::Result<()> {
Ok(())
}
}

let repos_dir = tempfile::TempDir::new().unwrap();
let repo_store = crate::git::repo_store::RepoStore::new(
repos_dir.path().to_path_buf(),
Some(std::sync::Arc::new(StallStore)),
pool.clone(),
)
.with_release_upload_timeout(Duration::from_millis(200));
let state = fork_test_state(repos_dir.path(), repo_store, pool).await;

let forker = "did:key:z6MkForkUploadFailAAAAAAAAAAAAAAAAAAAAAAAAA";
let fork_name = "failfork";

let resp =
tokio::time::timeout(Duration::from_secs(5), call_fork(&state, forker, fork_name))
.await
.expect(
"fork must fail within the upload bound when the durable upload \
stalls (RED on base: the unbounded foreground PUT hangs the handler)",
);
let status = match resp {
Ok(r) => r.status(),
Err(e) => e.into_response().status(),
};
assert!(
status.is_server_error(),
"a failed durable upload must fail the fork (5xx); got {status}, \
which the client trusts as a fork whose archive never landed"
);

// Publish-after-durability: no repos row for a fork with no archive.
assert!(
state
.db
.get_repo(forker, fork_name)
.await
.unwrap()
.is_none(),
"no repos row may exist for a fork whose durable upload failed"
);
let clone_dir = store::repo_disk_path(&state.config.repos_dir, forker, fork_name);
assert!(
!clone_dir.exists(),
"the cloned mirror must be removed when the durable upload fails"
);
}

// KTD-2 trap case (must-not-break negative): a store-less node has nothing
// to upload, so fork succeeds exactly as today: "no object store" is
// nothing-to-do, never a failed upload. GREEN both before and after the fix.
#[sqlx::test]
async fn fork_without_object_store_succeeds(pool: sqlx::PgPool) {
let repos_dir = tempfile::TempDir::new().unwrap();
let repo_store = crate::git::repo_store::RepoStore::for_testing(
repos_dir.path().to_path_buf(),
pool.clone(),
);
let state = fork_test_state(repos_dir.path(), repo_store, pool).await;

let forker = "did:key:z6MkForkNoStoreAAAAAAAAAAAAAAAAAAAAAAAAAAAA";
let fork_name = "nostorefork";

let resp = call_fork(&state, forker, fork_name)
.await
.expect("a store-less fork must succeed");
assert_eq!(resp.status(), StatusCode::CREATED);
assert!(
state
.db
.get_repo(forker, fork_name)
.await
.unwrap()
.is_some(),
"repos row present after a store-less fork"
);
let clone_dir = store::repo_disk_path(&state.config.repos_dir, forker, fork_name);
assert!(clone_dir.exists(), "cloned mirror present after the fork");
}

// R2: a target lock held past the fork's bounded wait must surface as the
// same retryable 503 create_repo returns, with no repos row and no clone
// residue. RED on base: fork ignores the lock and completes with a 201.
#[sqlx::test]
async fn fork_lock_contention_returns_503(
pool_opts: sqlx::postgres::PgPoolOptions,
connect_opts: sqlx::postgres::PgConnectOptions,
) {
use axum::response::IntoResponse;

let pool = pool_opts
.max_connections(5)
.connect_with(connect_opts)
.await
.unwrap();
let repos_dir = tempfile::TempDir::new().unwrap();
let repo_store = crate::git::repo_store::RepoStore::for_testing(
repos_dir.path().to_path_buf(),
pool.clone(),
);
let state = fork_test_state(repos_dir.path(), repo_store, pool).await;

let forker = "did:key:z6MkForkContendedAAAAAAAAAAAAAAAAAAAAAAAAAA";
let fork_name = "contendedfork";

// Held for the whole test: the fork's bounded wait must give up.
let held = state
.repo_store
.try_lock_repo(forker, fork_name)
.await
.unwrap()
.expect("target lock free before the fork");

let resp = call_fork(&state, forker, fork_name).await;
let status = match resp {
Ok(r) => r.status(),
Err(e) => e.into_response().status(),
};
assert_eq!(
status,
StatusCode::SERVICE_UNAVAILABLE,
"a target lock held past the bounded wait must 503 like create_repo \
(RED on base: fork ignores the lock and returns 201)"
);
assert!(
state
.db
.get_repo(forker, fork_name)
.await
.unwrap()
.is_none(),
"no repos row after a 503'd fork"
);
let clone_dir = store::repo_disk_path(&state.config.repos_dir, forker, fork_name);
assert!(!clone_dir.exists(), "no clone residue after a 503'd fork");
held.release().await;
}

// create_repo must NOT take its serialization lock from the APP pool. Doing so
// pins an app-pool connection in the lock guard while the create's own work
// (get_repo / proof.consume / db.create_repo, all on state.db = the app pool)
Expand Down
Loading