Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
57 commits
Select commit Hold shift + click to select a range
7af3067
fix(node): close the advisory-lock probe's session if it is dropped m…
beardthelion Jul 29, 2026
edcf0f2
fix(node): add a dedicated, lazily-connected advisory-lock pool
beardthelion Jul 29, 2026
5ce3c7f
fix(node): hold the lock-owning connection in RepoWriteGuard (#279)
beardthelion Jul 29, 2026
c49d3cd
fix(node): free the advisory lock when a write guard dies without rel…
beardthelion Jul 29, 2026
98f2daf
fix(node): read the advisory unlock's result instead of discarding it
beardthelion Jul 29, 2026
9321d3b
fix(node): bound the object-storage transfers that run under the writ…
beardthelion Jul 29, 2026
9da0939
fix(node): authorize before taking the write lock in close_issue
beardthelion Jul 29, 2026
5d254fb
fix(node): address the review findings on the advisory-lock series
beardthelion Jul 30, 2026
d346bde
fix(node): log lock-pool saturation in the request path, not via read…
beardthelion Jul 30, 2026
82143ad
fix(node): refuse a write when object storage is unknowable, and shed…
beardthelion Jul 30, 2026
8cf6b7a
fix(node): re-authorize close_issue under the guard instead of only r…
beardthelion Jul 30, 2026
af5948b
style(node): rustfmt the new contention test
beardthelion Jul 30, 2026
e5558fb
fix(node): shed an unrefreshable write as a retryable 503 with a fixe…
beardthelion Aug 3, 2026
63a9ef8
fix(node): refuse a fresh-copy write when the archive HEAD cannot be …
beardthelion Aug 3, 2026
10230ac
fix(node): log expected transient acquire failures at warn, not error
beardthelion Aug 3, 2026
259d587
fix(node): log the read failure the close_issue pre-check folds into …
beardthelion Aug 3, 2026
999ee72
test(node): poll for the closed session instead of sleeping a fixed 3…
beardthelion Aug 3, 2026
5fdb902
fix(node): raise a failed fresh download with no local copy as RepoUn…
beardthelion Aug 9, 2026
f81ebbe
test(node): assert the second writer sheds as RepoBusy instead of tim…
beardthelion Aug 9, 2026
c4dbd8a
fix(node): read the close_issue author pre-check from a non-mutating …
beardthelion Aug 10, 2026
9f313b9
fix(node): bound every await in the lock-acquire loop by the deadline
beardthelion Aug 10, 2026
60d70c7
test(node): add an S3 mock that really enforces conditional writes
beardthelion Aug 10, 2026
31f90c4
fix(node): stop pretending a held lock fences a late upload
beardthelion Aug 10, 2026
3eabf79
feat(node): let the storage client express and detect a conditional w…
beardthelion Aug 10, 2026
317841e
fix(node): fence the release publish so an abandoned upload cannot win
beardthelion Aug 10, 2026
3d803ff
test(node): drive the abandoned-upload race end to end, and probe the…
beardthelion Aug 10, 2026
2a92a1f
fix(node): reconcile F-series tests and test seams with the rebased g…
beardthelion Aug 18, 2026
9f35dca
fix(node): address the 2026-08-10 review round on the advisory-lock s…
beardthelion Aug 18, 2026
9dd71e9
fix(node): record why the close-issue read gate cannot carry a synced…
beardthelion Aug 18, 2026
4cee7e0
merge origin/main into fix/279-advisory-lock-session-affinity
beardthelion Aug 26, 2026
b35cb31
fix(node): address jatmn review on merged advisory-lock head
beardthelion Aug 26, 2026
3c20148
fix(node): cap close_issue snapshot work on the read pool
beardthelion Aug 26, 2026
7ba0fef
fix(node): mark close_issue mirror visibility gate for push hook
beardthelion Aug 26, 2026
3df71c9
fix: unblock pre-push after main merge
beardthelion Aug 26, 2026
98fe670
fix(node): carry transient acquire failures through every write handler
beardthelion Aug 28, 2026
4f1c1bb
fix(node): drop unused logcap helper after drain_faults seam
beardthelion Aug 28, 2026
4eb0c44
fix(node): route fork disk paths through validated_repo_disk_path
beardthelion Aug 28, 2026
d1ce2e3
fix(node): harden release-upload outcomes and replication tail gating
beardthelion Aug 28, 2026
8e66e77
fix(node): close jatmn round on publish authority and durability gating
beardthelion Aug 29, 2026
1911291
fix(node): harden issue close/create and fork publish paths on #285
beardthelion Aug 29, 2026
5d3a4ea
fix(node): close write-attempt ownership gaps on advisory-lock release
beardthelion Aug 29, 2026
2ad2f12
fix(node): gate replication tail on publish durability and split clos…
beardthelion Aug 29, 2026
487687c
fix(node): add ValidatedRepoDiskPath for CodeQL and reconcile tail du…
beardthelion Aug 29, 2026
9d02139
fix(node): close three jatmn findings on advisory-lock PR
beardthelion Aug 30, 2026
85c6f48
fix(node): verify adversarial findings on advisory-lock PR
beardthelion Aug 30, 2026
5cd6703
fix(node): compensate under the write lock and harden tail durability…
beardthelion Aug 30, 2026
22da2d6
fix(node): refuse stale under-lock refresh and gate tail durability o…
beardthelion Aug 30, 2026
c306933
style(node): rustfmt advisory-lock gap-closure commits
beardthelion Aug 30, 2026
3a82ccb
fix(node): satisfy clippy on stale-local download test path join
beardthelion Aug 30, 2026
c40f299
feat(node): add a publish attempt/stage boundary
kevincodex1 Aug 31, 2026
21646bc
feat(node): stamp uploads with an attempt id and preserve dispatch am…
kevincodex1 Aug 31, 2026
5e548d2
fix(node): key durability, cache validity and fork cleanup on the att…
kevincodex1 Aug 31, 2026
535c5dc
fix(node): settle the cache contract on the abandoned path and the at…
beardthelion Sep 7, 2026
f048a1e
fix(node): settle the abandoned tree at the disconnect, not at reaper…
beardthelion Sep 7, 2026
957d50b
fix(node): let exactly one classifier settle an abandoned tree
beardthelion Sep 7, 2026
892c752
Merge origin/main into fix/279-advisory-lock-session-affinity
beardthelion Nov 4, 2026
28f3fc2
test(node): use the test's own pool now that Db::pool is gone
beardthelion Nov 4, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
fix(node): settle the cache contract on the abandoned path and the at…
…tempt

The publication boundary this branch introduced was applied at call sites
that each had to be remembered, and five were not. Close the eight gaps
that survived, grouped by what actually failed rather than by symptom.

The marker's lifecycle was answered by the wrong event in three places.
The swap that replaces a live tree now owns the clear, so a cache-miss
download no longer leaves a stale marker that fences the archive it just
installed, and the under-lock refresh no longer lifts a quarantine on a
HEAD that downloaded nothing. init clears a predecessor's marker.

A cancelled handler never reached release, so it applied the tail
contract and not the cache contract. RepoWriteGuard::drop now settles the
tree it abandoned, keyed on how far the publish actually got, and a guard
that handed out its path for writing marks the tree whatever stage it
reached. A marker naming no attempt is a definite non-publication rather
than a permanent refusal: it invalidates and the next read serves the
stored generation, so an abandoned first push cannot wedge a repo at 503.
Only an unreadable sidecar still fails shut.

Both of the store's live-path hand-outs now go through one confirming
helper, so read_snapshot can no longer return an unconfirmed tree as an
ordinary snapshot. The gate binds ahead of the lazy-migration spawn,
because a quarantined tree backfilled to object storage by an IfAbsent
PUT would publish exactly the refs the marker exists to withhold. The
marker no longer rests on a single fs::write; an unwritable sidecar falls
back to an in-memory record that reads consult.

Fork cleanup claims ownership by renaming the stamp before it touches the
tree, so a successor that claims the same path cannot lose its directory
to a predecessor's check-then-unlink, and the directory-absent path no
longer unlinks a stamp it does not own. A create-only precondition loss
whose reconciliation HEAD fails is refused retryably instead of reported
as a permanent name conflict, and its clone is released so the retry can
re-clone rather than finding the destination occupied.

The pin sweep skips a tree whose publish is unresolved, so provider CIDs
are not rewritten from refs that may never become durable. That is the
sidecar marker, distinct from the operator-level quarantined row flag.

26 tests, each written and observed failing before its fix. Suite 1211
passed, 0 failed, 1 ignored; fmt, clippy -D warnings and --locked clean.
  • Loading branch information
beardthelion committed Sep 7, 2026
commit 535c5dca1866a5f830f18b84c303bdbc9e87745d
57 changes: 57 additions & 0 deletions crates/gitlawb-node/src/api/issues.rs
Original file line number Diff line number Diff line change
Expand Up @@ -938,6 +938,63 @@ mod lock_pool_shed_tests {
);
}

/// #285 U2, gap-driving. `close_issue` decides authorship out of a
/// `read_snapshot`, and on a node whose only copy is the live tree that
/// snapshot IS the live tree. When a write left that tree carrying refs the
/// store never confirmed, the pre-check reads authorship out of state no
/// other node can see, so it must be refused as retryable rather than
/// answered.
///
/// No backend is configured here, which is the sharpest form: nothing can
/// ever confirm the marked tree, so serving it is not a race, it is a
/// standing wrong answer.
#[sqlx::test]
async fn close_issue_refuses_a_quarantined_snapshot_before_authorization(pool: PgPool) {
let tmp = tempfile::TempDir::new().unwrap();
let owner = "did:key:zISSUECLOSEQUARANTINEBBBBBBBBBBBBBBBBBBB";
let mut state = crate::test_support::test_state(pool.clone()).await;
state.repo_store =
crate::git::repo_store::RepoStore::for_testing(tmp.path().to_path_buf(), pool.clone());
state
.db
.create_repo(&seed_repo(owner, "quar-close"))
.await
.expect("seed repo");

// A live tree with an unresolved publish beside it.
let live = state
.repo_store
.acquire(owner, "quar-close")
.await
.expect("resolve the live path");
let _ = std::fs::remove_dir_all(&live);
crate::git::store::init_bare(&live).expect("init bare repo");
crate::git::repo_store::quarantine_local_tree(
&live,
"quar-close",
Some(&crate::git::publish::PublishAttemptId::new()),
);

let refused = close_issue(
State(state.clone()),
Extension(AuthenticatedDid(
"did:key:zISSUECLOSEQUARSTRANGER".to_string(),
)),
Path((
owner.to_string(),
"quar-close".to_string(),
"deadbeef".to_string(),
)),
axum::http::HeaderMap::new(),
crate::rate_limit::PeerAddr(None),
)
.await;
assert!(
matches!(refused, Err(AppError::RepoUnavailable)),
"close_issue must not read authorship out of a quarantined tree the store cannot confirm"
);
}

#[sqlx::test]
async fn close_issue_lock_pool_exhaustion_sheds_503_not_500(pool: PgPool) {
let owner = "did:key:zISSUECLOSELOCKPOOLBBBBBBBBBBBBBBBBBBBBB";
Expand Down
278 changes: 262 additions & 16 deletions crates/gitlawb-node/src/api/repos.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3562,26 +3562,51 @@ pub async fn fork_repo(
// the create-only fence refuse the second copy. The stored object
// carrying our own attempt id says the publish succeeded, and
// refusing it would fence the fork name behind our own work.
if state
match state
.repo_store
.fork_attempt_landed(&forker_did, &fork_name, &attempt)
.await
.unwrap_or(false)
{
tracing::info!(
fork = %fork_name,
status,
"fork create-only PUT was refused but the stored archive is this \
attempt's own — continuing"
);
} else {
tracing::warn!(
forker = %forker_did,
fork = %fork_name,
status,
"fork refused: an archive already exists under the fork's key"
);
return Err(AppError::RepoExists(fork_name.clone()));
Ok(true) => {
tracing::info!(
fork = %fork_name,
status,
"fork create-only PUT was refused but the stored archive is this \
attempt's own — continuing"
);
}
Ok(false) => {
tracing::warn!(
forker = %forker_did,
fork = %fork_name,
status,
"fork refused: an archive already exists under the fork's key"
);
return Err(AppError::RepoExists(fork_name.clone()));
}
// The store could not be asked whether the stored archive is ours,
// so "not ours" is not an answer we have. Reporting a permanent name
// conflict from a transient store failure sends the client away from
// a name that may well be free; refuse retryably instead.
//
// The clone guard is deliberately NOT disarmed here. Unlike the
// ambiguous arm below, a lost precondition proves this attempt's PUT
// was refused, so the local clone protects nothing the store does not
// already hold. Keeping it would wedge the fork name: the retry this
// 503 invites re-clones into the same destination and fails because
// it exists, and the leftover carries a dead attempt's stamp that no
// cleanup will ever claim. Let Drop remove it.
Err(e) => {
tracing::warn!(
forker = %forker_did,
fork = %fork_name,
status,
err = %e,
"fork refused retryably: the store could not be asked whether the \
archive under the fork's key is this attempt's own"
);
return Err(AppError::RepoUnavailable);
}
}
}
crate::git::tigris::UploadError::NotPublished(other) => {
Expand Down Expand Up @@ -11347,6 +11372,227 @@ mod tests {
server.abort();
}

// ── #285 U3: the abandoned write path settles the live tree ────────────
//
// The sibling above proves the replication TAIL is withheld when a push is
// cancelled before its PUT is dispatched. Nothing yet stops the next READ
// from serving the very refs that tail refused to announce: the write guard
// drops without classifying the tree, so `acquire` finds a live directory,
// no marker, and hands it out.

/// The sidecar marker path for a live tree, derived the way production
/// derives it (`.{name}.git.quarantine` beside the validated path) rather
/// than returned by the state builder.
#[cfg(unix)]
fn p3_quarantine_marker(
repos_dir: &std::path::Path,
owner_did: &str,
name: &str,
) -> std::path::PathBuf {
let live = crate::git::repo_store::validated_repo_disk_path(repos_dir, owner_did, name)
.expect("test repo path");
let file = live
.file_name()
.expect("a live tree always has a file name")
.to_string_lossy()
.to_string();
live.parent()
.expect("a live tree always has a parent")
.join(format!(".{file}.quarantine"))
}

/// #285 U3, gap-driving. A push cancelled inside the release-side
/// compression definitely never attempted publication, so the refs it wrote
/// exist only on this node's disk. The next read must not serve them.
///
/// The p3 store answers every HEAD 404, so there is nothing stored to serve
/// in their place: the correct outcome is that the abandoned tree is gone,
/// not that a stale copy is handed out.
#[cfg(unix)]
#[sqlx::test]
async fn receive_pack_cancelled_during_compression_leaves_no_servable_refs(pool: sqlx::PgPool) {
let tmp = tempfile::TempDir::new().unwrap();
let gate = Arc::new(crate::git::tigris::BlockingGate::shut());
let (state, log, puts, server) =
p3_compression_gated_state(pool, tmp.path(), "z6p3quar", "c1", Arc::clone(&gate)).await;
let rec = state.db.get_repo("z6p3quar", "c1").await.unwrap().unwrap();
let repos_dir = tmp.path().join("repos");
let live =
crate::git::repo_store::validated_repo_disk_path(&repos_dir, &rec.owner_did, "c1")
.expect("test repo path")
.into_path_buf();
let marker = p3_quarantine_marker(&repos_dir, &rec.owner_did, "c1");

let mut fut = Box::pin(p2_push(&state, "z6p3quar", "c1"));
let mut ran = false;
for _ in 0..1000 {
let step = tokio::time::timeout(std::time::Duration::from_millis(10), &mut fut).await;
assert!(
step.is_err(),
"the handler must park inside the release-side compression, not return"
);
if p2_logged(&log, "receive-pack") {
ran = true;
break;
}
}
assert!(ran, "the push must reach receive-pack");
for _ in 0..10 {
let _ = tokio::time::timeout(std::time::Duration::from_millis(10), &mut fut).await;
}
assert_eq!(
puts.load(std::sync::atomic::Ordering::SeqCst),
0,
"compression has not finished, so no PUT can have been built yet"
);

// THE DISCONNECT, inside the compression window.
drop(fut);
tokio::time::sleep(std::time::Duration::from_millis(1500)).await;
// Release the orphaned compression BEFORE the assertions. The guard has
// already dropped, so nothing below depends on the gate, and a failing
// assertion would otherwise leave a blocking task parked forever and
// hang the runtime teardown instead of reporting.
gate.open();
tokio::time::sleep(std::time::Duration::from_millis(100)).await;

assert!(
marker.exists(),
"a push cancelled before its PUT was dispatched must leave a marker beside the live tree"
);
// The read resolves the marker: a definite non-publication, so the tree
// is dropped and the stored generation (there is none here) is what
// would serve.
let _ = state.repo_store.acquire(&rec.owner_did, "c1").await;
assert!(
!live.exists(),
"a push cancelled before its PUT was dispatched left its refs servable on the live path"
);

server.abort();
}

/// #285 U3, gap-driving. The disconnect one step earlier: receive-pack has
/// already APPLIED a ref and the client goes away before `release` is ever
/// entered, so the publish stage is still `Idle`. The guard rides the
/// admission guard into the detached reaper, so it drops once the git group
/// is torn down, and at that point the tree carries refs no PUT ever
/// described.
///
/// The fake git applies the ref and then hangs, which is what puts the
/// disconnect in that window; argv is
/// `<bin> receive-pack --stateless-rpc <repo path>`, so `$3` is the tree.
#[cfg(unix)]
#[sqlx::test]
async fn receive_pack_disconnected_after_refs_are_applied_leaves_no_servable_refs(
pool: sqlx::PgPool,
) {
let tmp = tempfile::TempDir::new().unwrap();
let gate = Arc::new(crate::git::tigris::BlockingGate::shut());
let (mut state, _log, _puts, server) =
p3_compression_gated_state(pool, tmp.path(), "z6p3disc", "c1", Arc::clone(&gate)).await;
// Its own directory: `write_fake_git` always writes `fakegit`, and the
// state builder has already put the logging shim at that name.
let gitdir = tmp.path().join("hanging-git");
std::fs::create_dir_all(&gitdir).unwrap();
state.git_bin = write_fake_git(
&gitdir,
r#"#!/bin/sh
case "$1" in
receive-pack)
mkdir -p "$3/refs/heads"
printf '%s\n' 1111111111111111111111111111111111111111 > "$3/refs/heads/main"
sleep 30
;;
*) : ;;
esac
exit 0
"#,
);

let rec = state.db.get_repo("z6p3disc", "c1").await.unwrap().unwrap();
let repos_dir = tmp.path().join("repos");
let live =
crate::git::repo_store::validated_repo_disk_path(&repos_dir, &rec.owner_did, "c1")
.expect("test repo path")
.into_path_buf();
let marker = p3_quarantine_marker(&repos_dir, &rec.owner_did, "c1");
let ref_path = live.join("refs").join("heads").join("main");

let mut fut = Box::pin(p2_push(&state, "z6p3disc", "c1"));
let mut applied = false;
for _ in 0..2000 {
let step = tokio::time::timeout(std::time::Duration::from_millis(10), &mut fut).await;
assert!(
step.is_err(),
"the handler must park inside the hanging receive-pack, not return"
);
if ref_path.exists() {
applied = true;
break;
}
}
assert!(applied, "the push must apply its ref before the disconnect");

// THE DISCONNECT, after the refs landed and before release is entered.
drop(fut);
// Nothing below depends on the gate: this disconnect never reaches
// release, so no compression is parked on it. Opened here anyway so a
// failing assertion reports rather than hanging the teardown.
gate.open();

let deadline = std::time::Instant::now() + std::time::Duration::from_secs(20);
while !marker.exists() {
assert!(
std::time::Instant::now() < deadline,
"a push disconnected after receive-pack applied its refs must mark the tree, or acquire serves refs object storage does not hold"
);
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
}

let _ = state.repo_store.acquire(&rec.owner_did, "c1").await;
assert!(
!ref_path.exists(),
"refs applied by a disconnected push must not be served"
);

server.abort();
}

/// THE CONTROL for both of the above. Same builder, no disconnect: the
/// publish completes, the store acknowledges the PUT, and the tree stays
/// readable with nothing withholding it. Without this a green refusal above
/// would prove only that this harness never serves.
#[cfg(unix)]
#[sqlx::test]
async fn receive_pack_that_completes_its_publish_leaves_the_tree_servable(pool: sqlx::PgPool) {
let tmp = tempfile::TempDir::new().unwrap();
let gate = Arc::new(crate::git::tigris::BlockingGate::shut());
gate.open();
let (state, _log, puts, server) =
p3_compression_gated_state(pool, tmp.path(), "z6p3serv", "c1", gate).await;
let rec = state.db.get_repo("z6p3serv", "c1").await.unwrap().unwrap();
let repos_dir = tmp.path().join("repos");
let marker = p3_quarantine_marker(&repos_dir, &rec.owner_did, "c1");

p2_push(&state, "z6p3serv", "c1")
.await
.expect("an uncontended push must succeed");
assert_eq!(
puts.load(std::sync::atomic::Ordering::SeqCst),
1,
"the release must have published exactly once"
);

let served = state.repo_store.acquire(&rec.owner_did, "c1").await;
assert!(
served.is_ok() && !marker.exists(),
"a confirmed publish must leave the tree readable with no marker"
);

server.abort();
}

// ── #285 P1 finding 3: fork confirmation is bound to the attempt ───────

/// A repo row under `owner/name` owned by some OTHER attempt.
Expand Down
Loading
Loading