Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
57 commits
Select commit Hold shift + click to select a range
7af3067
fix(node): close the advisory-lock probe's session if it is dropped m…
beardthelion Jul 29, 2026
edcf0f2
fix(node): add a dedicated, lazily-connected advisory-lock pool
beardthelion Jul 29, 2026
5ce3c7f
fix(node): hold the lock-owning connection in RepoWriteGuard (#279)
beardthelion Jul 29, 2026
c49d3cd
fix(node): free the advisory lock when a write guard dies without rel…
beardthelion Jul 29, 2026
98f2daf
fix(node): read the advisory unlock's result instead of discarding it
beardthelion Jul 29, 2026
9321d3b
fix(node): bound the object-storage transfers that run under the writ…
beardthelion Jul 29, 2026
9da0939
fix(node): authorize before taking the write lock in close_issue
beardthelion Jul 29, 2026
5d254fb
fix(node): address the review findings on the advisory-lock series
beardthelion Jul 30, 2026
d346bde
fix(node): log lock-pool saturation in the request path, not via read…
beardthelion Jul 30, 2026
82143ad
fix(node): refuse a write when object storage is unknowable, and shed…
beardthelion Jul 30, 2026
8cf6b7a
fix(node): re-authorize close_issue under the guard instead of only r…
beardthelion Jul 30, 2026
af5948b
style(node): rustfmt the new contention test
beardthelion Jul 30, 2026
e5558fb
fix(node): shed an unrefreshable write as a retryable 503 with a fixe…
beardthelion Aug 3, 2026
63a9ef8
fix(node): refuse a fresh-copy write when the archive HEAD cannot be …
beardthelion Aug 3, 2026
10230ac
fix(node): log expected transient acquire failures at warn, not error
beardthelion Aug 3, 2026
259d587
fix(node): log the read failure the close_issue pre-check folds into …
beardthelion Aug 3, 2026
999ee72
test(node): poll for the closed session instead of sleeping a fixed 3…
beardthelion Aug 3, 2026
5fdb902
fix(node): raise a failed fresh download with no local copy as RepoUn…
beardthelion Aug 9, 2026
f81ebbe
test(node): assert the second writer sheds as RepoBusy instead of tim…
beardthelion Aug 9, 2026
c4dbd8a
fix(node): read the close_issue author pre-check from a non-mutating …
beardthelion Aug 10, 2026
9f313b9
fix(node): bound every await in the lock-acquire loop by the deadline
beardthelion Aug 10, 2026
60d70c7
test(node): add an S3 mock that really enforces conditional writes
beardthelion Aug 10, 2026
31f90c4
fix(node): stop pretending a held lock fences a late upload
beardthelion Aug 10, 2026
3eabf79
feat(node): let the storage client express and detect a conditional w…
beardthelion Aug 10, 2026
317841e
fix(node): fence the release publish so an abandoned upload cannot win
beardthelion Aug 10, 2026
3d803ff
test(node): drive the abandoned-upload race end to end, and probe the…
beardthelion Aug 10, 2026
2a92a1f
fix(node): reconcile F-series tests and test seams with the rebased g…
beardthelion Aug 18, 2026
9f35dca
fix(node): address the 2026-08-10 review round on the advisory-lock s…
beardthelion Aug 18, 2026
9dd71e9
fix(node): record why the close-issue read gate cannot carry a synced…
beardthelion Aug 18, 2026
4cee7e0
merge origin/main into fix/279-advisory-lock-session-affinity
beardthelion Aug 26, 2026
b35cb31
fix(node): address jatmn review on merged advisory-lock head
beardthelion Aug 26, 2026
3c20148
fix(node): cap close_issue snapshot work on the read pool
beardthelion Aug 26, 2026
7ba0fef
fix(node): mark close_issue mirror visibility gate for push hook
beardthelion Aug 26, 2026
3df71c9
fix: unblock pre-push after main merge
beardthelion Aug 26, 2026
98fe670
fix(node): carry transient acquire failures through every write handler
beardthelion Aug 28, 2026
4f1c1bb
fix(node): drop unused logcap helper after drain_faults seam
beardthelion Aug 28, 2026
4eb0c44
fix(node): route fork disk paths through validated_repo_disk_path
beardthelion Aug 28, 2026
d1ce2e3
fix(node): harden release-upload outcomes and replication tail gating
beardthelion Aug 28, 2026
8e66e77
fix(node): close jatmn round on publish authority and durability gating
beardthelion Aug 29, 2026
1911291
fix(node): harden issue close/create and fork publish paths on #285
beardthelion Aug 29, 2026
5d3a4ea
fix(node): close write-attempt ownership gaps on advisory-lock release
beardthelion Aug 29, 2026
2ad2f12
fix(node): gate replication tail on publish durability and split clos…
beardthelion Aug 29, 2026
487687c
fix(node): add ValidatedRepoDiskPath for CodeQL and reconcile tail du…
beardthelion Aug 29, 2026
9d02139
fix(node): close three jatmn findings on advisory-lock PR
beardthelion Aug 30, 2026
85c6f48
fix(node): verify adversarial findings on advisory-lock PR
beardthelion Aug 30, 2026
5cd6703
fix(node): compensate under the write lock and harden tail durability…
beardthelion Aug 30, 2026
22da2d6
fix(node): refuse stale under-lock refresh and gate tail durability o…
beardthelion Aug 30, 2026
c306933
style(node): rustfmt advisory-lock gap-closure commits
beardthelion Aug 30, 2026
3a82ccb
fix(node): satisfy clippy on stale-local download test path join
beardthelion Aug 30, 2026
c40f299
feat(node): add a publish attempt/stage boundary
kevincodex1 Aug 31, 2026
21646bc
feat(node): stamp uploads with an attempt id and preserve dispatch am…
kevincodex1 Aug 31, 2026
5e548d2
fix(node): key durability, cache validity and fork cleanup on the att…
kevincodex1 Aug 31, 2026
535c5dc
fix(node): settle the cache contract on the abandoned path and the at…
beardthelion Sep 7, 2026
f048a1e
fix(node): settle the abandoned tree at the disconnect, not at reaper…
beardthelion Sep 7, 2026
957d50b
fix(node): let exactly one classifier settle an abandoned tree
beardthelion Sep 7, 2026
892c752
Merge origin/main into fix/279-advisory-lock-session-affinity
beardthelion Nov 4, 2026
28f3fc2
test(node): use the test's own pool now that Db::pool is gone
beardthelion Nov 4, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
fix(node): settle the abandoned tree at the disconnect, not at reaper…
… exit

The settlement that marks an abandoned write ran from RepoWriteGuard's
Drop, and on a client disconnect that guard is held by the Arc clone
riding the admission guard into the detached reaper, so it ran only once
the process group had been reaped. Reads take no advisory lock and
consult only the sidecar, so for the width of that window a concurrent
fetch was served the abandoned refs off the live tree with no marker
beside it. Confirmed by execution before the fix: no marker on disk,
acquire returning the live path, and the abandoned ref still on the tree
it served.

Only the lock release has to wait for the reaper; the settlement does
not. TreeSettlement is a token carrying the path, the publish stage and
the hand-out flag, holding the exhaustive stage match that used to live
in Drop. The handler takes it out of the guard before the guard is
shared with the reaper, so on a disconnect the handler future drops
first and the marker lands at the disconnect instant while the reaper
still holds the lock. A successful release disarms it. Drop still
settles a guard nobody took the token from, so the arms have one home.

The regression test reads inside the window and asserts the abandoned
refs are not served. It samples pg_locks from an independent session
first, so a run where the reaper had already finished reports itself
inconclusive rather than passing for the wrong reason.

Suite 1212 passed, 0 failed, 1 ignored. The five settlement mutations
are load-bearing, including one that reverts this split and reddens the
window test.
  • Loading branch information
beardthelion committed Sep 7, 2026
commit f048a1e50171438a147fbf539da3240e0e07ae1d
130 changes: 129 additions & 1 deletion crates/gitlawb-node/src/api/repos.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2278,7 +2278,7 @@ pub async fn git_receive_pack(
// permit is a handler local here (moved into the AdmissionGuard only after this),
// so the early return on timeout drops it and frees the slot; shed a bounded 503.
let acquire_deadline = std::time::Duration::from_secs(state.config.git_acquire_timeout_secs);
let guard = tokio::time::timeout(
let mut guard = tokio::time::timeout(
acquire_deadline,
state
.repo_store
Expand Down Expand Up @@ -2314,6 +2314,17 @@ pub async fn git_receive_pack(
// this copy dies with the future and the reaper's copy is last, so the lock frees
// after the reap with no upload, which is the release(success = false) semantics an
// interrupted push must have.
// THE SPLIT (#285 U3). Settling the tree and freeing the advisory lock do not
// have to happen at the same moment, and only the unlock has to wait for the
// reaper. The guard below is shared, so on a disconnect its last copy is the
// one riding `KillGroupOnDrop`, and a settlement done from `RepoWriteGuard::drop`
// would therefore land only after the process group is torn down. `acquire`
// takes no advisory lock on its way to the live tree, so a read inside that
// window would be served refs no publish ever confirmed. Holding the
// settlement in THIS future puts the marker on disk at the disconnect instant,
// while the reaper still holds the lock. The success path disarms it below,
// after `release` has classified the tree itself.
let mut settlement = guard.take_settlement();
let guard = std::sync::Arc::new(std::sync::Mutex::new(Some(guard)));
// Clone (a) of the write lease rides this AdmissionGuard: on a client disconnect the
// guard moves into KillGroupOnDrop's detached reaper, so the lease frees only after
Expand Down Expand Up @@ -2405,6 +2416,12 @@ pub async fn git_receive_pack(
// certificates or answering 200 would all be reporting a write no other
// node can read.
let outcome = reclaimed.release(push_succeeded).await;
// `release` classified the tree on whatever outcome it reached, so this
// future's copy of the settlement must not classify it a second time: a
// definite refusal has already deleted the tree and cleared its marker.
if let Some(settlement) = settlement.as_mut() {
settlement.disarm();
}
if push_succeeded {
publish_durability.record(outcome).await;
}
Expand Down Expand Up @@ -11559,6 +11576,117 @@ exit 0
server.abort();
}

/// #285 U3, the REAP WINDOW. The sibling above waits for the marker before
/// it reads, so it proves the settled state and cannot see the interval
/// between the disconnect and the settlement. This one reads inside that
/// interval: `RepoWriteGuard::drop` runs on the guard clone that rode the
/// admission guard into `KillGroupOnDrop`'s detached reaper, so the marker
/// cannot exist until the receive-pack group is torn down, and `acquire`
/// takes no advisory lock on the way to the live tree.
///
/// The read happens immediately after the handler future is dropped, while
/// the fake receive-pack is still sleeping, so the guard is still alive in
/// the reaper.
#[cfg(unix)]
#[sqlx::test]
async fn receive_pack_read_inside_the_reap_window_serves_no_abandoned_refs(pool: sqlx::PgPool) {
let tmp = tempfile::TempDir::new().unwrap();
let gate = Arc::new(crate::git::tigris::BlockingGate::shut());
let (mut state, _log, _puts, server) = p3_compression_gated_state(
pool.clone(),
tmp.path(),
"z6p3rwin",
"c1",
Arc::clone(&gate),
)
.await;
let gitdir = tmp.path().join("hanging-git");
std::fs::create_dir_all(&gitdir).unwrap();
state.git_bin = write_fake_git(
&gitdir,
r#"#!/bin/sh
case "$1" in
receive-pack)
mkdir -p "$3/refs/heads"
printf '%s\n' 1111111111111111111111111111111111111111 > "$3/refs/heads/main"
sleep 30
;;
*) : ;;
esac
exit 0
"#,
);

let rec = state.db.get_repo("z6p3rwin", "c1").await.unwrap().unwrap();
let repos_dir = tmp.path().join("repos");
let marker = p3_quarantine_marker(&repos_dir, &rec.owner_did, "c1");
let live =
crate::git::repo_store::validated_repo_disk_path(&repos_dir, &rec.owner_did, "c1")
.expect("test repo path")
.into_path_buf();
let ref_path = live.join("refs").join("heads").join("main");

let mut fut = Box::pin(p2_push(&state, "z6p3rwin", "c1"));
let mut applied = false;
for _ in 0..2000 {
let step = tokio::time::timeout(std::time::Duration::from_millis(10), &mut fut).await;
assert!(
step.is_err(),
"the handler must park inside the hanging receive-pack, not return"
);
if ref_path.exists() {
applied = true;
break;
}
}
assert!(applied, "the push must apply its ref before the disconnect");

// THE DISCONNECT. Nothing parks on the gate here, opened only so a
// failing assertion reports instead of hanging teardown.
drop(fut);
gate.open();
tokio::task::yield_now().await;

// THE PROBE, inside the window. What makes the read INSIDE it is that the
// receive-pack group is still being reaped, and the write lock rides that
// reaper: the guard's last copy is held by `KillGroupOnDrop`, so the
// advisory lock is still taken at this instant. Sampled before the read,
// from an independent session, and filtered to this test's own database,
// which `#[sqlx::test]` gives it exclusively.
//
// Keying conclusiveness on the LOCK rather than on the marker's absence:
// the marker is what the settlement writes, so requiring it to be missing
// would demand the very state the fix removes, and no correct
// implementation could satisfy it.
let locks_held: (i64,) = sqlx::query_as(
"SELECT count(*) FROM pg_locks WHERE locktype = 'advisory' \
AND database = (SELECT oid FROM pg_database WHERE datname = current_database())",
)
.fetch_one(&pool)
.await
.expect("count the advisory locks this test's database holds");
let marker_before = marker.exists();
let served = state.repo_store.acquire(&rec.owner_did, "c1").await;
let refs_after = ref_path.exists();
let served_ok = served.is_ok();

assert!(
locks_held.0 >= 1,
"INCONCLUSIVE: the write lock was already free when the probe ran, so the \
reaper had finished and this read never observed the reap window"
);
assert!(
!(served_ok && refs_after),
"a read inside the reap window served the abandoned refs: marker_before={}, \
acquire={:?}, ref still on the served tree={}",
marker_before,
served,
refs_after
);

server.abort();
}

/// THE CONTROL for both of the above. Same builder, no disconnect: the
/// publish completes, the store acknowledges the PUT, and the tree stays
/// readable with nothing withholding it. Without this a green refusal above
Expand Down
185 changes: 142 additions & 43 deletions crates/gitlawb-node/src/git/repo_store.rs
Original file line number Diff line number Diff line change
Expand Up @@ -669,7 +669,7 @@ impl RepoStore {
None => PublishStage::NoBackend,
})),
tree_settled: false,
path_handed_out: AtomicBool::new(false),
path_handed_out: Arc::new(AtomicBool::new(false)),
#[cfg(test)]
test_pre_unlock_gate: self.pre_unlock_gate.clone(),
#[cfg(test)]
Expand Down Expand Up @@ -1908,8 +1908,9 @@ pub struct RepoWriteGuard {
/// production writer obtains the tree through `path()`, so obtaining it IS
/// the declaration and no call site has a rule to remember.
///
/// An atomic rather than a `Cell` because `path()` takes `&self`.
path_handed_out: AtomicBool,
/// An atomic rather than a `Cell` because `path()` takes `&self`, and
/// shared with the settlement token so a hand-out is seen from either side.
path_handed_out: Arc<AtomicBool>,
/// Test-only seam: when set, `release` parks on this gate at the exact point
/// it is about to await `pg_advisory_unlock` (connection still owned, not yet
/// released). Dropping the `release` future while it is parked reproduces a
Expand Down Expand Up @@ -2060,6 +2061,36 @@ impl RepoWriteGuard {
Arc::clone(&self.publish_stage)
}

/// Take the tree settlement out of this guard.
///
/// The caller becomes responsible for classifying the live tree an
/// abandoned write leaves behind, and this guard stops doing it. That is
/// the point: a shared guard frees its advisory lock only when the last
/// copy drops, which on a disconnect is the detached reaper, whereas the
/// marker has to be on disk before the next read reaches the live tree.
///
/// `None` once the tree is settled: `release` classifies it on every
/// outcome it reaches, and there is nothing left to hand out.
pub fn take_settlement(&mut self) -> Option<TreeSettlement> {
if self.tree_settled {
return None;
}
self.tree_settled = true;
Some(self.settlement())
}

/// An armed settlement over this guard's tree. Shares the publish stage and
/// the hand-out flag, so it reads the same state the guard would.
fn settlement(&self) -> TreeSettlement {
TreeSettlement {
local_path: self.local_path.clone(),
repo_name: self.repo_name.clone(),
publish_stage: Arc::clone(&self.publish_stage),
path_handed_out: Arc::clone(&self.path_handed_out),
armed: true,
}
}

/// Decide what a publish that ran out its bound actually left behind.
///
/// "The bounded transfer returned `None`" is not one state. `publish` awaits
Expand Down Expand Up @@ -2344,6 +2375,92 @@ impl RepoWriteGuard {
}
}

/// The half of a write guard's settlement that does not need the advisory lock.
///
/// A `RepoWriteGuard` can be shared, and the lock frees only when the LAST copy
/// drops. On a receive-pack disconnect that copy rides the admission guard into
/// the detached process reaper, so a settlement done from `Drop` lands only
/// after the process group is torn down. `acquire` takes no advisory lock on
/// the way to the live tree, so a read inside that window is served the
/// abandoned refs.
///
/// Only the UNLOCK has to wait for the reaper. This token carries everything
/// the settlement itself needs, so the request future can hold it and settle
/// the tree at the disconnect instant while the reaper still holds the lock.
pub(crate) struct TreeSettlement {
local_path: ValidatedRepoDiskPath,
repo_name: String,
/// Shared with the guard, so the stage read here is the stage the publish
/// actually reached, not a copy taken when the token was handed out.
publish_stage: Arc<PublishStageCell>,
path_handed_out: Arc<AtomicBool>,
armed: bool,
}

impl TreeSettlement {
/// `release` classified the tree itself, so this token has nothing to do.
///
/// Disarming rather than forgetting the token: a settlement that ran after
/// a definite refusal would write a marker beside a tree the refusal has
/// already deleted.
pub(crate) fn disarm(&mut self) {
self.armed = false;
}

/// Classify the live tree an abandoned write left behind.
///
/// Exhaustive on purpose, no wildcard arm: a new stage must be classified
/// here rather than inheriting whatever the catch-all happened to do.
fn settle(&mut self) {
if !self.armed {
return;
}
self.armed = false;
let stage = self.publish_stage.get();
match &stage {
// The store acknowledged this write, so the live tree IS the
// stored generation. Mirrors `release`'s `Released` arm.
PublishStage::Published { .. } => clear_quarantine(&self.local_path, &self.repo_name),
// Nothing to publish to: the local write is the durable copy.
PublishStage::NoBackend => {}
PublishStage::Idle => {
// A publish was possible and never started. If the tree was
// handed out, refs may already be on it (the receive-pack
// disconnect: git applied them, the reaper killed the group,
// and the guard dropped before `release` was entered), and
// nothing dispatched, so this is a definite non-publication.
// If it was never handed out, nothing touched the tree.
if self.path_handed_out.load(Ordering::Acquire) {
quarantine_local_tree(&self.local_path, &self.repo_name, None);
}
}
// A definite non-publication after a successful write. The marker
// names no attempt, and the next read resolves it by invalidating
// the cache and serving the stored generation. Deliberately NOT a
// `remove_dir_all` here: that would run a blocking delete inside
// `Drop` on a runtime worker, when the read path already runs one.
PublishStage::PreparingArchive | PublishStage::Refused => {
quarantine_local_tree(&self.local_path, &self.repo_name, None)
}
// The PUT may have committed. The marker carries the attempt, so
// a landed PUT can still lift it through reconciliation.
PublishStage::PutDispatched { .. } | PublishStage::Ambiguous { .. } => {
quarantine_local_tree(
&self.local_path,
&self.repo_name,
stage.unresolved_attempt(),
)
}
}
}
}

impl Drop for TreeSettlement {
fn drop(&mut self) {
self.settle();
}
}

impl Drop for RepoWriteGuard {
fn drop(&mut self) {
if let Some(authority) = self.refresh_swap_authority.take() {
Expand All @@ -2358,45 +2475,15 @@ impl Drop for RepoWriteGuard {
//
// Exhaustive on purpose, no wildcard arm: a new stage must be classified
// here rather than inheriting whatever the catch-all happened to do.
// SETTLE THE TREE, if the handler did not take the settlement out.
//
// `take_settlement` exists because the lock and the tree do not have to
// be freed at the same moment; when it was called this guard's copy has
// nothing left to classify. Otherwise this is the only classifier a
// guard that never reached `release` gets.
if !self.tree_settled {
let stage = self.publish_stage.get();
match &stage {
// The store acknowledged this write, so the live tree IS the
// stored generation. Mirrors `release`'s `Released` arm.
PublishStage::Published { .. } => {
clear_quarantine(&self.local_path, &self.repo_name)
}
// Nothing to publish to: the local write is the durable copy.
PublishStage::NoBackend => {}
PublishStage::Idle => {
// A publish was possible and never started. If the tree was
// handed out, refs may already be on it (the receive-pack
// disconnect: git applied them, the reaper killed the group,
// and the guard dropped before `release` was entered), and
// nothing dispatched, so this is a definite non-publication.
// If it was never handed out, nothing touched the tree.
if self.path_handed_out.load(Ordering::Acquire) {
quarantine_local_tree(&self.local_path, &self.repo_name, None);
}
}
// A definite non-publication after a successful write. The marker
// names no attempt, and the next read resolves it by invalidating
// the cache and serving the stored generation. Deliberately NOT a
// `remove_dir_all` here: that would run a blocking delete inside
// `Drop` on a runtime worker, when the read path already runs one.
PublishStage::PreparingArchive | PublishStage::Refused => {
quarantine_local_tree(&self.local_path, &self.repo_name, None)
}
// The PUT may have committed. The marker carries the attempt, so
// a landed PUT can still lift it through reconciliation.
PublishStage::PutDispatched { .. } | PublishStage::Ambiguous { .. } => {
quarantine_local_tree(
&self.local_path,
&self.repo_name,
stage.unresolved_attempt(),
)
}
}
let mut settlement = self.settlement();
settlement.settle();
}

let Some(mut conn) = self.conn.take() else {
Expand Down Expand Up @@ -3728,7 +3815,7 @@ mod tests {
refresh_swap_authority: None,
publish_stage: Arc::new(PublishStageCell::new()),
tree_settled: false,
path_handed_out: AtomicBool::new(false),
path_handed_out: Arc::new(AtomicBool::new(false)),
#[cfg(test)]
test_pre_unlock_gate: None,
#[cfg(test)]
Expand Down Expand Up @@ -4098,7 +4185,7 @@ mod tests {
refresh_swap_authority: None,
publish_stage: Arc::new(PublishStageCell::new()),
tree_settled: false,
path_handed_out: AtomicBool::new(false),
path_handed_out: Arc::new(AtomicBool::new(false)),
#[cfg(test)]
test_pre_unlock_gate: None,
#[cfg(test)]
Expand Down Expand Up @@ -8529,6 +8616,18 @@ mod tests {
marker_path.exists(),
"the second quarantine must reach disk once the path is writable"
);
// THE PROPERTY, observed BEFORE any clear. `clear_quarantine` drops the
// map entry ahead of the file, so an assertion taken after one passes
// whatever the successful-write arm did with the shadow. This is the
// only point where the shadow is observable.
assert!(
!unwritten_quarantines()
.lock()
.expect("quarantine map poisoned")
.contains_key(&marker_path),
"a marker that reached disk left the in-memory shadow of the earlier failed \
write behind it, so the disk marker is no longer the authority for this path"
);

// Lift the second attempt and clear the file the way a confirmation
// does. Nothing may still be refusing after that.
Expand Down
Loading