Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
57 commits
Select commit Hold shift + click to select a range
7af3067
fix(node): close the advisory-lock probe's session if it is dropped m…
beardthelion Jul 29, 2026
edcf0f2
fix(node): add a dedicated, lazily-connected advisory-lock pool
beardthelion Jul 29, 2026
5ce3c7f
fix(node): hold the lock-owning connection in RepoWriteGuard (#279)
beardthelion Jul 29, 2026
c49d3cd
fix(node): free the advisory lock when a write guard dies without rel…
beardthelion Jul 29, 2026
98f2daf
fix(node): read the advisory unlock's result instead of discarding it
beardthelion Jul 29, 2026
9321d3b
fix(node): bound the object-storage transfers that run under the writ…
beardthelion Jul 29, 2026
9da0939
fix(node): authorize before taking the write lock in close_issue
beardthelion Jul 29, 2026
5d254fb
fix(node): address the review findings on the advisory-lock series
beardthelion Jul 30, 2026
d346bde
fix(node): log lock-pool saturation in the request path, not via read…
beardthelion Jul 30, 2026
82143ad
fix(node): refuse a write when object storage is unknowable, and shed…
beardthelion Jul 30, 2026
8cf6b7a
fix(node): re-authorize close_issue under the guard instead of only r…
beardthelion Jul 30, 2026
af5948b
style(node): rustfmt the new contention test
beardthelion Jul 30, 2026
e5558fb
fix(node): shed an unrefreshable write as a retryable 503 with a fixe…
beardthelion Aug 3, 2026
63a9ef8
fix(node): refuse a fresh-copy write when the archive HEAD cannot be …
beardthelion Aug 3, 2026
10230ac
fix(node): log expected transient acquire failures at warn, not error
beardthelion Aug 3, 2026
259d587
fix(node): log the read failure the close_issue pre-check folds into …
beardthelion Aug 3, 2026
999ee72
test(node): poll for the closed session instead of sleeping a fixed 3…
beardthelion Aug 3, 2026
5fdb902
fix(node): raise a failed fresh download with no local copy as RepoUn…
beardthelion Aug 9, 2026
f81ebbe
test(node): assert the second writer sheds as RepoBusy instead of tim…
beardthelion Aug 9, 2026
c4dbd8a
fix(node): read the close_issue author pre-check from a non-mutating …
beardthelion Aug 10, 2026
9f313b9
fix(node): bound every await in the lock-acquire loop by the deadline
beardthelion Aug 10, 2026
60d70c7
test(node): add an S3 mock that really enforces conditional writes
beardthelion Aug 10, 2026
31f90c4
fix(node): stop pretending a held lock fences a late upload
beardthelion Aug 10, 2026
3eabf79
feat(node): let the storage client express and detect a conditional w…
beardthelion Aug 10, 2026
317841e
fix(node): fence the release publish so an abandoned upload cannot win
beardthelion Aug 10, 2026
3d803ff
test(node): drive the abandoned-upload race end to end, and probe the…
beardthelion Aug 10, 2026
2a92a1f
fix(node): reconcile F-series tests and test seams with the rebased g…
beardthelion Aug 18, 2026
9f35dca
fix(node): address the 2026-08-10 review round on the advisory-lock s…
beardthelion Aug 18, 2026
9dd71e9
fix(node): record why the close-issue read gate cannot carry a synced…
beardthelion Aug 18, 2026
4cee7e0
merge origin/main into fix/279-advisory-lock-session-affinity
beardthelion Aug 26, 2026
b35cb31
fix(node): address jatmn review on merged advisory-lock head
beardthelion Aug 26, 2026
3c20148
fix(node): cap close_issue snapshot work on the read pool
beardthelion Aug 26, 2026
7ba0fef
fix(node): mark close_issue mirror visibility gate for push hook
beardthelion Aug 26, 2026
3df71c9
fix: unblock pre-push after main merge
beardthelion Aug 26, 2026
98fe670
fix(node): carry transient acquire failures through every write handler
beardthelion Aug 28, 2026
4f1c1bb
fix(node): drop unused logcap helper after drain_faults seam
beardthelion Aug 28, 2026
4eb0c44
fix(node): route fork disk paths through validated_repo_disk_path
beardthelion Aug 28, 2026
d1ce2e3
fix(node): harden release-upload outcomes and replication tail gating
beardthelion Aug 28, 2026
8e66e77
fix(node): close jatmn round on publish authority and durability gating
beardthelion Aug 29, 2026
1911291
fix(node): harden issue close/create and fork publish paths on #285
beardthelion Aug 29, 2026
5d3a4ea
fix(node): close write-attempt ownership gaps on advisory-lock release
beardthelion Aug 29, 2026
2ad2f12
fix(node): gate replication tail on publish durability and split clos…
beardthelion Aug 29, 2026
487687c
fix(node): add ValidatedRepoDiskPath for CodeQL and reconcile tail du…
beardthelion Aug 29, 2026
9d02139
fix(node): close three jatmn findings on advisory-lock PR
beardthelion Aug 30, 2026
85c6f48
fix(node): verify adversarial findings on advisory-lock PR
beardthelion Aug 30, 2026
5cd6703
fix(node): compensate under the write lock and harden tail durability…
beardthelion Aug 30, 2026
22da2d6
fix(node): refuse stale under-lock refresh and gate tail durability o…
beardthelion Aug 30, 2026
c306933
style(node): rustfmt advisory-lock gap-closure commits
beardthelion Aug 30, 2026
3a82ccb
fix(node): satisfy clippy on stale-local download test path join
beardthelion Aug 30, 2026
c40f299
feat(node): add a publish attempt/stage boundary
kevincodex1 Aug 31, 2026
21646bc
feat(node): stamp uploads with an attempt id and preserve dispatch am…
kevincodex1 Aug 31, 2026
5e548d2
fix(node): key durability, cache validity and fork cleanup on the att…
kevincodex1 Aug 31, 2026
535c5dc
fix(node): settle the cache contract on the abandoned path and the at…
beardthelion Sep 7, 2026
f048a1e
fix(node): settle the abandoned tree at the disconnect, not at reaper…
beardthelion Sep 7, 2026
957d50b
fix(node): let exactly one classifier settle an abandoned tree
beardthelion Sep 7, 2026
892c752
Merge origin/main into fix/279-advisory-lock-session-affinity
beardthelion Nov 4, 2026
28f3fc2
test(node): use the test's own pool now that Db::pool is gone
beardthelion Nov 4, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
test(node): drive the abandoned-upload race end to end, and probe the…
… real backend

The headline test is the one that had to exist. Writer A's release is parked past
its transfer bound and returns with the outcome unknowable, B acquires and
publishes, and A's captured PUT is then replayed: the store answers 412 and B's
archive survives. The create-only variant covers the arm whose real-world failure
is silent rather than loud.

The control is what makes those attributable. With no interleaved B, an
abandoned-then-replayed PUT whose generation still matches lands. Without it the
headline would only show that replays get rejected, not that staleness is what
rejects them.

Header assertions come last in all three, so a lost fence reds on the outcome it
is about rather than on a wire-format check.

A mock cannot prove Tigris honors any of this, and the vendor requires a
Single-region or Multi-region bucket for conditional operations, so against a
Global or Dual-region bucket the fence is a silent no-op. The credentials-gated
probe checks both arms against the real endpoint and cleans up unconditionally,
including when an assertion fails, which is the case it exists to catch. It
accepts 412 or 409 on the create-only arm because both mean the precondition was
enforced and both are already classified as a loss.
  • Loading branch information
beardthelion committed Aug 18, 2026
commit 3d803ff217b04fd43fa49e7776296d4bb9349d7c
290 changes: 290 additions & 0 deletions crates/gitlawb-node/src/git/repo_store.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4953,4 +4953,294 @@ mod tests {

mock.shutdown();
}

// ── the abandoned writer's late PUT (#279) ─────────────────────────────

/// A store whose under-lock transfer bound is short, so a parked PUT
/// actually runs the release past its budget instead of making the test sit
/// out `fenced_store`'s 30s. The acquire-side refresh shares the bound,
/// which is fine here: it moves a few KiB against an in-process mock.
async fn fenced_store_with_bound(
mock: &S3Mock,
opts: &sqlx::postgres::PgConnectOptions,
repos_dir: &Path,
bound: std::time::Duration,
) -> RepoStore {
RepoStore::new(
repos_dir.to_path_buf(),
Some(TigrisClient::for_testing_with_endpoint(
"test-bucket",
mock.endpoint(),
)),
no_reap_pool(opts, 2).await,
bound,
)
}

/// The statuses of every logged PUT attempt, which is what the
/// abandoned-writer tests assert their attempt counts on.
fn attempt_statuses(mock: &S3Mock) -> Vec<Option<u16>> {
mock.put_attempts().iter().map(|a| a.status).collect()
}

/// THE HEADLINE ARM. An abandoned writer's PUT that lands after a successor
/// has published must be refused by the store, and the successor's archive
/// must survive it.
///
/// This is the whole point of the change. Dropping the future of an
/// in-flight PUT does not cancel the request the server is already
/// processing, so the advisory lock cannot fence it: A's release returns,
/// the lock frees, B acquires and publishes, and A's bytes are still on
/// their way to a store that has already moved on. Only the conditional PUT
/// decides that race, and it decides it at COMMIT time, not at arrival.
///
/// The mock models that by capturing A's PUT when it arrives and evaluating
/// it on replay, against the state as of the replay. That is deliberately
/// not a timing test: a parked handler whose client has gone away is
/// cancelled with the connection, so a test that waited for it to resume on
/// its own would be waiting on nothing.
#[sqlx::test]
async fn an_abandoned_writers_late_put_loses_to_the_successor(pool: PgPool) {
let _sink = log_sink();
let mock = S3Mock::start().await;
let opts = (*pool.connect_options()).clone();
let repos = TempDir::new().unwrap();
let bound = std::time::Duration::from_millis(750);
let store = fenced_store_with_bound(&mock, &opts, repos.path(), bound).await;
let owner = "did:key:z6MkFenceLatePut";
let repo = "fence-late-put-repo";
let slug = owner_slug_of(owner);

// Seed the key, so A's acquire observes a generation and carries
// If-Match on it. This is the ordinary case: the repo already exists.
let seed = TempDir::new().unwrap();
marked_repo(seed.path(), "seed");
mock_tigris(&mock)
.upload(&slug, repo, seed.path(), UploadPrecondition::Unconditional)
.await
.expect("seeding the archive");
let seeded = mock.current_etag().expect("the seed minted an ETag");

// Writer A takes the lock, writes its tree, and has its publish parked
// past the transfer bound.
let guard_a = store.acquire_write(owner, repo).await.expect("A acquires");
std::fs::write(guard_a.local_path.join("MARKER"), "writer-a").unwrap();
mock.park_next_put();
let started = std::time::Instant::now();
let outcome_a = guard_a.release(true).await;
assert!(
started.elapsed() >= bound,
"A's release must have run out its transfer bound with the PUT in flight"
);
// A timeout is UNKNOWABLE rather than failed, so the release reports an
// ordinary success and the lock frees. That is exactly why the fence has
// to live in the store: nothing here knows A's bytes are still coming.
assert!(
matches!(outcome_a, ReleaseOutcome::Released),
"an abandoned publish reports a plain release, got {outcome_a:?}"
);

assert!(
mock.captured_put().is_some(),
"A's PUT must have arrived and been captured before the bound elapsed"
);

// Writer B acquires the freed lock and publishes for real.
let b_started = std::time::Instant::now();
let guard_b = store
.acquire_write(owner, repo)
.await
.expect("B must acquire once A's release frees the lock");
assert!(
b_started.elapsed() < std::time::Duration::from_secs(5),
"B's acquire must be prompt, not blocked behind A's abandoned transfer"
);
std::fs::write(guard_b.local_path.join("MARKER"), "writer-b").unwrap();
guard_b
.release(true)
.await
.into_result()
.expect("B's publish is the one that must land");
let after_b = mock.current_etag().expect("B's publish minted an ETag");
assert_ne!(
unquote_etag(&after_b),
unquote_etag(&seeded),
"B's publish must have moved the generation on"
);

// NOW A's bytes reach the store's commit point.
assert_eq!(
mock.replay_captured(),
412,
"A's late PUT must be refused: the generation it fenced on is gone"
);
assert_eq!(
stored_marker(&mock, &slug, repo).await,
"writer-b",
"the successor's archive must survive the abandoned writer's late PUT"
);
assert_eq!(
mock.current_etag().as_deref().map(unquote_etag),
Some(unquote_etag(&after_b)),
"a refused PUT must not rotate the generation either"
);

// Seed, A's parked PUT, B's publish, the deliberate replay. A never
// attempted a second PUT of its own: the timeout arm takes no
// compensating action precisely because the outcome is unknowable.
assert_eq!(
attempt_statuses(&mock),
vec![Some(200), None, Some(200), Some(412)],
"got {:?}",
mock.put_attempts()
);

// The header detail comes LAST on purpose. Asserting it up front would
// make a lost fence red here, on a wire-format check, rather than on the
// outcome above, and the outcome is what this test is for.
let captured = mock.captured_put().expect("A's PUT was captured");
assert_eq!(
captured.if_match.as_deref().map(unquote_etag),
Some(unquote_etag(&seeded)),
"A's in-flight PUT must carry the generation it observed under the lock"
);
assert_eq!(captured.if_none_match, None);

mock.open_gate();
mock.shutdown();
}

/// The create-only arm of the same race, and the one whose real-world
/// failure mode is SILENT: an ignored If-None-Match just returns 200, so a
/// publish that should have been fenced lands with no error anywhere.
#[sqlx::test]
async fn an_abandoned_writers_late_create_only_put_loses_to_the_successor(pool: PgPool) {
let _sink = log_sink();
let mock = S3Mock::start().await;
let opts = (*pool.connect_options()).clone();
let repos = TempDir::new().unwrap();
let bound = std::time::Duration::from_millis(750);
let store = fenced_store_with_bound(&mock, &opts, repos.path(), bound).await;
let owner = "did:key:z6MkFenceLateCreate";
let repo = "fence-late-create-repo";
let slug = owner_slug_of(owner);

// Nothing stored, so A's acquire records the absent case and is fenced
// create-only.
let guard_a = store.acquire_write(owner, repo).await.expect("A acquires");
marked_repo(&guard_a.local_path, "writer-a");
mock.park_next_put();
let outcome_a = guard_a.release(true).await;
assert!(
matches!(outcome_a, ReleaseOutcome::Released),
"an abandoned publish reports a plain release, got {outcome_a:?}"
);

assert!(
mock.captured_put().is_some(),
"A's PUT must have arrived and been captured before the bound elapsed"
);

// B wins the empty key.
let guard_b = store
.acquire_write(owner, repo)
.await
.expect("B must acquire once A's release frees the lock");
std::fs::write(guard_b.local_path.join("MARKER"), "writer-b").unwrap();
guard_b
.release(true)
.await
.into_result()
.expect("B's create must land");

assert_eq!(
mock.replay_captured(),
412,
"A's late create-only PUT must be refused now that the key exists"
);
assert_eq!(
stored_marker(&mock, &slug, repo).await,
"writer-b",
"the successor's archive must survive the abandoned create-only PUT"
);
assert_eq!(
attempt_statuses(&mock),
vec![None, Some(200), Some(412)],
"got {:?}",
mock.put_attempts()
);

// Last, for the same reason as the If-Match arm: the outcome is the
// claim, the header is the detail.
let captured = mock.captured_put().expect("A's PUT was captured");
assert_eq!(
captured.if_none_match.as_deref(),
Some("*"),
"A's in-flight PUT must carry the create-only fence it observed"
);
assert_eq!(captured.if_match, None);

mock.open_gate();
mock.shutdown();
}

/// THE CONTROL, and it is what makes the two arms above attributable.
///
/// Same abandonment, same replay, but no successor publishes in between, so
/// the generation A fenced on is still current when its bytes commit and
/// the PUT must LAND. Without this, a green headline test would prove only
/// that replays are rejected, not that STALENESS is what rejects them.
///
/// It must therefore stay green when the carried precondition is forced
/// back to `Unconditional`: it asserts an outcome the fence does not
/// change, which is the whole reason it can attribute the others' red.
#[sqlx::test]
async fn an_abandoned_put_still_on_the_current_generation_lands(pool: PgPool) {
let _sink = log_sink();
let mock = S3Mock::start().await;
let opts = (*pool.connect_options()).clone();
let repos = TempDir::new().unwrap();
let bound = std::time::Duration::from_millis(750);
let store = fenced_store_with_bound(&mock, &opts, repos.path(), bound).await;
let owner = "did:key:z6MkFenceControl";
let repo = "fence-control-repo";
let slug = owner_slug_of(owner);

let seed = TempDir::new().unwrap();
marked_repo(seed.path(), "seed");
mock_tigris(&mock)
.upload(&slug, repo, seed.path(), UploadPrecondition::Unconditional)
.await
.expect("seeding the archive");

let guard_a = store.acquire_write(owner, repo).await.expect("A acquires");
std::fs::write(guard_a.local_path.join("MARKER"), "writer-a").unwrap();
mock.park_next_put();
let outcome_a = guard_a.release(true).await;
assert!(
matches!(outcome_a, ReleaseOutcome::Released),
"an abandoned publish reports a plain release, got {outcome_a:?}"
);

assert_eq!(
mock.replay_captured(),
200,
"with nothing published in between, the abandoned PUT is still current \
and must be accepted"
);
assert_eq!(
stored_marker(&mock, &slug, repo).await,
"writer-a",
"the accepted late PUT must be what is stored"
);
assert_eq!(
attempt_statuses(&mock),
vec![Some(200), None, Some(200)],
"got {:?}",
mock.put_attempts()
);

mock.open_gate();
mock.shutdown();
}
}
Loading