Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
57 commits
Select commit Hold shift + click to select a range
7af3067
fix(node): close the advisory-lock probe's session if it is dropped m…
beardthelion Jul 29, 2026
edcf0f2
fix(node): add a dedicated, lazily-connected advisory-lock pool
beardthelion Jul 29, 2026
5ce3c7f
fix(node): hold the lock-owning connection in RepoWriteGuard (#279)
beardthelion Jul 29, 2026
c49d3cd
fix(node): free the advisory lock when a write guard dies without rel…
beardthelion Jul 29, 2026
98f2daf
fix(node): read the advisory unlock's result instead of discarding it
beardthelion Jul 29, 2026
9321d3b
fix(node): bound the object-storage transfers that run under the writ…
beardthelion Jul 29, 2026
9da0939
fix(node): authorize before taking the write lock in close_issue
beardthelion Jul 29, 2026
5d254fb
fix(node): address the review findings on the advisory-lock series
beardthelion Jul 30, 2026
d346bde
fix(node): log lock-pool saturation in the request path, not via read…
beardthelion Jul 30, 2026
82143ad
fix(node): refuse a write when object storage is unknowable, and shed…
beardthelion Jul 30, 2026
8cf6b7a
fix(node): re-authorize close_issue under the guard instead of only r…
beardthelion Jul 30, 2026
af5948b
style(node): rustfmt the new contention test
beardthelion Jul 30, 2026
e5558fb
fix(node): shed an unrefreshable write as a retryable 503 with a fixe…
beardthelion Aug 3, 2026
63a9ef8
fix(node): refuse a fresh-copy write when the archive HEAD cannot be …
beardthelion Aug 3, 2026
10230ac
fix(node): log expected transient acquire failures at warn, not error
beardthelion Aug 3, 2026
259d587
fix(node): log the read failure the close_issue pre-check folds into …
beardthelion Aug 3, 2026
999ee72
test(node): poll for the closed session instead of sleeping a fixed 3…
beardthelion Aug 3, 2026
5fdb902
fix(node): raise a failed fresh download with no local copy as RepoUn…
beardthelion Aug 9, 2026
f81ebbe
test(node): assert the second writer sheds as RepoBusy instead of tim…
beardthelion Aug 9, 2026
c4dbd8a
fix(node): read the close_issue author pre-check from a non-mutating …
beardthelion Aug 10, 2026
9f313b9
fix(node): bound every await in the lock-acquire loop by the deadline
beardthelion Aug 10, 2026
60d70c7
test(node): add an S3 mock that really enforces conditional writes
beardthelion Aug 10, 2026
31f90c4
fix(node): stop pretending a held lock fences a late upload
beardthelion Aug 10, 2026
3eabf79
feat(node): let the storage client express and detect a conditional w…
beardthelion Aug 10, 2026
317841e
fix(node): fence the release publish so an abandoned upload cannot win
beardthelion Aug 10, 2026
3d803ff
test(node): drive the abandoned-upload race end to end, and probe the…
beardthelion Aug 10, 2026
2a92a1f
fix(node): reconcile F-series tests and test seams with the rebased g…
beardthelion Aug 18, 2026
9f35dca
fix(node): address the 2026-08-10 review round on the advisory-lock s…
beardthelion Aug 18, 2026
9dd71e9
fix(node): record why the close-issue read gate cannot carry a synced…
beardthelion Aug 18, 2026
4cee7e0
merge origin/main into fix/279-advisory-lock-session-affinity
beardthelion Aug 26, 2026
b35cb31
fix(node): address jatmn review on merged advisory-lock head
beardthelion Aug 26, 2026
3c20148
fix(node): cap close_issue snapshot work on the read pool
beardthelion Aug 26, 2026
7ba0fef
fix(node): mark close_issue mirror visibility gate for push hook
beardthelion Aug 26, 2026
3df71c9
fix: unblock pre-push after main merge
beardthelion Aug 26, 2026
98fe670
fix(node): carry transient acquire failures through every write handler
beardthelion Aug 28, 2026
4f1c1bb
fix(node): drop unused logcap helper after drain_faults seam
beardthelion Aug 28, 2026
4eb0c44
fix(node): route fork disk paths through validated_repo_disk_path
beardthelion Aug 28, 2026
d1ce2e3
fix(node): harden release-upload outcomes and replication tail gating
beardthelion Aug 28, 2026
8e66e77
fix(node): close jatmn round on publish authority and durability gating
beardthelion Aug 29, 2026
1911291
fix(node): harden issue close/create and fork publish paths on #285
beardthelion Aug 29, 2026
5d3a4ea
fix(node): close write-attempt ownership gaps on advisory-lock release
beardthelion Aug 29, 2026
2ad2f12
fix(node): gate replication tail on publish durability and split clos…
beardthelion Aug 29, 2026
487687c
fix(node): add ValidatedRepoDiskPath for CodeQL and reconcile tail du…
beardthelion Aug 29, 2026
9d02139
fix(node): close three jatmn findings on advisory-lock PR
beardthelion Aug 30, 2026
85c6f48
fix(node): verify adversarial findings on advisory-lock PR
beardthelion Aug 30, 2026
5cd6703
fix(node): compensate under the write lock and harden tail durability…
beardthelion Aug 30, 2026
22da2d6
fix(node): refuse stale under-lock refresh and gate tail durability o…
beardthelion Aug 30, 2026
c306933
style(node): rustfmt advisory-lock gap-closure commits
beardthelion Aug 30, 2026
3a82ccb
fix(node): satisfy clippy on stale-local download test path join
beardthelion Aug 30, 2026
c40f299
feat(node): add a publish attempt/stage boundary
kevincodex1 Aug 31, 2026
21646bc
feat(node): stamp uploads with an attempt id and preserve dispatch am…
kevincodex1 Aug 31, 2026
5e548d2
fix(node): key durability, cache validity and fork cleanup on the att…
kevincodex1 Aug 31, 2026
535c5dc
fix(node): settle the cache contract on the abandoned path and the at…
beardthelion Sep 7, 2026
f048a1e
fix(node): settle the abandoned tree at the disconnect, not at reaper…
beardthelion Sep 7, 2026
957d50b
fix(node): let exactly one classifier settle an abandoned tree
beardthelion Sep 7, 2026
892c752
Merge origin/main into fix/279-advisory-lock-session-affinity
beardthelion Nov 4, 2026
28f3fc2
test(node): use the test's own pool now that Db::pool is gone
beardthelion Nov 4, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
merge origin/main into fix/279-advisory-lock-session-affinity
Reconcile the advisory-lock series with main after #173, #330, and related
landings. Conflict resolution in repo_store, repos, issues, pulls, ipfs, and
main.
  • Loading branch information
beardthelion committed Aug 26, 2026
commit 4cee7e0935be31c729422b59b7eb2f935de16bc7
26 changes: 7 additions & 19 deletions crates/gitlawb-node/src/api/ipfs.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3821,22 +3821,6 @@ mod tests {
// the row created first is the row iterated first.
state.repo_store =
crate::git::repo_store::RepoStore::for_testing(repos_dir.clone(), pool.clone());
let content = b"acquire taint continue proof\n";
let (_, oid) =
seed_repo_with_blob(&state, tmp.path(), "z6f2acqcont", "pubcopy", content).await;
// Swap in a Tigris-backed store over the SAME repos_dir (the seeded bare
// repo stays a fast local hit) and add a NEWER ghost row with no local
// copy: its acquire consults the silent local endpoint and stalls to the
// 1s timeout (endpoint-pinned test client, no AWS_* env reads).
let endpoint = crate::test_support::silent_http_endpoint().await;
let tigris =
crate::git::tigris::TigrisClient::for_testing_with_endpoint("test-bucket", &endpoint);
state.repo_store = crate::git::repo_store::RepoStore::new(
repos_dir,
Some(tigris),
pool,
std::time::Duration::from_secs(300),
);
state
.db
.upsert_mirror_repo("z6f2acqcont", "ghost", "/unused-ghost", None, false)
Expand All @@ -3851,9 +3835,13 @@ mod tests {
// (endpoint-pinned test client, no AWS_* env reads).
let endpoint = crate::test_support::silent_http_endpoint().await;
let tigris =
crate::git::tigris::TigrisClient::for_testing_with_endpoint("test-bucket", &endpoint)
.await;
state.repo_store = crate::git::repo_store::RepoStore::new(repos_dir, Some(tigris), pool);
crate::git::tigris::TigrisClient::for_testing_with_endpoint("test-bucket", &endpoint);
state.repo_store = crate::git::repo_store::RepoStore::new(
repos_dir,
Some(tigris),
pool,
std::time::Duration::from_secs(300),
);
let mut cfg = (*state.config).clone();
cfg.git_acquire_timeout_secs = 1;
state.config = Arc::new(cfg);
Expand Down
162 changes: 157 additions & 5 deletions crates/gitlawb-node/src/api/issues.rs
Original file line number Diff line number Diff line change
Expand Up @@ -61,13 +61,11 @@ pub async fn create_issue(
let json_str = serde_json::to_string(&issue)
.map_err(|e| AppError::BadRequest(format!("serialization error: {e}")))?;

// Shed 503 + Retry-After on an exhausted write-lock POOL instead of a generic
// git 500 (#173 F1). This path holds no admission permit, so it reaches the pool
// unthrottled; reuse the push handler's mapping so the two cannot drift.
let guard = state
.repo_store
.acquire_write(&record.owner_did, &record.name)
.await?;
.await
.map_err(|e| crate::api::repos::acquire_write_app_error(&e, &repo))?;
let disk_path = guard.path().to_path_buf();

let create_result = git_issues::create_issue(&disk_path, &issue_id, &json_str);
Expand Down Expand Up @@ -372,7 +370,8 @@ pub async fn close_issue(
let guard = state
.repo_store
.acquire_write(&record.owner_did, &record.name)
.await?;
.await
.map_err(|e| crate::api::repos::acquire_write_app_error(&e, &repo))?;
let disk_path = guard.path().to_path_buf();

// Re-read under the guard and RE-AUTHORIZE against what we read, rather than
Expand Down Expand Up @@ -773,3 +772,156 @@ mod tests {
);
}
}

/// #173 F1 follow-up: issue write paths reach `acquire_write` with no admission
/// permit, so an exhausted write-lock pool must shed 503 + Retry-After.
#[cfg(test)]
mod lock_pool_shed_tests {
use super::*;
use axum::response::IntoResponse;
use sqlx::PgPool;

fn seed_repo(owner_did: &str, name: &str) -> crate::db::RepoRecord {
let now = Utc::now();
crate::db::RepoRecord {
id: Uuid::new_v4().to_string(),
name: name.to_string(),
owner_did: owner_did.to_string(),
description: None,
is_public: true,
default_branch: "main".to_string(),
created_at: now,
updated_at: now,
disk_path: format!("/tmp/{name}"),
forked_from: None,
machine_id: None,
}
}

async fn one_connection_lock_pool_state(pool: &PgPool) -> AppState {
let mut state = crate::test_support::test_state(pool.clone()).await;
state.repo_store = crate::git::repo_store::RepoStore::new(
std::path::PathBuf::from("/tmp/gitlawb-issues-lockpool"),
None,
crate::git::repo_store::build_lock_pool(pool, 1, std::time::Duration::from_secs(1)),
std::time::Duration::from_secs(300),
);
state
}

fn assert_sheds_503_with_retry_after(err: AppError, what: &str) {
let resp = err.into_response();
assert_eq!(
resp.status(),
StatusCode::SERVICE_UNAVAILABLE,
"{what}: an exhausted write-lock pool must shed 503, not a 500 git error"
);
assert_eq!(
resp.headers()
.get("retry-after")
.map(|v| v.to_str().unwrap()),
Some("1"),
"{what}: a capacity shed must tell the client when to retry"
);
}

#[sqlx::test]
async fn create_issue_lock_pool_exhaustion_sheds_503_not_500(pool: PgPool) {
let owner = "did:key:zISSUECREATELOCKPOOLAAAAAAAAAAAAAAAAAAAA";
let state = one_connection_lock_pool_state(&pool).await;
state
.db
.create_repo(&seed_repo(owner, "lp-create"))
.await
.expect("seed repo");

let held = state
.repo_store
.acquire_write(owner, "other-repo")
.await
.expect("the first write takes the only lock-pool connection");

let shed = create_issue(
State(state.clone()),
Extension(AuthenticatedDid(owner.to_string())),
Path((owner.to_string(), "lp-create".to_string())),
Json(CreateIssueRequest {
title: "t".to_string(),
body: None,
signed_payload: None,
}),
)
.await;
let err = shed.expect_err("an exhausted lock pool must fail the call");
assert_sheds_503_with_retry_after(err, "create_issue");

let _ = held.release(false).await;
let admitted = create_issue(
State(state.clone()),
Extension(AuthenticatedDid(owner.to_string())),
Path((owner.to_string(), "lp-create".to_string())),
Json(CreateIssueRequest {
title: "t".to_string(),
body: None,
signed_payload: None,
}),
)
.await;
assert!(
!matches!(admitted, Err(AppError::Overloaded(_))),
"with the lock pool free, create_issue must not be shed as capacity; got {:?}",
admitted.err()
);
}

#[sqlx::test]
async fn close_issue_lock_pool_exhaustion_sheds_503_not_500(pool: PgPool) {
let owner = "did:key:zISSUECLOSELOCKPOOLBBBBBBBBBBBBBBBBBBBBB";
let state = one_connection_lock_pool_state(&pool).await;
state
.db
.create_repo(&seed_repo(owner, "lp-close"))
.await
.expect("seed repo");

let held = state
.repo_store
.acquire_write(owner, "other-repo")
.await
.expect("the first write takes the only lock-pool connection");

let shed = close_issue(
State(state.clone()),
Extension(AuthenticatedDid(owner.to_string())),
Path((
owner.to_string(),
"lp-close".to_string(),
"deadbeef".to_string(),
)),
axum::http::HeaderMap::new(),
crate::rate_limit::PeerAddr(None),
)
.await;
let err = shed.expect_err("an exhausted lock pool must fail the call");
assert_sheds_503_with_retry_after(err, "close_issue");

let _ = held.release(false).await;
let admitted = close_issue(
State(state.clone()),
Extension(AuthenticatedDid(owner.to_string())),
Path((
owner.to_string(),
"lp-close".to_string(),
"deadbeef".to_string(),
)),
axum::http::HeaderMap::new(),
crate::rate_limit::PeerAddr(None),
)
.await;
assert!(
!matches!(admitted, Err(AppError::Overloaded(_))),
"with the lock pool free, close_issue must not be shed as capacity; got {:?}",
admitted.err()
);
}
}
25 changes: 6 additions & 19 deletions crates/gitlawb-node/src/api/pulls.rs
Original file line number Diff line number Diff line change
Expand Up @@ -209,13 +209,11 @@ pub async fn merge_pr(
return Err(AppError::BadRequest(format!("PR is already {}", pr.status)));
}

// Shed 503 + Retry-After on an exhausted write-lock POOL instead of a generic
// git 500 (#173 F1). Merging holds no admission permit, so it reaches the pool
// unthrottled; reuse the push handler's mapping so the two cannot drift.
let guard = state
.repo_store
.acquire_write(&record.owner_did, &record.name)
.await?;
.await
.map_err(|e| crate::api::repos::acquire_write_app_error(&e, &name))?;
let disk_path = guard.path().to_path_buf();
let merger_did = auth.0;
let merge_result = store::merge_branch(
Expand Down Expand Up @@ -430,12 +428,8 @@ pub async fn list_comments(
Ok(Json(serde_json::json!({ "comments": comments })))
}

/// #173 F1 follow-up: `merge_pr` reaches `acquire_write` holding NO admission permit
/// (unlike the push handler, which is capped by the git-push semaphore), so it is one
/// of the callers most likely to meet an exhausted write-lock POOL under load. An
/// exhausted pool is a capacity signal, so the merge must shed 503 + Retry-After
/// (`AppError::Overloaded`) the way the push handler does, not report the generic
/// 500 git error that says nothing about retrying.
/// #173 F1 follow-up: merge holds no admission permit, so an exhausted write-lock
/// pool must shed 503 + Retry-After.
#[cfg(test)]
mod lock_pool_shed_tests {
use super::*;
Expand All @@ -459,20 +453,15 @@ mod lock_pool_shed_tests {
}
}

/// RED-before/GREEN-after for `merge_pr`. Both directions: the shed while the only
/// lock-pool connection is held by a guard on a DIFFERENT repo (so this is pool
/// capacity, not advisory-lock contention on this repo), and the must-not case
/// once that connection is back.
#[sqlx::test]
async fn merge_pr_lock_pool_exhaustion_sheds_503_not_500(pool: PgPool) {
let owner = "did:key:zMERGELOCKPOOLOWNERAAAAAAAAAAAAAAAAAAAAA";
let mut state = crate::test_support::test_state(pool.clone()).await;
// One lock-pool connection with a short checkout timeout, so a single held
// guard exhausts it promptly rather than at the pool default.
state.repo_store = crate::git::repo_store::RepoStore::new(
std::path::PathBuf::from("/tmp/gitlawb-pulls-lockpool"),
None,
crate::git::repo_store::build_lock_pool(&pool, 1, std::time::Duration::from_secs(1)),
std::time::Duration::from_secs(300),
);

let repo = seed_repo(owner, "lp-merge");
Expand Down Expand Up @@ -526,9 +515,7 @@ mod lock_pool_shed_tests {
"merge_pr: a capacity shed must tell the client when to retry"
);

// MUST-NOT: with the pool free again the merge is not shed as capacity (it
// fails later on the nonexistent on-disk repo, which is a git 500).
held.release(false).await;
let _ = held.release(false).await;
let admitted = merge_pr(
State(state.clone()),
Extension(AuthenticatedDid(owner.to_string())),
Expand Down
23 changes: 20 additions & 3 deletions crates/gitlawb-node/src/api/repos.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2256,12 +2256,17 @@ pub async fn git_receive_pack(
AppError::Overloaded("git service acquisition timed out, retry shortly".into())
})?
.map_err(|e| {
if is_expected_transient_acquire_failure(&e) {
if e.downcast_ref::<crate::git::repo_store::LockPoolBusy>()
.is_some()
{
acquire_write_app_error(&e, name)
} else if is_expected_transient_acquire_failure(&e) {
tracing::warn!(repo = %name, err = %e, "acquire_write failed");
AppError::from(e)
} else {
tracing::error!(repo = %name, err = %e, "acquire_write failed");
AppError::Git(e.to_string())
}
AppError::Git(e.to_string())
})?;
let disk_path = guard.path().to_path_buf();
tracing::debug!(repo = %name, path = %disk_path.display(), "running git receive-pack");
Expand Down Expand Up @@ -2361,12 +2366,21 @@ pub async fn git_receive_pack(
// Always release the advisory lock — even on error — to prevent stale locks
// from blocking subsequent pushes. Only upload to Tigris when the push
// succeeded; uploading a half-applied repo would propagate corruption.
// Reclaim the write lock from the shared cell (#173 F2). This is only reachable
// once `receive_pack` has returned, so the admission guard's copy can only ever
// DELAY release, never perform it early; on the disconnect path this line is not
// reached at all and the reaper's copy is last.
let reclaimed = guard
.lock()
.expect("repo write-lock mutex poisoned")
.take()
.expect("the write lock is only taken here, and only once");
// Short-circuit on a refused publish BEFORE anything downstream observes
// the push. The pack is on local disk but not in object storage, so
// touching the repo, recording the push, bumping trust, issuing ref
// certificates or answering 200 would all be reporting a write no other
// node can read.
guard.release(receive_result.is_ok()).await.into_result()?;
reclaimed.release(push_succeeded).await.into_result()?;
// Clean path: clone (a) already dropped inside run_git_service when the receive-pack
// group was reaped; clone (b) held here spanned the success-only Tigris upload that
// ran inside release() above. Drop it now so a second same-repo push proceeds the
Expand Down Expand Up @@ -6339,6 +6353,7 @@ mod tests {
repos_dir.path().to_path_buf(),
None,
crate::git::repo_store::build_lock_pool(&pool, 4, std::time::Duration::from_secs(5)),
std::time::Duration::from_secs(300),
);
let mut cfg = (*state.config).clone();
// Long enough that the git-service timeout is never what ends this push; the
Expand Down Expand Up @@ -6513,6 +6528,7 @@ mod tests {
repos_dir.path().to_path_buf(),
None,
crate::git::repo_store::build_lock_pool(&pool, 4, std::time::Duration::from_secs(5)),
std::time::Duration::from_secs(300),
);
state
.db
Expand Down Expand Up @@ -6606,6 +6622,7 @@ mod tests {
std::path::PathBuf::from("/tmp/gitlawb-lockpool-shed"),
None,
crate::git::repo_store::build_lock_pool(&pool, 1, std::time::Duration::from_secs(1)),
std::time::Duration::from_secs(300),
);
state
.db
Expand Down
Loading
You are viewing a condensed version of this merge commit. You can view the full changes here.