Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
57 commits
Select commit Hold shift + click to select a range
7af3067
fix(node): close the advisory-lock probe's session if it is dropped m…
beardthelion Jul 29, 2026
edcf0f2
fix(node): add a dedicated, lazily-connected advisory-lock pool
beardthelion Jul 29, 2026
5ce3c7f
fix(node): hold the lock-owning connection in RepoWriteGuard (#279)
beardthelion Jul 29, 2026
c49d3cd
fix(node): free the advisory lock when a write guard dies without rel…
beardthelion Jul 29, 2026
98f2daf
fix(node): read the advisory unlock's result instead of discarding it
beardthelion Jul 29, 2026
9321d3b
fix(node): bound the object-storage transfers that run under the writ…
beardthelion Jul 29, 2026
9da0939
fix(node): authorize before taking the write lock in close_issue
beardthelion Jul 29, 2026
5d254fb
fix(node): address the review findings on the advisory-lock series
beardthelion Jul 30, 2026
d346bde
fix(node): log lock-pool saturation in the request path, not via read…
beardthelion Jul 30, 2026
82143ad
fix(node): refuse a write when object storage is unknowable, and shed…
beardthelion Jul 30, 2026
8cf6b7a
fix(node): re-authorize close_issue under the guard instead of only r…
beardthelion Jul 30, 2026
af5948b
style(node): rustfmt the new contention test
beardthelion Jul 30, 2026
e5558fb
fix(node): shed an unrefreshable write as a retryable 503 with a fixe…
beardthelion Aug 3, 2026
63a9ef8
fix(node): refuse a fresh-copy write when the archive HEAD cannot be …
beardthelion Aug 3, 2026
10230ac
fix(node): log expected transient acquire failures at warn, not error
beardthelion Aug 3, 2026
259d587
fix(node): log the read failure the close_issue pre-check folds into …
beardthelion Aug 3, 2026
999ee72
test(node): poll for the closed session instead of sleeping a fixed 3…
beardthelion Aug 3, 2026
5fdb902
fix(node): raise a failed fresh download with no local copy as RepoUn…
beardthelion Aug 9, 2026
f81ebbe
test(node): assert the second writer sheds as RepoBusy instead of tim…
beardthelion Aug 9, 2026
c4dbd8a
fix(node): read the close_issue author pre-check from a non-mutating …
beardthelion Aug 10, 2026
9f313b9
fix(node): bound every await in the lock-acquire loop by the deadline
beardthelion Aug 10, 2026
60d70c7
test(node): add an S3 mock that really enforces conditional writes
beardthelion Aug 10, 2026
31f90c4
fix(node): stop pretending a held lock fences a late upload
beardthelion Aug 10, 2026
3eabf79
feat(node): let the storage client express and detect a conditional w…
beardthelion Aug 10, 2026
317841e
fix(node): fence the release publish so an abandoned upload cannot win
beardthelion Aug 10, 2026
3d803ff
test(node): drive the abandoned-upload race end to end, and probe the…
beardthelion Aug 10, 2026
2a92a1f
fix(node): reconcile F-series tests and test seams with the rebased g…
beardthelion Aug 18, 2026
9f35dca
fix(node): address the 2026-08-10 review round on the advisory-lock s…
beardthelion Aug 18, 2026
9dd71e9
fix(node): record why the close-issue read gate cannot carry a synced…
beardthelion Aug 18, 2026
4cee7e0
merge origin/main into fix/279-advisory-lock-session-affinity
beardthelion Aug 26, 2026
b35cb31
fix(node): address jatmn review on merged advisory-lock head
beardthelion Aug 26, 2026
3c20148
fix(node): cap close_issue snapshot work on the read pool
beardthelion Aug 26, 2026
7ba0fef
fix(node): mark close_issue mirror visibility gate for push hook
beardthelion Aug 26, 2026
3df71c9
fix: unblock pre-push after main merge
beardthelion Aug 26, 2026
98fe670
fix(node): carry transient acquire failures through every write handler
beardthelion Aug 28, 2026
4f1c1bb
fix(node): drop unused logcap helper after drain_faults seam
beardthelion Aug 28, 2026
4eb0c44
fix(node): route fork disk paths through validated_repo_disk_path
beardthelion Aug 28, 2026
d1ce2e3
fix(node): harden release-upload outcomes and replication tail gating
beardthelion Aug 28, 2026
8e66e77
fix(node): close jatmn round on publish authority and durability gating
beardthelion Aug 29, 2026
1911291
fix(node): harden issue close/create and fork publish paths on #285
beardthelion Aug 29, 2026
5d3a4ea
fix(node): close write-attempt ownership gaps on advisory-lock release
beardthelion Aug 29, 2026
2ad2f12
fix(node): gate replication tail on publish durability and split clos…
beardthelion Aug 29, 2026
487687c
fix(node): add ValidatedRepoDiskPath for CodeQL and reconcile tail du…
beardthelion Aug 29, 2026
9d02139
fix(node): close three jatmn findings on advisory-lock PR
beardthelion Aug 30, 2026
85c6f48
fix(node): verify adversarial findings on advisory-lock PR
beardthelion Aug 30, 2026
5cd6703
fix(node): compensate under the write lock and harden tail durability…
beardthelion Aug 30, 2026
22da2d6
fix(node): refuse stale under-lock refresh and gate tail durability o…
beardthelion Aug 30, 2026
c306933
style(node): rustfmt advisory-lock gap-closure commits
beardthelion Aug 30, 2026
3a82ccb
fix(node): satisfy clippy on stale-local download test path join
beardthelion Aug 30, 2026
c40f299
feat(node): add a publish attempt/stage boundary
kevincodex1 Aug 31, 2026
21646bc
feat(node): stamp uploads with an attempt id and preserve dispatch am…
kevincodex1 Aug 31, 2026
5e548d2
fix(node): key durability, cache validity and fork cleanup on the att…
kevincodex1 Aug 31, 2026
535c5dc
fix(node): settle the cache contract on the abandoned path and the at…
beardthelion Sep 7, 2026
f048a1e
fix(node): settle the abandoned tree at the disconnect, not at reaper…
beardthelion Sep 7, 2026
957d50b
fix(node): let exactly one classifier settle an abandoned tree
beardthelion Sep 7, 2026
892c752
Merge origin/main into fix/279-advisory-lock-session-affinity
beardthelion Nov 4, 2026
28f3fc2
test(node): use the test's own pool now that Db::pool is gone
beardthelion Nov 4, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
fix(node): harden release-upload outcomes and replication tail gating
Treat timed-out or failed archive uploads as non-durable in into_result,
gate pinata/gossip on confirmed publish durability while keeping inv22 spawn
order, split lock-pool vs main DB pool validation, map LockPoolBusy through
From<anyhow::Error>, and attach Retry-After on repo write contention errors.
  • Loading branch information
beardthelion committed Aug 28, 2026
commit d1ce2e3291494269ecdbaa79d75fc42906ced9da
56 changes: 55 additions & 1 deletion crates/gitlawb-node/src/api/repos.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2374,13 +2374,15 @@ pub async fn git_receive_pack(
// would return 200 to the pusher before the durable copy lands, which is a larger
// change to the client contract than the window it closes.
let push_succeeded = receive_result.is_ok();
let publish_durability = Arc::new(tokio::sync::Mutex::new(None));
if push_succeeded {
tokio::spawn(post_receive_replication_tail(
state.clone(),
record.clone(),
ref_updates.clone(),
disk_path.clone(),
auth.0.to_string(),
Some(publish_durability.clone()),
));
}

Expand All @@ -2401,7 +2403,11 @@ pub async fn git_receive_pack(
// touching the repo, recording the push, bumping trust, issuing ref
// certificates or answering 200 would all be reporting a write no other
// node can read.
reclaimed.release(push_succeeded).await.into_result()?;
let outcome = reclaimed.release(push_succeeded).await;
if push_succeeded {
*publish_durability.lock().await = Some(outcome);
}
outcome.into_result()?;
// Clean path: clone (a) already dropped inside run_git_service when the receive-pack
// group was reaped; clone (b) held here spanned the success-only Tigris upload that
// ran inside release() above. Drop it now so a second same-repo push proceeds the
Expand Down Expand Up @@ -2518,12 +2524,36 @@ pub async fn git_receive_pack(
/// the per-repo-coalesced pin/encrypt task, and this push's own Pinata + announce
/// task. Split out of `git_receive_pack` so the ordering the coalescing gate depends
/// on is directly testable; the handler spawns it and returns.
async fn publish_durability_confirmed(
slot: &Option<Arc<tokio::sync::Mutex<Option<crate::git::repo_store::ReleaseOutcome>>>>,
wait: std::time::Duration,
) -> bool {
let Some(slot) = slot else {
return true;
};
let start = std::time::Instant::now();
loop {
if let Some(outcome) = *slot.lock().await {
return matches!(outcome, crate::git::repo_store::ReleaseOutcome::Released);
}
if start.elapsed() >= wait {
// Handler disconnected during `release` without recording an outcome.
// Proceed so inv22's disconnect-during-release tail still runs.
return true;
}
tokio::time::sleep(std::time::Duration::from_millis(10)).await;
}
}

async fn post_receive_replication_tail(
state: AppState,
record: RepoRecord,
ref_updates: Vec<RefUpdate>,
disk_path: std::path::PathBuf,
did: String,
publish_durability: Option<
Arc<tokio::sync::Mutex<Option<crate::git::repo_store::ReleaseOutcome>>>,
>,
) {
// Replication enforcement (Phase 2): decide once per push whether the public
// may read this repo at all and, if so, which blob OIDs must not leave the
Expand Down Expand Up @@ -2697,6 +2727,21 @@ async fn post_receive_replication_tail(
));
}

let durability_wait = std::time::Duration::from_secs(
state
.config
.lock_held_transfer_timeout_secs
.saturating_add(5),
);
if announce_at_root && !publish_durability_confirmed(&publish_durability, durability_wait).await
{
tracing::warn!(
repo = %record.id,
"skipping pinata/gossip tail: publish durability was not confirmed"
);
return;
}

// Pin new git objects to Pinata, then record branch→CID and gossip.
//
// #174 P2-2 scope note: this SECOND detached spawn is deliberately NOT brought
Expand Down Expand Up @@ -9819,6 +9864,7 @@ mod tests {
f2a_update("refs/heads/main", &c2),
repo.path().to_path_buf(),
F2A_PUSHER.to_string(),
None,
)
.await;
let after_first = f2a_walks(&log);
Expand All @@ -9839,6 +9885,7 @@ mod tests {
f2a_update("refs/heads/second", &c1),
repo.path().to_path_buf(),
F2A_PUSHER.to_string(),
None,
)
.await;

Expand Down Expand Up @@ -9912,6 +9959,7 @@ mod tests {
f2a_update("refs/heads/main", &c2),
repo.path().to_path_buf(),
F2A_PUSHER.to_string(),
None,
)
.await;

Expand Down Expand Up @@ -9993,6 +10041,7 @@ mod tests {
f2a_update("refs/heads/main", &c2),
repo.path().to_path_buf(),
F2A_PUSHER.to_string(),
None,
));
f2a_wait_for(|| started.exists(), "the admitted push's walk to start").await;

Expand Down Expand Up @@ -10106,6 +10155,7 @@ mod tests {
}],
repo.path().to_path_buf(),
F2A_PUSHER.to_string(),
None,
)
.await;

Expand Down Expand Up @@ -10384,6 +10434,7 @@ mod tests {
}],
repo.path().to_path_buf(),
F2A_PUSHER.to_string(),
None,
)
.await;

Expand Down Expand Up @@ -10439,6 +10490,7 @@ mod tests {
f2a_update("refs/heads/main", &c1),
repo.path().to_path_buf(),
F2A_PUSHER.to_string(),
None,
)
.await;
tokio::time::sleep(std::time::Duration::from_millis(300)).await;
Expand Down Expand Up @@ -10480,6 +10532,7 @@ mod tests {
}],
repo.path().to_path_buf(),
F2A_PUSHER.to_string(),
None,
)
.await;

Expand Down Expand Up @@ -10577,6 +10630,7 @@ mod tests {
}],
repo.path().to_path_buf(),
F2A_PUSHER.to_string(),
None,
)
.await;

Expand Down
77 changes: 42 additions & 35 deletions crates/gitlawb-node/src/config.rs
Original file line number Diff line number Diff line change
Expand Up @@ -252,11 +252,9 @@ pub struct Config {

/// Maximum connections in the PostgreSQL pool. This is a cap, not a floor
/// (connections open lazily). Size against the database server's
/// max_connections, remembering admin tooling opens its own pool. Each
/// concurrent write pins one pooled connection for its whole duration (the
/// advisory lock in `repo_store::acquire_write` is connection-affine), so this
/// must exceed `max_concurrent_git_pushes` by `DB_POOL_APP_HEADROOM` or slow
/// pushes starve every other DB path — enforced by `Config::validate`.
/// max_connections, remembering admin tooling opens its own pool. Writes pin
/// the dedicated lock pool, not this one; this pool still needs enough headroom
/// for ordinary request handlers and metadata paths (`DB_POOL_APP_HEADROOM`).
#[arg(
long,
env = "GITLAWB_DB_MAX_CONNECTIONS",
Expand Down Expand Up @@ -769,21 +767,23 @@ impl Config {
/// catches combinations that ship a denial-of-service under otherwise-valid
/// values. Call once at startup and fail fast on `Err`.
pub fn validate(&self) -> Result<(), String> {
// A write pins one pooled connection for its whole duration (the
// connection-affine advisory lock in repo_store::acquire_write), and
// concurrent writes are capped at max_concurrent_git_pushes. If the pool
// does not exceed that cap by DB_POOL_APP_HEADROOM, a burst of slow pushes
// drains every connection and every other DB path 503s. (#174 F1)
let floor = (self.max_concurrent_git_pushes as u64) + (Self::DB_POOL_APP_HEADROOM as u64);
if (self.db_max_connections as u64) < floor {
// Concurrent git writes pin the dedicated lock pool for their whole
// duration (connection-affine advisory lock in acquire_write). The main
// pool no longer carries that occupancy.
if (self.db_lock_pool_max_connections as usize) < self.max_concurrent_git_pushes {
return Err(format!(
"GITLAWB_DB_MAX_CONNECTIONS ({}) must be at least max_concurrent_git_pushes ({}) \
+ {} headroom = {}: each concurrent write pins one pooled connection for its whole \
duration, so a smaller pool lets a burst of slow pushes starve every other DB path.",
self.db_max_connections,
self.max_concurrent_git_pushes,
Self::DB_POOL_APP_HEADROOM,
floor
"GITLAWB_DB_LOCK_POOL_MAX_CONNECTIONS ({}) must be at least \
max_concurrent_git_pushes ({}) so every admitted push can pin a \
lock-pool connection for its whole duration",
self.db_lock_pool_max_connections, self.max_concurrent_git_pushes
));
}
let main_floor = Self::DB_POOL_APP_HEADROOM as u64;
if (self.db_max_connections as u64) < main_floor {
return Err(format!(
"GITLAWB_DB_MAX_CONNECTIONS ({}) must be at least {} for non-git \
database paths",
self.db_max_connections, main_floor
));
}
Ok(())
Expand Down Expand Up @@ -1422,41 +1422,48 @@ mod tests {
);
}

/// #174 F1: a connection-affine write lock pins a pooled connection per
/// concurrent write, so the pool must clear `max_concurrent_git_pushes` by
/// `DB_POOL_APP_HEADROOM` or a push burst starves every other DB path.
/// `validate()` must reject an under-sized pool at boot.
/// #174 F1: concurrent git writes pin the dedicated lock pool, not the main
/// pool. `validate()` must reject an under-sized lock pool at boot.
#[test]
fn db_pool_must_clear_the_git_push_cap() {
// Shipped defaults validate (48 >= 32 + 8).
// Shipped defaults validate (lock pool 32 >= pushes 32, main pool >= headroom).
Config::parse_from(["gitlawb-node"])
.validate()
.expect("default config must validate");

// An under-sized pool relative to the push cap is rejected (20 < 32 + 8).
let under = Config::parse_from([
// An under-sized lock pool relative to the push cap is rejected.
let under_lock = Config::parse_from([
"gitlawb-node",
"--db-max-connections",
"20",
"--db-lock-pool-max-connections",
"16",
"--max-concurrent-git-pushes",
"32",
]);
assert!(
under.validate().is_err(),
"db_max_connections 20 below max_concurrent_git_pushes 32 + headroom must be rejected"
under_lock.validate().is_err(),
"db_lock_pool_max_connections below max_concurrent_git_pushes must be rejected"
);

// Exactly at the floor validates (40 == 32 + 8).
let at_floor = Config::parse_from([
// Main pool can be smaller than pushes + headroom when the lock pool carries writes.
let split = Config::parse_from([
"gitlawb-node",
"--db-max-connections",
"40",
"16",
"--db-lock-pool-max-connections",
"32",
"--max-concurrent-git-pushes",
"32",
]);
assert!(
at_floor.validate().is_ok(),
"db_max_connections at the floor (pushes + headroom) must validate"
split.validate().is_ok(),
"a small main pool with a large lock pool must validate"
);

// Main pool still needs the app headroom floor.
let under_main = Config::parse_from(["gitlawb-node", "--db-max-connections", "4"]);
assert!(
under_main.validate().is_err(),
"db_max_connections below DB_POOL_APP_HEADROOM must be rejected"
);
}

Expand Down
65 changes: 50 additions & 15 deletions crates/gitlawb-node/src/error.rs
Original file line number Diff line number Diff line change
Expand Up @@ -123,20 +123,24 @@ impl From<anyhow::Error> for AppError {
// 500. The internal message names the owner slug and repo, so the
// variant carries nothing: the detail stays in the log at the raise
// site and the client gets a fixed retryable body.
Err(err) => match err.downcast::<crate::git::repo_store::RepoBusy>() {
Ok(_) => AppError::RepoBusy,
// Same reasoning one rung down: a refused under-lock refresh is a
// transient storage condition, and its internal message names the
// owner slug and repo, so the variant carries nothing.
Err(err) => match err.downcast::<crate::git::repo_store::RepoUnavailable>() {
Ok(_) => AppError::RepoUnavailable,
// And one more rung: a publish the store refused twice is
// transient in the same way, and the retry is the client's
// to make. The variant carries nothing for the same reason
// as the two above.
Err(err) => match err.downcast::<crate::git::repo_store::RepoWriteFenced>() {
Ok(_) => AppError::RepoWriteFenced,
Err(err) => AppError::Internal(err),
Err(err) => match err.downcast::<crate::git::repo_store::LockPoolBusy>() {
Ok(_) => AppError::Overloaded("git write locks at capacity, retry shortly".into()),
Err(err) => match err.downcast::<crate::git::repo_store::RepoBusy>() {
Ok(_) => AppError::RepoBusy,
// Same reasoning one rung down: a refused under-lock refresh is a
// transient storage condition, and its internal message names the
// owner slug and repo, so the variant carries nothing.
Err(err) => match err.downcast::<crate::git::repo_store::RepoUnavailable>() {
Ok(_) => AppError::RepoUnavailable,
// And one more rung: a publish the store refused twice is
// transient in the same way, and the retry is the client's
// to make. The variant carries nothing for the same reason
// as the two above.
Err(err) => match err.downcast::<crate::git::repo_store::RepoWriteFenced>()
{
Ok(_) => AppError::RepoWriteFenced,
Err(err) => AppError::Internal(err),
},
},
},
},
Expand Down Expand Up @@ -296,7 +300,11 @@ impl IntoResponse for AppError {
// here rather than in bespoke early returns, keeping each variant handled once.
if matches!(
self,
AppError::Overloaded(_) | AppError::SearchIncomplete { .. }
AppError::Overloaded(_)
| AppError::SearchIncomplete { .. }
| AppError::RepoBusy
| AppError::RepoUnavailable
| AppError::RepoWriteFenced
) {
resp.headers_mut().insert(
axum::http::header::RETRY_AFTER,
Expand Down Expand Up @@ -414,4 +422,31 @@ mod tests {
let resp = err.into_response();
assert_eq!(resp.status(), StatusCode::SERVICE_UNAVAILABLE);
}

#[test]
fn lock_pool_busy_via_anyhow_from_is_503_overloaded() {
let err: AppError = anyhow::Error::new(crate::git::repo_store::LockPoolBusy).into();
let resp = err.into_response();
assert_eq!(resp.status(), StatusCode::SERVICE_UNAVAILABLE);
assert_eq!(
resp.headers().get("retry-after").unwrap().to_str().unwrap(),
"1"
);
}

#[test]
fn repo_busy_unavailable_and_fenced_advertise_retry_after() {
for err in [
AppError::RepoBusy,
AppError::RepoUnavailable,
AppError::RepoWriteFenced,
] {
let resp = err.into_response();
assert_eq!(resp.status(), StatusCode::SERVICE_UNAVAILABLE);
assert_eq!(
resp.headers().get("retry-after").unwrap().to_str().unwrap(),
"1"
);
}
}
}
Loading
Loading