Skip to content
Merged
Changes from 1 commit
Commits
Show all changes
57 commits
Select commit Hold shift + click to select a range
7af3067
fix(node): close the advisory-lock probe's session if it is dropped m…
beardthelion Jul 29, 2026
edcf0f2
fix(node): add a dedicated, lazily-connected advisory-lock pool
beardthelion Jul 29, 2026
5ce3c7f
fix(node): hold the lock-owning connection in RepoWriteGuard (#279)
beardthelion Jul 29, 2026
c49d3cd
fix(node): free the advisory lock when a write guard dies without rel…
beardthelion Jul 29, 2026
98f2daf
fix(node): read the advisory unlock's result instead of discarding it
beardthelion Jul 29, 2026
9321d3b
fix(node): bound the object-storage transfers that run under the writ…
beardthelion Jul 29, 2026
9da0939
fix(node): authorize before taking the write lock in close_issue
beardthelion Jul 29, 2026
5d254fb
fix(node): address the review findings on the advisory-lock series
beardthelion Jul 30, 2026
d346bde
fix(node): log lock-pool saturation in the request path, not via read…
beardthelion Jul 30, 2026
82143ad
fix(node): refuse a write when object storage is unknowable, and shed…
beardthelion Jul 30, 2026
8cf6b7a
fix(node): re-authorize close_issue under the guard instead of only r…
beardthelion Jul 30, 2026
af5948b
style(node): rustfmt the new contention test
beardthelion Jul 30, 2026
e5558fb
fix(node): shed an unrefreshable write as a retryable 503 with a fixe…
beardthelion Aug 3, 2026
63a9ef8
fix(node): refuse a fresh-copy write when the archive HEAD cannot be …
beardthelion Aug 3, 2026
10230ac
fix(node): log expected transient acquire failures at warn, not error
beardthelion Aug 3, 2026
259d587
fix(node): log the read failure the close_issue pre-check folds into …
beardthelion Aug 3, 2026
999ee72
test(node): poll for the closed session instead of sleeping a fixed 3…
beardthelion Aug 3, 2026
5fdb902
fix(node): raise a failed fresh download with no local copy as RepoUn…
beardthelion Aug 9, 2026
f81ebbe
test(node): assert the second writer sheds as RepoBusy instead of tim…
beardthelion Aug 9, 2026
c4dbd8a
fix(node): read the close_issue author pre-check from a non-mutating …
beardthelion Aug 10, 2026
9f313b9
fix(node): bound every await in the lock-acquire loop by the deadline
beardthelion Aug 10, 2026
60d70c7
test(node): add an S3 mock that really enforces conditional writes
beardthelion Aug 10, 2026
31f90c4
fix(node): stop pretending a held lock fences a late upload
beardthelion Aug 10, 2026
3eabf79
feat(node): let the storage client express and detect a conditional w…
beardthelion Aug 10, 2026
317841e
fix(node): fence the release publish so an abandoned upload cannot win
beardthelion Aug 10, 2026
3d803ff
test(node): drive the abandoned-upload race end to end, and probe the…
beardthelion Aug 10, 2026
2a92a1f
fix(node): reconcile F-series tests and test seams with the rebased g…
beardthelion Aug 18, 2026
9f35dca
fix(node): address the 2026-08-10 review round on the advisory-lock s…
beardthelion Aug 18, 2026
9dd71e9
fix(node): record why the close-issue read gate cannot carry a synced…
beardthelion Aug 18, 2026
4cee7e0
merge origin/main into fix/279-advisory-lock-session-affinity
beardthelion Aug 26, 2026
b35cb31
fix(node): address jatmn review on merged advisory-lock head
beardthelion Aug 26, 2026
3c20148
fix(node): cap close_issue snapshot work on the read pool
beardthelion Aug 26, 2026
7ba0fef
fix(node): mark close_issue mirror visibility gate for push hook
beardthelion Aug 26, 2026
3df71c9
fix: unblock pre-push after main merge
beardthelion Aug 26, 2026
98fe670
fix(node): carry transient acquire failures through every write handler
beardthelion Aug 28, 2026
4f1c1bb
fix(node): drop unused logcap helper after drain_faults seam
beardthelion Aug 28, 2026
4eb0c44
fix(node): route fork disk paths through validated_repo_disk_path
beardthelion Aug 28, 2026
d1ce2e3
fix(node): harden release-upload outcomes and replication tail gating
beardthelion Aug 28, 2026
8e66e77
fix(node): close jatmn round on publish authority and durability gating
beardthelion Aug 29, 2026
1911291
fix(node): harden issue close/create and fork publish paths on #285
beardthelion Aug 29, 2026
5d3a4ea
fix(node): close write-attempt ownership gaps on advisory-lock release
beardthelion Aug 29, 2026
2ad2f12
fix(node): gate replication tail on publish durability and split clos…
beardthelion Aug 29, 2026
487687c
fix(node): add ValidatedRepoDiskPath for CodeQL and reconcile tail du…
beardthelion Aug 29, 2026
9d02139
fix(node): close three jatmn findings on advisory-lock PR
beardthelion Aug 30, 2026
85c6f48
fix(node): verify adversarial findings on advisory-lock PR
beardthelion Aug 30, 2026
5cd6703
fix(node): compensate under the write lock and harden tail durability…
beardthelion Aug 30, 2026
22da2d6
fix(node): refuse stale under-lock refresh and gate tail durability o…
beardthelion Aug 30, 2026
c306933
style(node): rustfmt advisory-lock gap-closure commits
beardthelion Aug 30, 2026
3a82ccb
fix(node): satisfy clippy on stale-local download test path join
beardthelion Aug 30, 2026
c40f299
feat(node): add a publish attempt/stage boundary
kevincodex1 Aug 31, 2026
21646bc
feat(node): stamp uploads with an attempt id and preserve dispatch am…
kevincodex1 Aug 31, 2026
5e548d2
fix(node): key durability, cache validity and fork cleanup on the att…
kevincodex1 Aug 31, 2026
535c5dc
fix(node): settle the cache contract on the abandoned path and the at…
beardthelion Sep 7, 2026
f048a1e
fix(node): settle the abandoned tree at the disconnect, not at reaper…
beardthelion Sep 7, 2026
957d50b
fix(node): let exactly one classifier settle an abandoned tree
beardthelion Sep 7, 2026
892c752
Merge origin/main into fix/279-advisory-lock-session-affinity
beardthelion Nov 4, 2026
28f3fc2
test(node): use the test's own pool now that Db::pool is gone
beardthelion Nov 4, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Next Next commit
fix(node): close the advisory-lock probe's session if it is dropped m…
…id-acquire

A cancelled .await does not cancel an already-sent SQL statement, so a
pg_try_advisory_lock whose future is dropped still takes the lock
server-side while the caller abandons the result, leaving nothing to
release it. The connection then returns to the pool holding the lock and
wedges that repo until sqlx recycles the session.

Introduce LockProbe, which owns the connection across the in-flight
try-lock and closes it in its own Drop if it is still held. close_on_drop
is a one-way setter, so the arming lives in Drop rather than being set up
front and cleared on success; disarming is Option::take, which is what
into_conn does once an acquire is actually observed. This is now the only
place that issues pg_try_advisory_lock.

The committed gate drops a probe without taking its connection, which is
the state a cancellation leaves behind, and polls a standalone observer
until the lock frees. Deterministic on purpose: the timing sweep that
found this window leaks roughly 1 in 600, which is not something a CI
gate can rest on. Observed RED before this change with the lock still
held for the full 10s window.

Refs #279
  • Loading branch information
beardthelion committed Aug 18, 2026
commit 7af306701d72e1ea87986a759eb94122b711f3e3
210 changes: 210 additions & 0 deletions crates/gitlawb-node/src/git/repo_store.rs
Original file line number Diff line number Diff line change
Expand Up @@ -544,6 +544,70 @@ fn validate_repo_name(repo_name: &str) -> Result<()> {
Ok(())
}

/// Owns a lock-pool connection across an in-flight `pg_try_advisory_lock`.
///
/// A cancelled `.await` does not cancel an already-sent SQL statement, so a
/// try-lock whose future is dropped still takes the lock server-side while the
/// caller abandons the result. Protection therefore has to exist *before* the
/// statement goes out, which is what this type is: its `Drop` closes any
/// connection still held, ending the session so Postgres frees the lock.
///
/// `close_on_drop()` is a one-way setter, so the arming lives here in `Drop`
/// rather than being set up front and cleared on success; "disarming" is
/// `Option::take`, which is what `take_conn` does once an acquire is observed.
/// This is the only place that issues `pg_try_advisory_lock`.
// No production caller until U3 wires this into `acquire_write`; the attribute
// comes off in that unit.
#[allow(dead_code)]
struct LockProbe {
conn: Option<sqlx::pool::PoolConnection<sqlx::Postgres>>,
}

#[allow(dead_code)] // ditto: U3 removes this with the wiring
impl LockProbe {
fn new(conn: sqlx::pool::PoolConnection<sqlx::Postgres>) -> Self {
Self { conn: Some(conn) }
}

/// Send the try-lock on the owned connection.
async fn try_lock(&mut self, key: i64) -> Result<bool> {
let conn = self
.conn
.as_mut()
.context("LockProbe::try_lock after the connection was taken")?;
let row: (bool,) = sqlx::query_as("SELECT pg_try_advisory_lock($1)")
.bind(key)
.fetch_one(&mut **conn)
.await
.context("trying advisory lock")?;
Ok(row.0)
}

/// Hand the lock-owning connection out, leaving `Drop` with nothing to close.
/// Only call this after `try_lock` returned true.
///
/// Named `take_` rather than `into_` deliberately: clippy expects an `into_*`
/// method to consume `self`, which a type implementing `Drop` cannot do
/// without tripping E0509.
fn take_conn(&mut self) -> Option<sqlx::pool::PoolConnection<sqlx::Postgres>> {
self.conn.take()
}
}

impl Drop for LockProbe {
fn drop(&mut self) {
if let Some(mut conn) = self.conn.take() {
// Still holding the connection here means the try-lock's future was
// dropped before `take_conn` ran, so the statement may well have
// completed server-side and taken the lock with nobody left to
// release it. Close the connection instead of returning it to the
// pool: ending the session is what makes Postgres free the lock.
warn!("advisory-lock probe dropped before handing off its connection — closing the session to free the lock");
conn.close_on_drop();
}
}
}

/// Guard returned by `acquire_write()`. Holds the Postgres advisory lock and
/// uploads to Tigris + releases the lock on `release()`.
pub struct RepoWriteGuard {
Expand Down Expand Up @@ -1839,5 +1903,151 @@ mod tests {
.bind(key)
.execute(&mut *checker)
.await;
// ── U1: cancellation-safe lock probe ───────────────────────────────────

/// A pool with every reaping path disabled, so a leaked lock persists through
/// the observation window instead of being freed by ambient recycling.
async fn no_reap_pool(opts: &sqlx::postgres::PgConnectOptions, max: u32) -> PgPool {
sqlx::postgres::PgPoolOptions::new()
.max_connections(max)
.acquire_timeout(std::time::Duration::from_secs(5))
.min_connections(0)
.idle_timeout(None)
.max_lifetime(None)
.test_before_acquire(false)
.connect_with(opts.clone())
.await
.expect("no-reap pool")
}

/// Poll a STANDALONE connection until the key is free, or the deadline passes.
///
/// Standalone, never from the pool under test: pool reuse would hand the
/// observer the lock-holding session itself, where `pg_try_advisory_lock`
/// succeeds reentrantly and hides the very leak being measured. Polling rather
/// than asserting once because `PoolConnection::drop` spawns the close.
async fn poll_until_free(
opts: &sqlx::postgres::PgConnectOptions,
key: i64,
deadline: std::time::Duration,
) -> bool {
use sqlx::Connection;
let start = std::time::Instant::now();
let mut observer = sqlx::PgConnection::connect_with(opts)
.await
.expect("standalone observer connection");
while start.elapsed() < deadline {
let got: (bool,) = sqlx::query_as("SELECT pg_try_advisory_lock($1)")
.bind(key)
.fetch_one(&mut observer)
.await
.expect("observer try-lock");
if got.0 {
let _: (bool,) = sqlx::query_as("SELECT pg_advisory_unlock($1)")
.bind(key)
.fetch_one(&mut observer)
.await
.expect("observer unlock");
return true;
}
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
}
false
}

/// THE COMMITTED GATE for the cancellation window (U1).
///
/// Dropping the probe without taking its connection is exactly the state a
/// cancellation between the try-lock's send and the guard's construction
/// leaves behind. Deterministic on purpose: the timing sweep that first found
/// this window leaks about 1 in 600, which is not a signal a CI gate can rest
/// on. That sweep stays a local repro.
#[sqlx::test]
async fn lock_probe_dropped_without_taking_frees_the_lock(pool: PgPool) {
let opts = (*pool.connect_options()).clone();
let lock_pool = no_reap_pool(&opts, 4).await;
let key: i64 = 990_001;

{
let mut probe = LockProbe::new(lock_pool.acquire().await.unwrap());
assert!(
probe.try_lock(key).await.unwrap(),
"probe should take a free key"
);
// dropped here WITHOUT take_conn(): the cancellation shape
}

assert!(
poll_until_free(&opts, key, std::time::Duration::from_secs(10)).await,
"lock must be freed after a probe is dropped without taking its connection"
);
}

/// Must-not: a successful acquire hands the connection out intact, so the
/// normal path does not pay a reconnect per write.
#[sqlx::test]
async fn lock_probe_take_conn_yields_a_usable_connection(pool: PgPool) {
let opts = (*pool.connect_options()).clone();
let lock_pool = no_reap_pool(&opts, 4).await;
let key: i64 = 990_002;

let mut probe = LockProbe::new(lock_pool.acquire().await.unwrap());
assert!(probe.try_lock(key).await.unwrap());
let mut conn = probe
.take_conn()
.expect("connection after a successful acquire");
drop(probe);

let one: (i32,) = sqlx::query_as("SELECT 1")
.fetch_one(&mut *conn)
.await
.expect("handed-out connection must still be usable");
assert_eq!(one.0, 1);

let released: (bool,) = sqlx::query_as("SELECT pg_advisory_unlock($1)")
.bind(key)
.fetch_one(&mut *conn)
.await
.unwrap();
assert!(released.0, "the handed-out connection still owns the lock");
}

/// Must-not: a failed probe returns its connection without closing it. Nothing
/// was locked, so closing would be pure churn, and closing on every failed
/// probe would make a 60-attempt spinner tear down 60 backends.
#[sqlx::test]
async fn lock_probe_failed_acquire_does_not_hold_anything(pool: PgPool) {
let opts = (*pool.connect_options()).clone();
let lock_pool = no_reap_pool(&opts, 4).await;
let key: i64 = 990_003;

// a standalone holder takes the key first
use sqlx::Connection;
let mut holder = sqlx::PgConnection::connect_with(&opts).await.unwrap();
let held: (bool,) = sqlx::query_as("SELECT pg_try_advisory_lock($1)")
.bind(key)
.fetch_one(&mut holder)
.await
.unwrap();
assert!(held.0);

{
let mut probe = LockProbe::new(lock_pool.acquire().await.unwrap());
assert!(
!probe.try_lock(key).await.unwrap(),
"probe must observe false for a key held elsewhere"
);
}

// the holder still owns it: the failed probe neither took nor released it
let still: (i64,) = sqlx::query_as(
"SELECT count(*) FROM pg_locks WHERE locktype='advisory' \
AND ((classid::bigint<<32)|objid::bigint) = $1",
)
.bind(key)
.fetch_one(&pool)
.await
.unwrap();
assert_eq!(still.0, 1, "the original holder must still own the key");
}
}