Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions lib/local-bridge.ts
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,10 @@ function forwardHeaders(headers: Headers, runtimeClientApiKey?: string): Headers
// caller's local credential across the bridge boundary and could change which
// auth the runtime proxy evaluates — strip it unconditionally.
result.delete("x-api-key");
// Same contract: never carry an inbound Cookie / proxy-auth header upstream
// alongside the managed token.
result.delete("cookie");
result.delete("proxy-authorization");
// runtime-proxy-03: present the runtime proxy's client token. We replace the
// inbound client's Authorization (already validated by the bridge) rather than
// forwarding it verbatim, so the bridge can authenticate to an auth-enabled
Expand Down
11 changes: 10 additions & 1 deletion lib/quota-cache.ts
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,16 @@ interface QuotaCacheFile {

const QUOTA_CACHE_PATH = join(getCodexMultiAuthDir(), "quota-cache.json");
const QUOTA_CACHE_LABEL = basename(QUOTA_CACHE_PATH);
const RETRYABLE_FS_CODES = new Set(["EBUSY", "EPERM"]);
// Align with the shared FILE_RETRY_CODES taxonomy (lib/fs-retry.ts) so a
// transient Windows lock (AV/indexer/concurrent reader) on the quota cache is
// retried consistently with every other fs path, not just EBUSY/EPERM.
const RETRYABLE_FS_CODES = new Set([
"EBUSY",
"EPERM",
"EAGAIN",
"ENOTEMPTY",
"EACCES",
]);
Comment thread
ndycode marked this conversation as resolved.
Resolved
let quotaCacheWriteQueue: Promise<void> = Promise.resolve();

function isRetryableFsError(error: unknown): boolean {
Expand Down
69 changes: 63 additions & 6 deletions lib/runtime-rotation-proxy.ts
Original file line number Diff line number Diff line change
Expand Up @@ -268,6 +268,10 @@ function createOutboundHeaders(
}
headers.delete("host");
headers.delete("x-api-key");
// Never forward inbound client credentials upstream: a Cookie / proxy-auth
// header would ride along with the managed OAuth Bearer to OpenAI.
headers.delete("cookie");
headers.delete("proxy-authorization");
headers.set("authorization", `Bearer ${accessToken}`);
headers.set(OPENAI_HEADERS.ACCOUNT_ID, accountId);
headers.set(OPENAI_HEADERS.BETA, OPENAI_HEADER_VALUES.BETA_RESPONSES);
Expand Down Expand Up @@ -892,9 +896,62 @@ function parseRetryAfterBodyMs(bodyText: string, now: number): number | null {
return null;
}

async function readErrorBody(response: Response): Promise<string> {
async function readErrorBody(
response: Response,
timeoutMs: number,
maxBytes = 1024 * 1024,
): Promise<string> {
// The outbound fetch's abort timer is cleared once headers arrive, so a
// stalled error body would otherwise hang this handler forever (the success
// path is per-chunk stall-bounded; the error path was not). Read the body via
// a reader, bound it by an idle timeout AND a size cap, and cancel the stream
// on timeout/overflow so the socket is released.
const body = response.body;
if (!body || typeof body.getReader !== "function") {
// Fallback for impls without a streamable body: race text() against a timer.
try {
return await withTimeout(
response.text(),
timeoutMs,
() => undefined,
"error body stalled",
);
} catch {
return "";
}
}
const reader = body.getReader();
const chunks: Uint8Array[] = [];
let total = 0;
try {
for (;;) {
let idleTimer: ReturnType<typeof setTimeout> | undefined;
const idle = new Promise<never>((_resolve, reject) => {
idleTimer = setTimeout(
() => reject(new Error("error body stalled")),
Math.max(1, timeoutMs),
);
});
let result: Awaited<ReturnType<typeof reader.read>>;
try {
result = await Promise.race([reader.read(), idle]);
} finally {
if (idleTimer) clearTimeout(idleTimer);
}
if (result.done) break;
if (result.value) {
total += result.value.byteLength;
if (total > maxBytes) break; // cap: enough for diagnostics, no OOM
chunks.push(result.value);
}
}
} catch {
// stalled or errored — fall through with whatever we have
} finally {
await reader.cancel().catch(() => undefined);
}
try {
return await response.text();
return Buffer.concat(chunks).toString("utf8");
} catch {
return "";
}
Expand Down Expand Up @@ -1734,7 +1791,7 @@ export async function startRuntimeRotationProxy(
}

if (upstream.status === HTTP_STATUS.TOO_MANY_REQUESTS) {
const bodyText = await readErrorBody(upstream);
const bodyText = await readErrorBody(upstream, streamStallTimeoutMs);
const retryAfterMs =
parseRetryAfterHeaderMs(upstream.headers, now()) ??
parseRetryAfterBodyMs(bodyText, now()) ??
Expand All @@ -1759,7 +1816,7 @@ export async function startRuntimeRotationProxy(
}

if (upstream.status === 402 || upstream.status === HTTP_STATUS.FORBIDDEN) {
const bodyText = await readErrorBody(upstream);
const bodyText = await readErrorBody(upstream, streamStallTimeoutMs);
const errorCode = extractErrorCodeFromBody(bodyText);
if (isWorkspaceDisabledError(upstream.status, errorCode, bodyText)) {
const accountWasEnabled =
Expand Down Expand Up @@ -1856,7 +1913,7 @@ export async function startRuntimeRotationProxy(
}

if (upstream.status === HTTP_STATUS.UNAUTHORIZED) {
const bodyText = await readErrorBody(upstream);
const bodyText = await readErrorBody(upstream, streamStallTimeoutMs);
accountManager.refundToken(refreshed.account, context.family, context.model);
accountManager.recordFailure(refreshed.account, context.family, context.model);
if (isTokenInvalidationError(bodyText)) {
Expand Down Expand Up @@ -1902,7 +1959,7 @@ export async function startRuntimeRotationProxy(
}

if (upstream.status >= 500) {
await readErrorBody(upstream);
await readErrorBody(upstream, streamStallTimeoutMs);
accountManager.refundToken(refreshed.account, context.family, context.model);
accountManager.recordFailure(refreshed.account, context.family, context.model);
accountManager.markAccountCoolingDown(
Expand Down
10 changes: 8 additions & 2 deletions lib/runtime/runtime-observability.ts
Original file line number Diff line number Diff line change
Expand Up @@ -195,13 +195,19 @@ function ensureSnapshotState(): RuntimeObservabilitySnapshot {
async function writeSnapshot(snapshot: RuntimeObservabilitySnapshot): Promise<void> {
const dir = getCodexMultiAuthDir();
const path = getSnapshotPath();
await fs.mkdir(dir, { recursive: true });
// The snapshot persists account identifiers (lastAccountId/label/index), so
// keep it owner-only on POSIX like the other sensitive writers (logger,
// local-client-tokens). mode is a no-op on win32 (ACL-based).
await fs.mkdir(dir, { recursive: true, mode: 0o700 });
let lastError: unknown = null;
for (let attempt = 0; attempt < 3; attempt += 1) {
const tempPath = `${path}.${process.pid}.${Date.now()}.${attempt}.tmp`;
let moved = false;
try {
await fs.writeFile(tempPath, JSON.stringify(snapshot, null, 2), "utf-8");
await fs.writeFile(tempPath, JSON.stringify(snapshot, null, 2), {
encoding: "utf-8",
mode: 0o600,
});
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
await fs.rename(tempPath, path);
moved = true;
return;
Expand Down
30 changes: 19 additions & 11 deletions lib/storage.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ import { existsSync, promises as fs, readFileSync } from "node:fs";
import { basename, dirname, join } from "node:path";
import { ACCOUNT_LIMITS } from "./constants.js";
import { StorageError } from "./errors.js";
import { shouldRetryFileOperation, withFileOperationRetry } from "./fs-retry.js";
import { createLogger } from "./logger.js";
import {
exportNamedBackupFile,
Expand Down Expand Up @@ -273,8 +274,7 @@ async function copyFileWithRetry(
return;
}
const canRetry =
(code === "EPERM" || code === "EBUSY") &&
attempt + 1 < BACKUP_COPY_MAX_ATTEMPTS;
shouldRetryFileOperation(error) && attempt + 1 < BACKUP_COPY_MAX_ATTEMPTS;
if (canRetry) {
await new Promise((resolve) =>
setTimeout(resolve, BACKUP_COPY_BASE_DELAY_MS * 2 ** attempt),
Expand All @@ -295,10 +295,8 @@ async function renameFileWithRetry(
await fs.rename(sourcePath, destinationPath);
return;
} catch (error) {
const code = (error as NodeJS.ErrnoException).code;
const canRetry =
(code === "EPERM" || code === "EBUSY" || code === "EAGAIN") &&
attempt + 1 < BACKUP_COPY_MAX_ATTEMPTS;
shouldRetryFileOperation(error) && attempt + 1 < BACKUP_COPY_MAX_ATTEMPTS;
if (!canRetry) {
throw error;
}
Expand Down Expand Up @@ -1834,13 +1832,20 @@ async function saveAccountsUnlocked(storage: AccountStorageV3): Promise<void> {
checksum: computeSha256(content),
content,
};
await fs.writeFile(walPath, JSON.stringify(journalEntry), {
encoding: "utf-8",
mode: 0o600,
});
// Secret-bearing WAL write: retry transient Windows locks via the shared
// taxonomy so a momentary AV/indexer/concurrent-reader lock can't fail an
// otherwise-valid save (EBUSY/EPERM/EAGAIN/ENOTEMPTY/EACCES).
await withFileOperationRetry(() =>
fs.writeFile(walPath, JSON.stringify(journalEntry), {
encoding: "utf-8",
mode: 0o600,
}),
);
},
writeTemp: (tempPath: string, content: string) =>
fs.writeFile(tempPath, content, { encoding: "utf-8", mode: 0o600 }),
withFileOperationRetry(() =>
fs.writeFile(tempPath, content, { encoding: "utf-8", mode: 0o600 }),
),
statTemp: (tempPath: string) => fs.stat(tempPath),
renameTempToPath: async (tempPath: string) => {
let lastError: NodeJS.ErrnoException | null = null;
Expand Down Expand Up @@ -1875,8 +1880,11 @@ async function saveAccountsUnlocked(storage: AccountStorageV3): Promise<void> {
}
},
cleanupTemp: async (tempPath: string) => {
// The temp file holds the full account store (refresh tokens, 0o600).
// Retry cleanup so a transient lock can't strand a secret-bearing *.tmp
// next to the destination; swallow a persistent failure (best effort).
try {
await fs.unlink(tempPath);
await withFileOperationRetry(() => fs.unlink(tempPath));
} catch {
// Ignore cleanup failure.
}
Expand Down
4 changes: 3 additions & 1 deletion test/local-bridge.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -317,13 +317,15 @@ describe("local bridge", () => {
headers: {
authorization: "Bearer inbound-client-token",
"x-api-key": "inbound-secret-key",
cookie: "session=inbound-cookie-secret",
},
});

const forwarded = calls.find((c) => c.url.endsWith("/v1/models"));
const headers = new Headers(forwarded?.init?.headers as HeadersInit);
// runtime-proxy-02: neither inbound credential header crosses the bridge.
// runtime-proxy-02: no inbound credential header crosses the bridge.
expect(headers.get("authorization")).toBeNull();
expect(headers.get("x-api-key")).toBeNull();
expect(headers.get("cookie")).toBeNull();
});
});
11 changes: 3 additions & 8 deletions test/mcodex-statusline-scope.test.ts
Original file line number Diff line number Diff line change
@@ -1,12 +1,12 @@
import { spawnSync } from "node:child_process";
import {
existsSync,
mkdirSync,
mkdtempSync,
rmSync,
writeFileSync,
} from "node:fs";
import { tmpdir } from "node:os";
import { createHash } from "node:crypto";
import { dirname, join } from "node:path";
import { fileURLToPath } from "node:url";
import { afterEach, describe, expect, it } from "vitest";
Expand Down Expand Up @@ -82,13 +82,8 @@ function runWrapper(cwd: string, env: NodeJS.ProcessEnv) {
}

function hasDistBuild(): boolean {
try {
// paths.js is emitted by `npm run build`; skip if the tree isn't built.
// eslint-disable-next-line @typescript-eslint/no-require-imports
return require("node:fs").existsSync(distPathsModule);
} catch {
return false;
}
// paths.js is emitted by `npm run build`; skip if the tree isn't built.
return existsSync(distPathsModule);
}

describe.runIf(hasDistBuild())("mcodex statusline per-project accounts (PR #500)", () => {
Expand Down
3 changes: 3 additions & 0 deletions test/runtime-rotation-proxy.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1120,6 +1120,7 @@ describe("runtime rotation proxy", () => {
accept: "application/json",
connection: "close",
"x-custom-trace": "trace-1",
cookie: "session=inbound-cookie-secret",
},
)
).text();
Expand All @@ -1130,6 +1131,8 @@ describe("runtime rotation proxy", () => {
expect(calls[0]?.headers.get("connection")).toBeNull();
expect(calls[0]?.headers.get("authorization")).toBe("Bearer access-1");
expect(calls[0]?.headers.get("x-api-key")).toBeNull();
// Inbound client credentials must never ride upstream with the managed token.
expect(calls[0]?.headers.get("cookie")).toBeNull();
});
Comment thread
greptile-apps[bot] marked this conversation as resolved.
Resolved

it("strips expect before forwarding to fetch", async () => {
Expand Down