Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
fix mixed runtime exhaustion reason
  • Loading branch information
ndycode committed May 1, 2026
commit be6fc7da77c9e335111ad1a450c266ca07b0b1b1
12 changes: 12 additions & 0 deletions lib/runtime-rotation-proxy.ts
Original file line number Diff line number Diff line change
Expand Up @@ -951,6 +951,7 @@ export async function startRuntimeRotationProxy(
Math.min(accountCount, maxRuntimeAccountAttempts),
);
let transientAttempts = 0;
let transientExhaustionReason: ExhaustionReason | null = null;

while (
attemptedIndexes.size < accountCount &&
Expand Down Expand Up @@ -987,6 +988,7 @@ export async function startRuntimeRotationProxy(
exhaustionReason = "auth-failure";
if (!refreshed.retryable) continue;
transientAttempts += 1;
transientExhaustionReason = "auth-failure";
status.retries += 1;
status.rotations += 1;
continue;
Expand All @@ -1003,6 +1005,7 @@ export async function startRuntimeRotationProxy(
);
exhaustionReason = "auth-failure";
transientAttempts += 1;
transientExhaustionReason = "auth-failure";
status.retries += 1;
status.rotations += 1;
continue;
Expand Down Expand Up @@ -1048,6 +1051,7 @@ export async function startRuntimeRotationProxy(
accountManager.saveToDiskDebounced();
exhaustionReason = "network-error";
transientAttempts += 1;
transientExhaustionReason = "network-error";
status.retries += 1;
status.rotations += 1;
continue;
Expand All @@ -1072,6 +1076,7 @@ export async function startRuntimeRotationProxy(
accountManager.saveToDiskDebounced();
exhaustionReason = "rate-limit";
transientAttempts += 1;
transientExhaustionReason = "rate-limit";
status.retries += 1;
status.rotations += 1;
continue;
Expand Down Expand Up @@ -1123,6 +1128,7 @@ export async function startRuntimeRotationProxy(
accountManager.saveToDiskDebounced();
exhaustionReason = "auth-failure";
transientAttempts += 1;
transientExhaustionReason = "auth-failure";
status.retries += 1;
status.rotations += 1;
continue;
Expand All @@ -1140,6 +1146,7 @@ export async function startRuntimeRotationProxy(
accountManager.saveToDiskDebounced();
exhaustionReason = "server-error";
transientAttempts += 1;
transientExhaustionReason = "server-error";
status.retries += 1;
status.rotations += 1;
continue;
Expand Down Expand Up @@ -1209,6 +1216,11 @@ export async function startRuntimeRotationProxy(
exhaustionReason !== "deactivated"
) {
exhaustionReason = "budget";
} else if (
exhaustionReason === "deactivated" &&
transientExhaustionReason
) {
exhaustionReason = transientExhaustionReason;
}
Comment thread
greptile-apps[bot] marked this conversation as resolved.
Resolved

await usageRecorder?.record({
Expand Down
66 changes: 66 additions & 0 deletions test/runtime-rotation-proxy.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -828,6 +828,40 @@ describe("runtime rotation proxy", () => {
).toEqual(["acc_2"]);
});

it("disables a 403 workspace-disabled account and retries another account", async () => {
const now = Date.now();
const accountManager = new AccountManager(undefined, createStorage(now, 2));
const { calls, fetchImpl } = createRecordingFetch((_call, attempt) => {
if (attempt === 1) {
return new Response(
JSON.stringify({ error: { code: "workspace_disabled" } }),
{
status: HTTP_STATUS.FORBIDDEN,
headers: { "content-type": "application/json" },
},
);
}
return textEventStream("data: recovered\n\n");
});
const proxy = await startProxy({ accountManager, fetchImpl });

const response = await postResponses(proxy, { model: "gpt-5-codex" });

expect(response.status).toBe(HTTP_STATUS.OK);
expect(await response.text()).toBe("data: recovered\n\n");
expect(calls.map((call) => call.headers.get(OPENAI_HEADERS.ACCOUNT_ID))).toEqual([
"acc_1",
"acc_2",
]);
expect(accountManager.getAccountByIndex(0)?.enabled).toBe(false);
expect(accountManager.getAccountByIndex(1)?.enabled).toBe(true);
expect(proxy.getStatus()).toMatchObject({
retries: 1,
rotations: 1,
lastAccountIndex: 1,
});
});

it("returns pool exhaustion after all accounts are deactivated", async () => {
const now = Date.now();
const accountManager = new AccountManager(undefined, createStorage(now, 6));
Expand All @@ -853,6 +887,38 @@ describe("runtime rotation proxy", () => {
);
});

it("reports a transient exhaustion reason when deactivated skips also occurred", async () => {
const now = Date.now();
const accountManager = new AccountManager(undefined, createStorage(now, 3));
const { calls, fetchImpl } = createRecordingFetch((_call, attempt) => {
if (attempt === 2) {
return new Response("upstream failed", { status: 503 });
}
return new Response(
JSON.stringify({ error: { code: "deactivated_workspace" } }),
{
status: 402,
headers: { "content-type": "application/json" },
},
);
});
const proxy = await startProxy({ accountManager, fetchImpl });

const response = await postResponses(proxy, { model: "gpt-5-codex" });
const payload = (await response.json()) as { error: { reason: string } };

expect(response.status).toBe(HTTP_STATUS.SERVICE_UNAVAILABLE);
expect(payload.error.reason).toBe("server-error");
expect(calls.map((call) => call.headers.get(OPENAI_HEADERS.ACCOUNT_ID))).toEqual([
"acc_1",
"acc_2",
"acc_3",
]);
expect(accountManager.getAccountByIndex(0)?.enabled).toBe(false);
expect(accountManager.getAccountByIndex(1)?.cooldownReason).toBe("server-error");
expect(accountManager.getAccountByIndex(2)?.enabled).toBe(false);
});

it("forwards unrelated 402 errors without disabling the account", async () => {
const now = Date.now();
const accountManager = new AccountManager(undefined, createStorage(now, 2));
Expand Down