Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
fix remaining runtime review findings
  • Loading branch information
ndycode committed May 1, 2026
commit e446dc32eb031ba9bf7dc8cd07de314f2aa6906a
2 changes: 1 addition & 1 deletion lib/auto-update-checker.ts
Original file line number Diff line number Diff line change
Expand Up @@ -542,7 +542,7 @@ function taskkillProcessTree(pid: number): Promise<boolean> {
"/d",
"/s",
"/c",
`taskkill /PID ${pid} /T /F >NUL 2>NUL`,
`taskkill /PID ${pid} /T /F`,
],
{
stdio: "ignore",
Expand Down
15 changes: 10 additions & 5 deletions lib/request/fetch-helpers.ts
Original file line number Diff line number Diff line change
Expand Up @@ -329,13 +329,12 @@ export function isWorkspaceDisabledError(
bodyText: string,
): boolean {
const normalizedCode = typeof code === "string" ? code.trim().toLowerCase() : "";
const haystack = `${normalizedCode} ${bodyText}`.toLowerCase();
const normalizedTokens = normalizedCode
.split(/[^a-z0-9_]+/i)
.map((token) => token.trim())
.filter((token) => token.length > 0);

if (status === 402) {
const normalizedTokens = normalizedCode
.split(/[^a-z0-9_]+/i)
.map((token) => token.trim())
.filter((token) => token.length > 0);
return (
normalizedTokens.includes("deactivated_workspace") ||
/\bdeactivated_workspace\b/i.test(bodyText)
Expand All @@ -346,6 +345,12 @@ export function isWorkspaceDisabledError(
return false;
}

const haystack = `${normalizedCode} ${bodyText}`.toLowerCase();
const normalizedTokens = normalizedCode
.split(/[^a-z0-9_]+/i)
.map((token) => token.trim())
.filter((token) => token.length > 0);

const disabledPatterns = [
/workspace.*(?:disabled|expired|deactivated|terminated)/i,
/account\s+(?:has\s+been|is)\s+(?:disabled|expired|deactivated|terminated|closed)/i,
Expand Down
19 changes: 12 additions & 7 deletions lib/runtime-rotation-proxy.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1086,19 +1086,24 @@ export async function startRuntimeRotationProxy(
const bodyText = await readErrorBody(upstream);
const errorCode = extractErrorCodeFromBody(bodyText);
if (isWorkspaceDisabledError(upstream.status, errorCode, bodyText)) {
const accountWasEnabled =
accountManager.getAccountByIndex(refreshed.account.index)?.enabled !==
false;
accountManager.refundToken(
refreshed.account,
context.family,
context.model,
);
accountManager.recordFailure(
refreshed.account,
context.family,
context.model,
);
accountManager.setAccountEnabled(refreshed.account.index, false);
if (accountWasEnabled) {
accountManager.recordFailure(
refreshed.account,
context.family,
context.model,
);
accountManager.setAccountEnabled(refreshed.account.index, false);
accountManager.saveToDiskDebounced();
}
sessionAffinityStore?.forgetSession(context.sessionKey);
accountManager.saveToDiskDebounced();
exhaustionReason = "deactivated";
status.retries += 1;
status.rotations += 1;
Expand Down
2 changes: 1 addition & 1 deletion scripts/test-model-matrix.js
Original file line number Diff line number Diff line change
Expand Up @@ -153,7 +153,7 @@ function runQuietWindowsTaskkill(pid) {
"/d",
"/s",
"/c",
`taskkill /F /T /PID ${pid} >NUL 2>NUL`,
`taskkill /F /T /PID ${pid}`,
]);
}

Expand Down
2 changes: 1 addition & 1 deletion test/auto-update-checker.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -894,7 +894,7 @@ describe("auto-update-checker", () => {
"/d",
"/s",
"/c",
"taskkill /PID 1234 /T /F >NUL 2>NUL",
"taskkill /PID 1234 /T /F",
],
expect.objectContaining({ stdio: "ignore", windowsHide: true }),
);
Expand Down
54 changes: 54 additions & 0 deletions test/runtime-rotation-proxy.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -862,6 +862,60 @@ describe("runtime rotation proxy", () => {
});
});

it("records a concurrent deactivation failure once for the account", async () => {
const now = Date.now();
const accountManager = new AccountManager(undefined, createStorage(now, 1));
const recordFailureSpy = vi.spyOn(accountManager, "recordFailure");
let disabledCalls = 0;
let releaseDisabledCalls: (() => void) | null = null;
const allDisabledCallsArrived = new Promise<void>((resolve) => {
releaseDisabledCalls = resolve;
});
const { calls, fetchImpl } = createRecordingFetch(async (call) => {
if (call.headers.get(OPENAI_HEADERS.ACCOUNT_ID) === "acc_1") {
disabledCalls += 1;
if (disabledCalls === 2) releaseDisabledCalls?.();
await allDisabledCallsArrived;
return new Response(
JSON.stringify({ error: { code: "deactivated_workspace" } }),
{
status: 402,
headers: { "content-type": "application/json" },
},
);
}
return textEventStream("data: recovered\n\n");
});
const proxy = await startProxy({ accountManager, fetchImpl });
const body = {
model: "gpt-5-codex",
stream: true,
metadata: { session_id: "thread-concurrent-deactivated" },
};

const responses = await Promise.all([postResponses(proxy, body), postResponses(proxy, body)]);
const payloads = (await Promise.all(responses.map((response) => response.json()))) as Array<{
error: { reason: string };
}>;

expect(responses.map((response) => response.status)).toEqual([
HTTP_STATUS.SERVICE_UNAVAILABLE,
HTTP_STATUS.SERVICE_UNAVAILABLE,
]);
expect(payloads.map((payload) => payload.error.reason)).toEqual([
"deactivated",
"deactivated",
]);
expect(calls.map((call) => call.headers.get(OPENAI_HEADERS.ACCOUNT_ID))).toEqual([
"acc_1",
"acc_1",
]);
expect(
recordFailureSpy.mock.calls.filter(([account]) => account.index === 0),
).toHaveLength(1);
expect(accountManager.getAccountByIndex(0)?.enabled).toBe(false);
});

it("returns pool exhaustion after all accounts are deactivated", async () => {
const now = Date.now();
const accountManager = new AccountManager(undefined, createStorage(now, 6));
Expand Down
4 changes: 2 additions & 2 deletions test/test-model-matrix-script.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -247,13 +247,13 @@ describe("test-model-matrix script helpers", () => {
expect(spawnSync).toHaveBeenNthCalledWith(
1,
"cmd.exe",
["/d", "/s", "/c", "taskkill /F /T /PID 1001 >NUL 2>NUL"],
["/d", "/s", "/c", "taskkill /F /T /PID 1001"],
expect.objectContaining({ windowsHide: true, stdio: "ignore" }),
);
expect(spawnSync).toHaveBeenNthCalledWith(
2,
"cmd.exe",
["/d", "/s", "/c", "taskkill /F /T /PID 2002 >NUL 2>NUL"],
["/d", "/s", "/c", "taskkill /F /T /PID 2002"],
expect.objectContaining({ windowsHide: true, stdio: "ignore" }),
);
} finally {
Expand Down