Skip to content
Merged
Show file tree
Hide file tree
Changes from 7 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 7 additions & 2 deletions lib/auto-update-checker.ts
Original file line number Diff line number Diff line change
Expand Up @@ -537,8 +537,13 @@ function taskkillProcessTree(pid: number): Promise<boolean> {
timeout.unref?.();
try {
const killer = spawn(
"taskkill",
["/PID", String(pid), "/T", "/F"],
"cmd.exe",
[
"/d",
"/s",
"/c",
`taskkill /PID ${pid} /T /F`,
],
{
stdio: "ignore",
windowsHide: true,
Expand Down
23 changes: 17 additions & 6 deletions lib/request/fetch-helpers.ts
Original file line number Diff line number Diff line change
Expand Up @@ -328,12 +328,28 @@ export function isWorkspaceDisabledError(
code: unknown,
bodyText: string,
): boolean {
const normalizedCode = typeof code === "string" ? code.trim().toLowerCase() : "";

if (status === 402) {
const normalizedTokens = normalizedCode
.split(/[^a-z0-9_]+/i)
.map((token) => token.trim())
.filter((token) => token.length > 0);
return (
normalizedTokens.includes("deactivated_workspace") ||
/\bdeactivated_workspace\b/i.test(bodyText)
);
}

if (status !== 403) {
return false;
}

const normalizedCode = typeof code === "string" ? code.trim().toLowerCase() : "";
const haystack = `${normalizedCode} ${bodyText}`.toLowerCase();
const normalizedTokens = normalizedCode
.split(/[^a-z0-9_]+/i)
.map((token) => token.trim())
.filter((token) => token.length > 0);

const disabledPatterns = [
/workspace.*(?:disabled|expired|deactivated|terminated)/i,
Expand Down Expand Up @@ -362,11 +378,6 @@ export function isWorkspaceDisabledError(
return true;
}

const normalizedTokens = normalizedCode
.split(/[^a-z0-9_]+/i)
.map((token) => token.trim())
.filter((token) => token.length > 0);

return normalizedTokens.some((token) => workspaceErrorCodes.has(token));
}

Expand Down
94 changes: 89 additions & 5 deletions lib/runtime-rotation-proxy.ts
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ import {
loadRuntimePolicyState,
type RuntimePolicyDecision,
} from "./policy/runtime-policy.js";
import { isWorkspaceDisabledError } from "./request/fetch-helpers.js";
import { SessionAffinityStore } from "./session-affinity.js";
import type { OAuthAuthDetails, RequestBody, TokenResult } from "./types.js";
import { isRecord } from "./utils.js";
Expand Down Expand Up @@ -91,6 +92,7 @@ type ExhaustionReason =
| "network-error"
| "auth-failure"
| "budget"
| "deactivated"
| "no-account";
type RuntimeProxyHttpError = Error & {
statusCode: number;
Expand Down Expand Up @@ -558,6 +560,26 @@ async function readErrorBody(response: Response): Promise<string> {
}
}

function extractErrorCodeFromBody(bodyText: string): string | null {
if (!bodyText.trim()) return null;
try {
const parsed = JSON.parse(bodyText) as unknown;
if (!isRecord(parsed)) return null;
const directCode = parsed.code;
if (typeof directCode === "string" && directCode.trim()) {
return directCode.trim();
}
const maybeError = parsed.error;
if (!isRecord(maybeError)) return null;
const nestedCode = maybeError.code;
return typeof nestedCode === "string" && nestedCode.trim()
? nestedCode.trim()
: null;
} catch {
return null;
}
}

function getQuotaWindowWaitMs(headers: Headers, prefix: string, now: number): number {
const resetAfterSeconds = Number.parseInt(
headers.get(`${prefix}-reset-after-seconds`) ?? "",
Expand Down Expand Up @@ -923,12 +945,18 @@ export async function startRuntimeRotationProxy(
);
const attemptedIndexes = new Set<number>();
let exhaustionReason: ExhaustionReason = "no-account";
const accountAttemptLimit = Math.max(
const accountCount = accountManager.getAccountCount();
const transientAttemptLimit = Math.max(
1,
Math.min(accountManager.getAccountCount(), maxRuntimeAccountAttempts),
Math.min(accountCount, maxRuntimeAccountAttempts),
);
let transientAttempts = 0;
let transientExhaustionReason: ExhaustionReason | null = null;

while (attemptedIndexes.size < accountAttemptLimit) {
while (
attemptedIndexes.size < accountCount &&
transientAttempts < transientAttemptLimit
) {
const selected = chooseAccount({
accountManager,
sessionAffinityStore,
Expand Down Expand Up @@ -959,6 +987,8 @@ export async function startRuntimeRotationProxy(
accountManager.refundToken(selected, context.family, context.model);
exhaustionReason = "auth-failure";
if (!refreshed.retryable) continue;
transientAttempts += 1;
transientExhaustionReason = "auth-failure";
status.retries += 1;
status.rotations += 1;
continue;
Expand All @@ -974,6 +1004,8 @@ export async function startRuntimeRotationProxy(
"auth-failure",
);
exhaustionReason = "auth-failure";
transientAttempts += 1;
transientExhaustionReason = "auth-failure";
status.retries += 1;
status.rotations += 1;
continue;
Expand Down Expand Up @@ -1018,6 +1050,8 @@ export async function startRuntimeRotationProxy(
);
accountManager.saveToDiskDebounced();
exhaustionReason = "network-error";
transientAttempts += 1;
transientExhaustionReason = "network-error";
status.retries += 1;
status.rotations += 1;
continue;
Expand All @@ -1041,11 +1075,52 @@ export async function startRuntimeRotationProxy(
);
accountManager.saveToDiskDebounced();
exhaustionReason = "rate-limit";
transientAttempts += 1;
transientExhaustionReason = "rate-limit";
status.retries += 1;
status.rotations += 1;
continue;
}

if (upstream.status === 402 || upstream.status === HTTP_STATUS.FORBIDDEN) {
const bodyText = await readErrorBody(upstream);
const errorCode = extractErrorCodeFromBody(bodyText);
if (isWorkspaceDisabledError(upstream.status, errorCode, bodyText)) {
const accountWasEnabled =
accountManager.getAccountByIndex(refreshed.account.index)?.enabled !==
false;
accountManager.refundToken(
refreshed.account,
context.family,
context.model,
);
if (accountWasEnabled) {
accountManager.recordFailure(
refreshed.account,
context.family,
context.model,
);
accountManager.setAccountEnabled(refreshed.account.index, false);
accountManager.saveToDiskDebounced();
}
sessionAffinityStore?.forgetSession(context.sessionKey);
exhaustionReason = "deactivated";
status.retries += 1;
status.rotations += 1;
continue;
}
Comment thread
greptile-apps[bot] marked this conversation as resolved.
Resolved

res.writeHead(upstream.status, responseHeadersForClient(upstream.headers));
res.end(bodyText);
await usageRecorder.record({
outcome: "failure",
statusCode: upstream.status,
errorCode,
account: refreshed.account,
});
return;
}

if (upstream.status === HTTP_STATUS.UNAUTHORIZED) {
await readErrorBody(upstream);
accountManager.refundToken(refreshed.account, context.family, context.model);
Expand All @@ -1057,6 +1132,8 @@ export async function startRuntimeRotationProxy(
);
accountManager.saveToDiskDebounced();
exhaustionReason = "auth-failure";
transientAttempts += 1;
transientExhaustionReason = "auth-failure";
status.retries += 1;
status.rotations += 1;
continue;
Expand All @@ -1073,6 +1150,8 @@ export async function startRuntimeRotationProxy(
);
accountManager.saveToDiskDebounced();
exhaustionReason = "server-error";
transientAttempts += 1;
transientExhaustionReason = "server-error";
status.retries += 1;
status.rotations += 1;
continue;
Expand Down Expand Up @@ -1137,10 +1216,15 @@ export async function startRuntimeRotationProxy(
}

if (
attemptedIndexes.size >= accountAttemptLimit &&
accountAttemptLimit < accountManager.getAccountCount()
transientAttempts >= transientAttemptLimit &&
attemptedIndexes.size < accountCount
) {
exhaustionReason = "budget";
} else if (
exhaustionReason === "deactivated" &&
transientExhaustionReason
) {
exhaustionReason = transientExhaustionReason;
}
Comment thread
greptile-apps[bot] marked this conversation as resolved.
Resolved

await usageRecorder?.record({
Expand Down
4 changes: 4 additions & 0 deletions scripts/codex-multi-auth.js
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,10 @@ function resolveCliVersion() {
return "";
}

/**
* @param {string[]} args
* @returns {string[]}
*/
function normalizeStandaloneArgs(args) {
if (args[0] === "auth") return args;
const firstArg = args[0] ?? "";
Expand Down
8 changes: 8 additions & 0 deletions scripts/codex-routing.js
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,10 @@ const AUTH_SUBCOMMANDS = new Set([
"debug",
]);

/**
* @param {string[]} args
* @returns {string[]}
*/
export function normalizeAuthAlias(args) {
if (args.length >= 2 && args[0] === "multi" && args[1] === "auth") {
return ["auth", ...args.slice(2)];
Expand All @@ -36,6 +40,10 @@ export function normalizeAuthAlias(args) {
return args;
}

/**
* @param {string[]} args
* @returns {boolean}
*/
export function shouldHandleMultiAuthAuth(args) {
if (args[0] !== "auth") return false;
if (args.length === 1) return true;
Expand Down
11 changes: 10 additions & 1 deletion scripts/test-model-matrix.js
Original file line number Diff line number Diff line change
Expand Up @@ -148,6 +148,15 @@ function runQuiet(command, commandArgs) {
}
}

function runQuietWindowsTaskkill(pid) {
runQuiet("cmd.exe", [
"/d",
"/s",
"/c",
`taskkill /F /T /PID ${pid}`,
]);
}

export function registerSpawnedCodex(pid) {
if (!Number.isInteger(pid) || pid <= 0) return;
spawnedCodexPids.add(pid);
Expand Down Expand Up @@ -228,7 +237,7 @@ function stopCodexServersInternal() {
spawnedCodexPids.clear();
for (const pid of tracked) {
if (process.platform === "win32") {
runQuiet("taskkill", ["/F", "/T", "/PID", String(pid)]);
runQuietWindowsTaskkill(pid);
continue;
}
runQuiet("kill", ["-9", String(pid)]);
Expand Down
9 changes: 7 additions & 2 deletions test/auto-update-checker.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -889,8 +889,13 @@ describe("auto-update-checker", () => {
await flushUpdateStartup();
await vi.advanceTimersByTimeAsync(25);
expect(childProcess.spawn).toHaveBeenLastCalledWith(
"taskkill",
["/PID", "1234", "/T", "/F"],
"cmd.exe",
[
"/d",
"/s",
"/c",
"taskkill /PID 1234 /T /F",
],
expect.objectContaining({ stdio: "ignore", windowsHide: true }),
);
killer.emit("exit", 0, null);
Expand Down
11 changes: 10 additions & 1 deletion test/fetch-helpers.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -688,12 +688,21 @@ describe('isWorkspaceDisabledError', () => {
expect(isWorkspaceDisabledError(403, 'error.usage_not_included', '')).toBe(false);
});

it('returns false for non-403 status even with disabled message', () => {
it('returns true for 402 deactivated_workspace signals', () => {
expect(isWorkspaceDisabledError(402, 'deactivated_workspace', '')).toBe(true);
expect(isWorkspaceDisabledError(402, 'error.deactivated_workspace', '')).toBe(true);
expect(
isWorkspaceDisabledError(402, '', '{"error":{"code":"deactivated_workspace"}}'),
).toBe(true);
});

it('returns false for statuses without supported disabled signals', () => {
expect(isWorkspaceDisabledError(400, '', 'Your workspace has been disabled')).toBe(false);
expect(isWorkspaceDisabledError(401, '', 'Your workspace has been disabled')).toBe(false);
expect(isWorkspaceDisabledError(500, '', 'Your workspace has been disabled')).toBe(false);
expect(isWorkspaceDisabledError(400, 'workspace_disabled', '')).toBe(false);
expect(isWorkspaceDisabledError(402, 'payment_required', '')).toBe(false);
expect(isWorkspaceDisabledError(402, '', 'Your workspace has been disabled')).toBe(false);
});

it('returns false for 403 with unrelated messages', () => {
Expand Down
Loading