Skip to content

feat(retry): rotate credentials on credential-scoped failures #8

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
yelixir-dev merged 2 commits into from
Oct 3, 2026
Merged

feat(retry): rotate credentials on credential-scoped failures #8

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
187107e
fix(routing): treat 400 insufficient-credits responses as billing fai…
A8Cl233395 Oct 2, 2026
5f5141a
feat(retry): rotate credentials on credential-scoped failures
A8Cl233395 Oct 2, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
63 changes: 47 additions & 16 deletions src/commandcode.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import {
CommandCodeCredentialRouter,
type CommandCodeCredentialDiagnostic,
isInsufficientCreditsMessage,
NoAvailableCommandCodeCredentialError,
type SelectCredentialOptions,
} from "./credential-router.js";
Expand Down Expand Up @@ -188,22 +189,43 @@ function errorStatusCodeFromUnknown(error: unknown): number | undefined {
return error instanceof CommandCodeHttpError ? error.status : undefined;
}

function shouldRetry(statusCode: number | undefined): boolean {
export function shouldRetryStatus(statusCode: number | undefined, errorMessage?: string): boolean {
return (
statusCode === undefined ||
statusCode === 401 ||
statusCode === 402 ||
statusCode === 403 ||
statusCode === 429 ||
statusCode >= 500
statusCode >= 500 ||
// A bare 400 is client-scoped and must fail fast, but the
// insufficient-credits variant blames this credential only: another key can
// still serve the request, so rotate instead of failing.
(statusCode === 400 && isInsufficientCreditsMessage(errorMessage))
);
}

function isClientVisibleEvent(event: CommandCodeEvent): boolean {
return ["text-delta", "reasoning-delta", "tool-call", "finish"].includes(event.type);
}

export function isFatalCredFailure(statusCode: number | undefined): boolean {
return statusCode === 401 || statusCode === 402 || statusCode === 403;
export function isFatalCredFailure(statusCode: number | undefined, errorMessage?: string): boolean {
return (
statusCode === 401 ||
statusCode === 402 ||
statusCode === 403 ||
// A bare 400 is provider-scoped (bad model, invalid body), but the same
// status with an insufficient-credits body blames this credential only.
(statusCode === 400 && isInsufficientCreditsMessage(errorMessage))
);
}

/** Extracts the upstream error text from a parsed HTTP or SSE body for failure classification. */
export function upstreamErrorMessage(body: unknown): string | undefined {
if (typeof body === "string") return body;
if (!isRecord(body)) return undefined;
const error = isRecord(body.error) ? body.error : undefined;
const message = error?.message ?? body.message;
return typeof message === "string" && message.length > 0 ? message : undefined;
}

export function retryBackoff(attempt: number, baseMs: number): Promise<void> {
Expand Down Expand Up @@ -413,10 +435,12 @@ export class CommandCodeClient implements CommandCodeUpstream {
this.router.recordSuccess(credential.id);
finalized = true;
};
const finalizeFailure = (statusCode?: number) => {
const finalizeFailure = (statusCode?: number, errorMessage?: string) => {
if (finalized) return;
if (statusCode === undefined) this.router.recordFailure(credential.id);
else this.router.recordFailure(credential.id, { statusCode });
this.router.recordFailure(credential.id, {
...(statusCode !== undefined ? { statusCode } : {}),
...(errorMessage !== undefined ? { errorMessage } : {}),
});
finalized = true;
};
const finalizeRelease = () => {
Expand All @@ -434,13 +458,16 @@ export class CommandCodeClient implements CommandCodeUpstream {
await responseBody(response),
);
lastError = error;
const fatal = isFatalCredFailure(response.status);
const upstreamMessage = upstreamErrorMessage(error.body);
const fatal = isFatalCredFailure(response.status, upstreamMessage);
const willRetry =
attempt < maxAttempts - 1 && shouldRetry(response.status) && !effectiveSignal.aborted;
attempt < maxAttempts - 1 &&
shouldRetryStatus(response.status, upstreamMessage) &&
!effectiveSignal.aborted;
// Only credential-scoped failures may start a cooldown. Provider-scoped
// statuses like 429/5xx hit every credential at once, so cooling them
// down would bench the whole pool over a single upstream incident.
if (fatal) finalizeFailure(response.status);
if (fatal) finalizeFailure(response.status, upstreamMessage);
else finalizeRelease();
if (willRetry) {
if (fatal) fatalIds.add(credential.id);
Expand Down Expand Up @@ -476,13 +503,15 @@ export class CommandCodeClient implements CommandCodeUpstream {
"CommandCode stream error",
event,
);
const fatal = statusCode !== undefined && isFatalCredFailure(statusCode);
const upstreamMessage = upstreamErrorMessage(event);
const fatal =
statusCode !== undefined && isFatalCredFailure(statusCode, upstreamMessage);
const willRetry =
!emittedVisibleEvent &&
attempt < maxAttempts - 1 &&
shouldRetry(statusCode) &&
shouldRetryStatus(statusCode, upstreamMessage) &&
!effectiveSignal.aborted;
if (fatal) finalizeFailure(statusCode);
if (fatal) finalizeFailure(statusCode, upstreamMessage);
else finalizeRelease();
if (willRetry) {
if (fatal) fatalIds.add(credential.id);
Expand All @@ -501,13 +530,15 @@ export class CommandCodeClient implements CommandCodeUpstream {
} catch (error) {
const statusCode = errorStatusCodeFromUnknown(error);
lastError = error;
const fatal = statusCode !== undefined && isFatalCredFailure(statusCode);
const upstreamMessage =
error instanceof CommandCodeHttpError ? upstreamErrorMessage(error.body) : undefined;
const fatal = statusCode !== undefined && isFatalCredFailure(statusCode, upstreamMessage);
const willRetry =
signal?.aborted !== true &&
attempt < maxAttempts - 1 &&
shouldRetry(statusCode) &&
shouldRetryStatus(statusCode, upstreamMessage) &&
!effectiveSignal.aborted;
if (fatal) finalizeFailure(statusCode);
if (fatal) finalizeFailure(statusCode, upstreamMessage);
else finalizeRelease();
if (willRetry) {
if (fatal) fatalIds.add(credential.id);
Expand Down
29 changes: 26 additions & 3 deletions src/credential-router.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ export interface CommandCodeCredentialState {
credential: CommandCodeCredential;
billing?: CommandCodeBillingSnapshot;
billingError: string | undefined;
disabledReason: "auth" | "billing" | "cooldown" | "expired" | undefined;
disabledReason: "auth" | "billing" | "cooldown" | "expired" | "insufficient_credits" | undefined;
disabledUntil: number;
inFlight: number;
lastSelectedAt: number;
Expand Down Expand Up @@ -102,6 +102,15 @@ export interface SelectCredentialOptions {

export interface RecordFailureOptions {
statusCode?: number;
/** Upstream error text; lets a generic 400 carry credential-scoped meaning (insufficient credits). */
errorMessage?: string;
}

const INSUFFICIENT_CREDITS_PATTERN = /insufficient (?:credits|balance)/i;

/** CommandCode reports an empty balance as a 400 whose body says "insufficient credits". */
export function isInsufficientCreditsMessage(message: string | undefined): boolean {
return message !== undefined && INSUFFICIENT_CREDITS_PATTERN.test(message);
}

export class NoAvailableCommandCodeCredentialError extends Error {
Expand Down Expand Up @@ -500,6 +509,13 @@ export class CommandCodeCredentialRouter {
} else if (statusCode === 402) {
state.disabledUntil = this.now() + Math.max(this.cooldownMs, this.billingRefreshMs);
state.disabledReason = "billing";
} else if (statusCode === 400 && isInsufficientCreditsMessage(options.errorMessage)) {
// Upstream pre-charges the estimated request cost, so a positive balance
// can still be too small to serve. Billing snapshots cannot see that
// threshold, so this cooldown must survive billing refreshes and expire
// on its own before the key is probed again.
state.disabledUntil = this.now() + Math.max(this.cooldownMs, this.billingRefreshMs);
state.disabledReason = "insufficient_credits";
} else if (
statusCode === undefined ||
statusCode === 429 ||
Expand Down Expand Up @@ -545,10 +561,17 @@ export class CommandCodeCredentialRouter {
if (expired) {
state.disabledUntil = Number.MAX_SAFE_INTEGER;
state.disabledReason = "expired";
} else if (remaining > 0 && state.disabledReason !== "auth") {
} else if (
remaining > 0 &&
state.disabledReason !== "auth" &&
state.disabledReason !== "insufficient_credits"
) {
state.disabledUntil = 0;
state.disabledReason = undefined;
} else if (state.disabledUntil !== Number.MAX_SAFE_INTEGER) {
} else if (
state.disabledUntil !== Number.MAX_SAFE_INTEGER &&
state.disabledReason !== "insufficient_credits"
) {
state.disabledUntil = this.now() + this.billingRefreshMs;
state.disabledReason = "billing";
}
Expand Down
38 changes: 17 additions & 21 deletions src/provider.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,8 @@ import {
isFatalCredFailure,
responseBody,
retryBackoff,
shouldRetryStatus,
upstreamErrorMessage,
} from "./commandcode.js";
import {
NoAvailableCommandCodeCredentialError,
Expand Down Expand Up @@ -48,17 +50,6 @@ export function buildProviderChatRequestBody(
return body;
}

function shouldRetryStatus(statusCode: number | undefined): boolean {
return (
statusCode === undefined ||
statusCode === 401 ||
statusCode === 402 ||
statusCode === 403 ||
statusCode === 429 ||
statusCode >= 500
);
}

export async function probeProviderAccess(
config: BridgeConfig,
options: {
Expand Down Expand Up @@ -148,10 +139,12 @@ export class CommandCodeProviderClient {
this.router.recordSuccess(credential.id);
finalized = true;
};
const finalizeFailure = (statusCode?: number) => {
const finalizeFailure = (statusCode?: number, errorMessage?: string) => {
if (finalized) return;
if (statusCode === undefined) this.router.recordFailure(credential.id);
else this.router.recordFailure(credential.id, { statusCode });
this.router.recordFailure(credential.id, {
...(statusCode !== undefined ? { statusCode } : {}),
...(errorMessage !== undefined ? { errorMessage } : {}),
});
finalized = true;
};
const finalizeRelease = () => {
Expand All @@ -168,15 +161,16 @@ export class CommandCodeProviderClient {
response.statusText,
await responseBody(response),
);
const fatal = isFatalCredFailure(response.status);
const upstreamMessage = upstreamErrorMessage(error.body);
const fatal = isFatalCredFailure(response.status, upstreamMessage);
// Only credential-scoped failures may start a cooldown; provider-scoped
// statuses like 429/5xx hit every credential at once.
if (fatal) finalizeFailure(response.status);
if (fatal) finalizeFailure(response.status, upstreamMessage);
else finalizeRelease();
lastError = error;
if (
attempt < maxAttempts - 1 &&
shouldRetryStatus(response.status) &&
shouldRetryStatus(response.status, upstreamMessage) &&
!effectiveSignal.aborted
) {
if (fatal) fatalIds.add(credential.id);
Expand All @@ -191,16 +185,18 @@ export class CommandCodeProviderClient {
} catch (error) {
if (error instanceof CommandCodeHttpError && finalized) throw error;
const statusCode = error instanceof CommandCodeHttpError ? error.status : undefined;
if (statusCode !== undefined && isFatalCredFailure(statusCode)) finalizeFailure(statusCode);
const upstreamMessage =
error instanceof CommandCodeHttpError ? upstreamErrorMessage(error.body) : undefined;
const fatal = statusCode !== undefined && isFatalCredFailure(statusCode, upstreamMessage);
if (fatal) finalizeFailure(statusCode, upstreamMessage);
else finalizeRelease();
lastError = error;
if (
attempt < maxAttempts - 1 &&
shouldRetryStatus(statusCode) &&
shouldRetryStatus(statusCode, upstreamMessage) &&
!effectiveSignal.aborted
) {
if (statusCode !== undefined && isFatalCredFailure(statusCode))
fatalIds.add(credential.id);
if (fatal) fatalIds.add(credential.id);
else if (statusCode !== undefined) retryableFailed.add(credential.id);
await retryBackoff(attempt, this.config.commandCodeRetryBackoffMs ?? 250);
continue;
Expand Down
Loading
Loading