scriptoriumadmin/Roo-Code
1
0
Fork 0
mirror of https://github.com/RooVetGit/Roo-Code.git synced 2026-09-07 08:26:51 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Add exponential backoff for mid-stream retry failures (#8888)

Browse source 
* Add exponential backoff for mid-stream retry failures

- Extend StackItem with retryAttempt counter
- Extract shared backoffAndAnnounce helper for consistent retry UX
- Apply exponential backoff to mid-stream failures when auto-approval enabled
- Add debug throw for testing mid-stream retry path

* Add abort check in retry countdown loop

Allows early exit from exponential backoff if task is cancelled during delay
This commit is contained in:
Daniel 2025-10-28 11:55:13 -05:00 • committed by GitHub
parent a3101aa92b
commit be119bcc5a
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1 changed files with 89 additions and 41 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

130
src/core/task/Task.ts
View file

@ -1749,9 +1749,10 @@ export class Task extends EventEmitter<TaskEvents> implements TaskLike {
interface StackItem {
userContent: Anthropic.Messages.ContentBlockParam[]
includeFileDetails: boolean
retryAttempt?: number
}
const stack: StackItem[] = [{ userContent, includeFileDetails }]
const stack: StackItem[] = [{ userContent, includeFileDetails, retryAttempt: 0 }]
while (stack.length > 0) {
const currentItem = stack.pop()!
@ -2231,10 +2232,21 @@ export class Task extends EventEmitter<TaskEvents> implements TaskLike {
`[Task#${this.taskId}.${this.instanceId}] Stream failed, will retry: ${streamingFailedMessage}`,
)
// Push the same content back onto the stack to retry
// Apply exponential backoff similar to first-chunk errors when auto-resubmit is enabled
const stateForBackoff = await this.providerRef.deref()?.getState()
if (stateForBackoff?.autoApprovalEnabled && stateForBackoff?.alwaysApproveResubmit) {
await this.backoffAndAnnounce(
currentItem.retryAttempt ?? 0,
error,
streamingFailedMessage,
)
}
// Push the same content back onto the stack to retry, incrementing the retry attempt counter
stack.push({
userContent: currentUserContent,
includeFileDetails: false,
retryAttempt: (currentItem.retryAttempt ?? 0) + 1,
})
// Continue to retry the request
@ -2775,45 +2787,8 @@ export class Task extends EventEmitter<TaskEvents> implements TaskLike {
errorMsg = "Unknown error"
}
const baseDelay = requestDelaySeconds || 5
let exponentialDelay = Math.min(
Math.ceil(baseDelay * Math.pow(2, retryAttempt)),
MAX_EXPONENTIAL_BACKOFF_SECONDS,
)
// If the error is a 429, and the error details contain a retry delay, use that delay instead of exponential backoff
if (error.status === 429) {
const geminiRetryDetails = error.errorDetails?.find(
(detail: any) => detail["@type"] === "type.googleapis.com/google.rpc.RetryInfo",
)
if (geminiRetryDetails) {
const match = geminiRetryDetails?.retryDelay?.match(/^(\d+)s$/)
if (match) {
exponentialDelay = Number(match[1]) + 1
}
}
}
// Wait for the greater of the exponential delay or the rate limit delay
const finalDelay = Math.max(exponentialDelay, rateLimitDelay)
// Show countdown timer with exponential backoff
for (let i = finalDelay; i > 0; i--) {
await this.say(
"api_req_retry_delayed",
`${errorMsg}\n\nRetry attempt ${retryAttempt + 1}\nRetrying in ${i} seconds...`,
undefined,
true,
)
await delay(1000)
}
await this.say(
"api_req_retry_delayed",
`${errorMsg}\n\nRetry attempt ${retryAttempt + 1}\nRetrying now...`,
undefined,
false,
)
// Apply shared exponential backoff and countdown UX
await this.backoffAndAnnounce(retryAttempt, error, errorMsg)
// Delegate generator output from the recursive call with
// incremented retry count.
@ -2851,6 +2826,79 @@ export class Task extends EventEmitter<TaskEvents> implements TaskLike {
yield* iterator
}
// Shared exponential backoff for retries (first-chunk and mid-stream)
private async backoffAndAnnounce(retryAttempt: number, error: any, header?: string): Promise<void> {
try {
const state = await this.providerRef.deref()?.getState()
const baseDelay = state?.requestDelaySeconds || 5
let exponentialDelay = Math.min(
Math.ceil(baseDelay * Math.pow(2, retryAttempt)),
MAX_EXPONENTIAL_BACKOFF_SECONDS,
)
// Respect provider rate limit window
let rateLimitDelay = 0
const rateLimit = state?.apiConfiguration?.rateLimitSeconds || 0
if (Task.lastGlobalApiRequestTime && rateLimit > 0) {
const elapsed = performance.now() - Task.lastGlobalApiRequestTime
rateLimitDelay = Math.ceil(Math.min(rateLimit, Math.max(0, rateLimit * 1000 - elapsed) / 1000))
}
// Prefer RetryInfo on 429 if present
if (error?.status === 429) {
const retryInfo = error?.errorDetails?.find(
(d: any) => d["@type"] === "type.googleapis.com/google.rpc.RetryInfo",
)
const match = retryInfo?.retryDelay?.match?.(/^(\d+)s$/)
if (match) {
exponentialDelay = Number(match[1]) + 1
}
}
const finalDelay = Math.max(exponentialDelay, rateLimitDelay)
if (finalDelay <= 0) return
// Build header text; fall back to error message if none provided
let headerText = header
if (!headerText) {
if (error?.error?.metadata?.raw) {
headerText = JSON.stringify(error.error.metadata.raw, null, 2)
} else if (error?.message) {
headerText = error.message
} else {
headerText = "Unknown error"
}
}
headerText = headerText ? `${headerText}\n\n` : ""
// Show countdown timer with exponential backoff
for (let i = finalDelay; i > 0; i--) {
// Check abort flag during countdown to allow early exit
if (this.abort) {
throw new Error(`[Task#${this.taskId}] Aborted during retry countdown`)
}
await this.say(
"api_req_retry_delayed",
`${headerText}Retry attempt ${retryAttempt + 1}\nRetrying in ${i} seconds...`,
undefined,
true,
)
await delay(1000)
}
await this.say(
"api_req_retry_delayed",
`${headerText}Retry attempt ${retryAttempt + 1}\nRetrying now...`,
undefined,
false,
)
} catch (err) {
console.error("Exponential backoff failed:", err)
}
}
// Checkpoints
public async checkpointSave(force: boolean = false, suppressMessage: boolean = false) {