mirror of
https://github.com/RooVetGit/Roo-Code.git
synced 2026-09-05 08:10:14 +00:00
feat(openai-native): background mode + auto-resume and poll fallback
Enable OpenAI Responses background mode with resilient streaming for GPT‑5 Pro and any model flagged via metadata.
Key changes:
- Background mode enablement
• Auto-enable for models with info.backgroundMode === true (e.g., gpt-5-pro-2025-10-06) defined in [packages/types/src/providers/openai.ts](packages/types/src/providers/openai.ts).
• Also respects manual override (openAiNativeBackgroundMode) from ProviderSettings/ApiHandlerOptions.
- Request shape (Responses API)
• background:true, stream:true, store:true set in [OpenAiNativeHandler.buildRequestBody()](src/api/providers/openai-native.ts:224).
- Streaming UX and status events
• New ApiStreamStatusChunk in [src/api/transform/stream.ts](src/api/transform/stream.ts) with statuses: queued, in_progress, completed, failed, canceled, reconnecting, polling.
• Provider emits status chunks in SDK + SSE paths via [OpenAiNativeHandler.processEvent()](src/api/providers/openai-native.ts:1100) and [OpenAiNativeHandler.handleStreamResponse()](src/api/providers/openai-native.ts:651).
• UI spinner shows background lifecycle labels in [webview-ui/src/components/chat/ChatRow.tsx](webview-ui/src/components/chat/ChatRow.tsx) using [webview-ui/src/utils/backgroundStatus.ts](webview-ui/src/utils/backgroundStatus.ts).
- Resilience: auto-resume + poll fallback
• On stream drop for background tasks, attempt SSE resume using response.id and last sequence_number with exponential backoff in [OpenAiNativeHandler.attemptResumeOrPoll()](src/api/providers/openai-native.ts:1215).
• If resume fails, poll GET /v1/responses/{id} every 2s until terminal and synthesize final output/usage.
• Deduplicate resumed events via resumeCutoffSequence in [handleStreamResponse()](src/api/providers/openai-native.ts:737).
- Settings (no new UI switch)
• Added optional provider settings and ApiHandlerOptions: autoResume, resumeMaxRetries, resumeBaseDelayMs, pollIntervalMs, pollMaxMinutes in [packages/types/src/provider-settings.ts](packages/types/src/provider-settings.ts) and [src/shared/api.ts](src/shared/api.ts).
- Cleanup
• Removed VS Code contributes toggle for background mode; behavior now model-driven + programmatic override.
- Tests
• Provider: coverage for background status emission, auto-resume success, resume→poll fallback, non-background negative in [src/api/providers/__tests__/openai-native.spec.ts](src/api/providers/__tests__/openai-native.spec.ts).
• Usage parity unchanged validated in [src/api/providers/__tests__/openai-native-usage.spec.ts](src/api/providers/__tests__/openai-native-usage.spec.ts).
• UI: label mapping tests for background statuses in [webview-ui/src/utils/__tests__/backgroundStatus.spec.ts](webview-ui/src/utils/__tests__/backgroundStatus.spec.ts).
Notes:
- Aligns with TEMP_OPENAI_BACKGROUND_TASK_DOCS.DM: background requires store=true; supports streaming resume via response.id + sequence_number.
- Default behavior unchanged for non-background models; no breaking changes.
This commit is contained in:
parent
d73bdf36d7
commit
3949621cf1
14 changed files with 1309 additions and 11 deletions
212
TEMP_OPENAI_BACKGROUND_TASK_DOCS.DM
Normal file
212
TEMP_OPENAI_BACKGROUND_TASK_DOCS.DM
Normal file
|
|
@ -0,0 +1,212 @@
|
|||
Background mode
|
||||
===============
|
||||
|
||||
Run long running tasks asynchronously in the background.
|
||||
|
||||
Agents like [Codex](https://openai.com/index/introducing-codex/) and [Deep Research](https://openai.com/index/introducing-deep-research/) show that reasoning models can take several minutes to solve complex problems. Background mode enables you to execute long-running tasks on models like o3 and o1-pro reliably, without having to worry about timeouts or other connectivity issues.
|
||||
|
||||
Background mode kicks off these tasks asynchronously, and developers can poll response objects to check status over time. To start response generation in the background, make an API request with `background` set to `true`:
|
||||
|
||||
Generate a response in the background
|
||||
|
||||
```bash
|
||||
curl https://api.openai.com/v1/responses \
|
||||
-H "Content-Type: application/json" \
|
||||
-H "Authorization: Bearer $OPENAI_API_KEY" \
|
||||
-d '{
|
||||
"model": "o3",
|
||||
"input": "Write a very long novel about otters in space.",
|
||||
"background": true
|
||||
}'
|
||||
```
|
||||
|
||||
```javascript
|
||||
import OpenAI from "openai";
|
||||
const client = new OpenAI();
|
||||
|
||||
const resp = await client.responses.create({
|
||||
model: "o3",
|
||||
input: "Write a very long novel about otters in space.",
|
||||
background: true,
|
||||
});
|
||||
|
||||
console.log(resp.status);
|
||||
```
|
||||
|
||||
```python
|
||||
from openai import OpenAI
|
||||
|
||||
client = OpenAI()
|
||||
|
||||
resp = client.responses.create(
|
||||
model="o3",
|
||||
input="Write a very long novel about otters in space.",
|
||||
background=True,
|
||||
)
|
||||
|
||||
print(resp.status)
|
||||
```
|
||||
|
||||
Polling background responses
|
||||
----------------------------
|
||||
|
||||
To check the status of background requests, use the GET endpoint for Responses. Keep polling while the request is in the queued or in\_progress state. When it leaves these states, it has reached a final (terminal) state.
|
||||
|
||||
Retrieve a response executing in the background
|
||||
|
||||
```bash
|
||||
curl https://api.openai.com/v1/responses/resp_123 \
|
||||
-H "Content-Type: application/json" \
|
||||
-H "Authorization: Bearer $OPENAI_API_KEY"
|
||||
```
|
||||
|
||||
```javascript
|
||||
import OpenAI from "openai";
|
||||
const client = new OpenAI();
|
||||
|
||||
let resp = await client.responses.create({
|
||||
model: "o3",
|
||||
input: "Write a very long novel about otters in space.",
|
||||
background: true,
|
||||
});
|
||||
|
||||
while (resp.status === "queued" || resp.status === "in_progress") {
|
||||
console.log("Current status: " + resp.status);
|
||||
await new Promise(resolve => setTimeout(resolve, 2000)); // wait 2 seconds
|
||||
resp = await client.responses.retrieve(resp.id);
|
||||
}
|
||||
|
||||
console.log("Final status: " + resp.status + "\nOutput:\n" + resp.output_text);
|
||||
```
|
||||
|
||||
```python
|
||||
from openai import OpenAI
|
||||
from time import sleep
|
||||
|
||||
client = OpenAI()
|
||||
|
||||
resp = client.responses.create(
|
||||
model="o3",
|
||||
input="Write a very long novel about otters in space.",
|
||||
background=True,
|
||||
)
|
||||
|
||||
while resp.status in {"queued", "in_progress"}:
|
||||
print(f"Current status: {resp.status}")
|
||||
sleep(2)
|
||||
resp = client.responses.retrieve(resp.id)
|
||||
|
||||
print(f"Final status: {resp.status}\nOutput:\n{resp.output_text}")
|
||||
```
|
||||
|
||||
Cancelling a background response
|
||||
--------------------------------
|
||||
|
||||
You can also cancel an in-flight response like this:
|
||||
|
||||
Cancel an ongoing response
|
||||
|
||||
```bash
|
||||
curl -X POST https://api.openai.com/v1/responses/resp_123/cancel \
|
||||
-H "Content-Type: application/json" \
|
||||
-H "Authorization: Bearer $OPENAI_API_KEY"
|
||||
```
|
||||
|
||||
```javascript
|
||||
import OpenAI from "openai";
|
||||
const client = new OpenAI();
|
||||
|
||||
const resp = await client.responses.cancel("resp_123");
|
||||
|
||||
console.log(resp.status);
|
||||
```
|
||||
|
||||
```python
|
||||
from openai import OpenAI
|
||||
client = OpenAI()
|
||||
|
||||
resp = client.responses.cancel("resp_123")
|
||||
|
||||
print(resp.status)
|
||||
```
|
||||
|
||||
Cancelling twice is idempotent - subsequent calls simply return the final `Response` object.
|
||||
|
||||
Streaming a background response
|
||||
-------------------------------
|
||||
|
||||
You can create a background Response and start streaming events from it right away. This may be helpful if you expect the client to drop the stream and want the option of picking it back up later. To do this, create a Response with both `background` and `stream` set to `true`. You will want to keep track of a "cursor" corresponding to the `sequence_number` you receive in each streaming event.
|
||||
|
||||
Currently, the time to first token you receive from a background response is higher than what you receive from a synchronous one. We are working to reduce this latency gap in the coming weeks.
|
||||
|
||||
Generate and stream a background response
|
||||
|
||||
```bash
|
||||
curl https://api.openai.com/v1/responses \
|
||||
-H "Content-Type: application/json" \
|
||||
-H "Authorization: Bearer $OPENAI_API_KEY" \
|
||||
-d '{
|
||||
"model": "o3",
|
||||
"input": "Write a very long novel about otters in space.",
|
||||
"background": true,
|
||||
"stream": true
|
||||
}'
|
||||
|
||||
// To resume:
|
||||
curl "https://api.openai.com/v1/responses/resp_123?stream=true&starting_after=42" \
|
||||
-H "Content-Type: application/json" \
|
||||
-H "Authorization: Bearer $OPENAI_API_KEY"
|
||||
```
|
||||
|
||||
```javascript
|
||||
import OpenAI from "openai";
|
||||
const client = new OpenAI();
|
||||
|
||||
const stream = await client.responses.create({
|
||||
model: "o3",
|
||||
input: "Write a very long novel about otters in space.",
|
||||
background: true,
|
||||
stream: true,
|
||||
});
|
||||
|
||||
let cursor = null;
|
||||
for await (const event of stream) {
|
||||
console.log(event);
|
||||
cursor = event.sequence_number;
|
||||
}
|
||||
|
||||
// If the connection drops, you can resume streaming from the last cursor (SDK support coming soon):
|
||||
// const resumedStream = await client.responses.stream(resp.id, { starting_after: cursor });
|
||||
// for await (const event of resumedStream) { ... }
|
||||
```
|
||||
|
||||
```python
|
||||
from openai import OpenAI
|
||||
|
||||
client = OpenAI()
|
||||
|
||||
# Fire off an async response but also start streaming immediately
|
||||
stream = client.responses.create(
|
||||
model="o3",
|
||||
input="Write a very long novel about otters in space.",
|
||||
background=True,
|
||||
stream=True,
|
||||
)
|
||||
|
||||
cursor = None
|
||||
for event in stream:
|
||||
print(event)
|
||||
cursor = event.sequence_number
|
||||
|
||||
# If your connection drops, the response continues running and you can reconnect:
|
||||
# SDK support for resuming the stream is coming soon.
|
||||
# for event in client.responses.stream(resp.id, starting_after=cursor):
|
||||
# print(event)
|
||||
```
|
||||
|
||||
Limits
|
||||
------
|
||||
|
||||
1. Background sampling requires `store=true`; stateless requests are rejected.
|
||||
2. To cancel a synchronous response, terminate the connection
|
||||
3. You can only start a new stream from a background response if you created it with `stream=true`.
|
||||
|
|
@ -88,6 +88,9 @@ export const modelInfoSchema = z.object({
|
|||
defaultTemperature: z.number().optional(),
|
||||
// When true, force-disable request timeouts for this model (providers will set timeout=0)
|
||||
disableTimeout: z.boolean().optional(),
|
||||
// When true, this model must be invoked using Responses background mode.
|
||||
// Providers should auto-enable background:true, stream:true, and store:true.
|
||||
backgroundMode: z.boolean().optional(),
|
||||
requiredReasoningBudget: z.boolean().optional(),
|
||||
supportsReasoningEffort: z
|
||||
.union([z.boolean(), z.array(z.enum(["disable", "none", "minimal", "low", "medium", "high", "xhigh"]))])
|
||||
|
|
|
|||
|
|
@ -304,6 +304,15 @@ const openAiNativeSchema = apiModelIdProviderModelSchema.extend({
|
|||
// OpenAI Responses API service tier for openai-native provider only.
|
||||
// UI should only expose this when the selected model supports flex/priority.
|
||||
openAiNativeServiceTier: serviceTierSchema.optional(),
|
||||
// Enable OpenAI Responses background mode when using Responses API.
|
||||
// Opt-in; defaults to false when omitted.
|
||||
openAiNativeBackgroundMode: z.boolean().optional(),
|
||||
// Background auto-resume/poll settings (no UI; plumbed via options)
|
||||
openAiNativeBackgroundAutoResume: z.boolean().optional(),
|
||||
openAiNativeBackgroundResumeMaxRetries: z.number().int().min(0).optional(),
|
||||
openAiNativeBackgroundResumeBaseDelayMs: z.number().int().min(0).optional(),
|
||||
openAiNativeBackgroundPollIntervalMs: z.number().int().min(0).optional(),
|
||||
openAiNativeBackgroundPollMaxMinutes: z.number().int().min(1).optional(),
|
||||
})
|
||||
|
||||
const mistralSchema = apiModelIdProviderModelSchema.extend({
|
||||
|
|
|
|||
|
|
@ -60,6 +60,7 @@ export const openAiNativeModels = {
|
|||
"GPT-5 Pro: a slow, reasoning-focused model built to tackle tough problems. Requests can take several minutes to finish. Responses API only; no streaming, so it may appear stuck until the reply is ready.",
|
||||
supportsVerbosity: true,
|
||||
supportsTemperature: false,
|
||||
backgroundMode: true,
|
||||
},
|
||||
"gpt-5.1-codex": {
|
||||
maxTokens: 128000,
|
||||
|
|
|
|||
|
|
@ -389,6 +389,38 @@ describe("OpenAiNativeHandler - normalizeUsage", () => {
|
|||
})
|
||||
})
|
||||
|
||||
it("should produce identical usage chunk when background mode is enabled", () => {
|
||||
const usage = {
|
||||
input_tokens: 120,
|
||||
output_tokens: 60,
|
||||
cache_creation_input_tokens: 10,
|
||||
cache_read_input_tokens: 30,
|
||||
}
|
||||
|
||||
const baselineHandler = new OpenAiNativeHandler({
|
||||
openAiNativeApiKey: "test-key",
|
||||
apiModelId: "gpt-5-pro-2025-10-06",
|
||||
})
|
||||
const backgroundHandler = new OpenAiNativeHandler({
|
||||
openAiNativeApiKey: "test-key",
|
||||
apiModelId: "gpt-5-pro-2025-10-06",
|
||||
openAiNativeBackgroundMode: true,
|
||||
})
|
||||
|
||||
const baselineUsage = (baselineHandler as any).normalizeUsage(usage, baselineHandler.getModel())
|
||||
const backgroundUsage = (backgroundHandler as any).normalizeUsage(usage, backgroundHandler.getModel())
|
||||
|
||||
expect(baselineUsage).toMatchObject({
|
||||
type: "usage",
|
||||
inputTokens: 120,
|
||||
outputTokens: 60,
|
||||
cacheWriteTokens: 10,
|
||||
cacheReadTokens: 30,
|
||||
totalCost: expect.any(Number),
|
||||
})
|
||||
expect(backgroundUsage).toEqual(baselineUsage)
|
||||
})
|
||||
|
||||
describe("cost calculation", () => {
|
||||
it("should pass total input tokens to calculateApiCostOpenAI", () => {
|
||||
const usage = {
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@
|
|||
import { Anthropic } from "@anthropic-ai/sdk"
|
||||
|
||||
import { OpenAiNativeHandler } from "../openai-native"
|
||||
import type { ApiHandlerCreateMessageMetadata } from "../../index"
|
||||
import { ApiHandlerOptions } from "../../../shared/api"
|
||||
|
||||
// Mock OpenAI client - now everything uses Responses API
|
||||
|
|
@ -1402,3 +1403,537 @@ describe("GPT-5 streaming event coverage (additional)", () => {
|
|||
})
|
||||
})
|
||||
})
|
||||
|
||||
describe("OpenAI Native background mode behavior", () => {
|
||||
const systemPrompt = "System prompt"
|
||||
const baseMessages: Anthropic.Messages.MessageParam[] = [{ role: "user", content: "hi" }]
|
||||
const createMinimalIterable = () => ({
|
||||
async *[Symbol.asyncIterator]() {
|
||||
yield {
|
||||
type: "response.done",
|
||||
response: { id: "resp_minimal", usage: { input_tokens: 1, output_tokens: 1 } },
|
||||
}
|
||||
},
|
||||
})
|
||||
const createUsageIterable = () => ({
|
||||
async *[Symbol.asyncIterator]() {
|
||||
yield { type: "response.text.delta", delta: "Hello" }
|
||||
yield {
|
||||
type: "response.done",
|
||||
response: {
|
||||
id: "resp_usage",
|
||||
usage: { input_tokens: 120, output_tokens: 60 },
|
||||
},
|
||||
}
|
||||
},
|
||||
})
|
||||
|
||||
beforeEach(() => {
|
||||
mockResponsesCreate.mockClear()
|
||||
})
|
||||
|
||||
afterEach(() => {
|
||||
if ((global as any).fetch) {
|
||||
delete (global as any).fetch
|
||||
}
|
||||
})
|
||||
|
||||
const metadataStoreFalse: ApiHandlerCreateMessageMetadata = { taskId: "background-test", store: false }
|
||||
|
||||
it("auto-enables background mode for gpt-5-pro when no override is specified", async () => {
|
||||
const handler = new OpenAiNativeHandler({
|
||||
apiModelId: "gpt-5-pro-2025-10-06",
|
||||
openAiNativeApiKey: "test",
|
||||
// openAiNativeBackgroundMode is undefined
|
||||
})
|
||||
|
||||
mockResponsesCreate.mockResolvedValueOnce(createMinimalIterable())
|
||||
|
||||
const chunks: any[] = []
|
||||
for await (const chunk of handler.createMessage(systemPrompt, baseMessages, metadataStoreFalse)) {
|
||||
chunks.push(chunk)
|
||||
}
|
||||
|
||||
expect(chunks).not.toHaveLength(0)
|
||||
const requestBody = mockResponsesCreate.mock.calls[0][0]
|
||||
expect(requestBody.background).toBe(true)
|
||||
expect(requestBody.stream).toBe(true)
|
||||
expect(requestBody.store).toBe(true)
|
||||
})
|
||||
it("sends background:true, stream:true, and forces store:true for gpt-5-pro when background mode is enabled", async () => {
|
||||
const handler = new OpenAiNativeHandler({
|
||||
apiModelId: "gpt-5-pro-2025-10-06",
|
||||
openAiNativeApiKey: "test",
|
||||
openAiNativeBackgroundMode: true,
|
||||
})
|
||||
|
||||
mockResponsesCreate.mockResolvedValueOnce(createMinimalIterable())
|
||||
|
||||
const chunks: any[] = []
|
||||
for await (const chunk of handler.createMessage(systemPrompt, baseMessages, metadataStoreFalse)) {
|
||||
chunks.push(chunk)
|
||||
}
|
||||
|
||||
expect(chunks).not.toHaveLength(0)
|
||||
|
||||
const requestBody = mockResponsesCreate.mock.calls[0][0]
|
||||
expect(requestBody.background).toBe(true)
|
||||
expect(requestBody.stream).toBe(true)
|
||||
expect(requestBody.store).toBe(true)
|
||||
expect(requestBody.instructions).toBe(systemPrompt)
|
||||
expect(requestBody.model).toBe("gpt-5-pro-2025-10-06")
|
||||
expect(Array.isArray(requestBody.input)).toBe(true)
|
||||
expect(requestBody.input.length).toBeGreaterThan(0)
|
||||
|
||||
mockResponsesCreate.mockClear()
|
||||
|
||||
const handlerWithOptionFalse = new OpenAiNativeHandler({
|
||||
apiModelId: "gpt-5-pro-2025-10-06",
|
||||
openAiNativeApiKey: "test",
|
||||
openAiNativeBackgroundMode: false, // metadata still enforces background mode
|
||||
})
|
||||
|
||||
mockResponsesCreate.mockResolvedValueOnce(createMinimalIterable())
|
||||
|
||||
for await (const chunk of handlerWithOptionFalse.createMessage(
|
||||
systemPrompt,
|
||||
baseMessages,
|
||||
metadataStoreFalse,
|
||||
)) {
|
||||
chunks.push(chunk)
|
||||
}
|
||||
|
||||
const requestBodyWithOptionFalse = mockResponsesCreate.mock.calls[0][0]
|
||||
// Still enabled due to model.info.backgroundMode
|
||||
expect(requestBodyWithOptionFalse.background).toBe(true)
|
||||
expect(requestBodyWithOptionFalse.store).toBe(true)
|
||||
expect(requestBodyWithOptionFalse.stream).toBe(true)
|
||||
})
|
||||
|
||||
it("auto-enables background mode for gpt-5-pro when no override is specified", async () => {
|
||||
const handler = new OpenAiNativeHandler({
|
||||
apiModelId: "gpt-5-pro-2025-10-06",
|
||||
openAiNativeApiKey: "test",
|
||||
// no openAiNativeBackgroundMode provided
|
||||
})
|
||||
|
||||
mockResponsesCreate.mockResolvedValueOnce(createMinimalIterable())
|
||||
|
||||
const chunks: any[] = []
|
||||
for await (const chunk of handler.createMessage(systemPrompt, baseMessages, metadataStoreFalse)) {
|
||||
chunks.push(chunk)
|
||||
}
|
||||
|
||||
expect(chunks).not.toHaveLength(0)
|
||||
const requestBody = mockResponsesCreate.mock.calls[0][0]
|
||||
expect(requestBody.background).toBe(true)
|
||||
expect(requestBody.stream).toBe(true)
|
||||
expect(requestBody.store).toBe(true)
|
||||
})
|
||||
it("forces store:true and includes background:true when falling back to SSE", async () => {
|
||||
const handler = new OpenAiNativeHandler({
|
||||
apiModelId: "gpt-5-pro-2025-10-06",
|
||||
openAiNativeApiKey: "test",
|
||||
openAiNativeBackgroundMode: true,
|
||||
})
|
||||
|
||||
mockResponsesCreate.mockResolvedValueOnce({})
|
||||
|
||||
const encoder = new TextEncoder()
|
||||
const sseStream = new ReadableStream({
|
||||
start(controller) {
|
||||
controller.enqueue(
|
||||
encoder.encode(
|
||||
'data: {"type":"response.done","response":{"id":"resp_1","usage":{"input_tokens":1,"output_tokens":1}}}\n\n',
|
||||
),
|
||||
)
|
||||
controller.enqueue(encoder.encode("data: [DONE]\n\n"))
|
||||
controller.close()
|
||||
},
|
||||
})
|
||||
|
||||
const mockFetch = vitest.fn().mockResolvedValue(
|
||||
new Response(sseStream, {
|
||||
status: 200,
|
||||
headers: { "Content-Type": "text/event-stream" },
|
||||
}),
|
||||
)
|
||||
global.fetch = mockFetch as any
|
||||
|
||||
const chunks: any[] = []
|
||||
for await (const chunk of handler.createMessage(systemPrompt, baseMessages, metadataStoreFalse)) {
|
||||
chunks.push(chunk)
|
||||
}
|
||||
|
||||
expect(mockFetch).toHaveBeenCalledTimes(1)
|
||||
const requestInit = mockFetch.mock.calls[0][1] as RequestInit
|
||||
expect(requestInit?.body).toBeDefined()
|
||||
|
||||
const parsedBody = JSON.parse(requestInit?.body as string)
|
||||
expect(parsedBody.background).toBe(true)
|
||||
expect(parsedBody.store).toBe(true)
|
||||
expect(parsedBody.stream).toBe(true)
|
||||
expect(parsedBody.model).toBe("gpt-5-pro-2025-10-06")
|
||||
})
|
||||
|
||||
it("emits identical usage chunk when background mode is enabled", async () => {
|
||||
const collectUsageChunk = async (options: ApiHandlerOptions) => {
|
||||
mockResponsesCreate.mockResolvedValueOnce(createUsageIterable())
|
||||
const handler = new OpenAiNativeHandler(options)
|
||||
const chunks: any[] = []
|
||||
for await (const chunk of handler.createMessage(systemPrompt, baseMessages)) {
|
||||
chunks.push(chunk)
|
||||
}
|
||||
const usageChunk = chunks.find((chunk) => chunk.type === "usage")
|
||||
mockResponsesCreate.mockClear()
|
||||
return usageChunk
|
||||
}
|
||||
|
||||
const baselineUsage = await collectUsageChunk({
|
||||
apiModelId: "gpt-5-pro-2025-10-06",
|
||||
openAiNativeApiKey: "test",
|
||||
})
|
||||
|
||||
expect(baselineUsage).toBeDefined()
|
||||
|
||||
const backgroundUsage = await collectUsageChunk({
|
||||
apiModelId: "gpt-5-pro-2025-10-06",
|
||||
openAiNativeApiKey: "test",
|
||||
openAiNativeBackgroundMode: true,
|
||||
})
|
||||
|
||||
expect(backgroundUsage).toBeDefined()
|
||||
expect(backgroundUsage).toEqual(baselineUsage)
|
||||
})
|
||||
|
||||
it("emits background status chunks for Responses events (SDK path)", async () => {
|
||||
const handler = new OpenAiNativeHandler({
|
||||
apiModelId: "gpt-5-pro-2025-10-06",
|
||||
openAiNativeApiKey: "test",
|
||||
openAiNativeBackgroundMode: true,
|
||||
})
|
||||
|
||||
const createStatusIterable = () => ({
|
||||
async *[Symbol.asyncIterator]() {
|
||||
yield { type: "response.queued", response: { id: "resp_bg" } }
|
||||
yield { type: "response.in_progress" }
|
||||
yield { type: "response.text.delta", delta: "Hello" }
|
||||
yield {
|
||||
type: "response.done",
|
||||
response: { id: "resp_bg", usage: { input_tokens: 1, output_tokens: 1 } },
|
||||
}
|
||||
},
|
||||
})
|
||||
mockResponsesCreate.mockResolvedValueOnce(createStatusIterable())
|
||||
|
||||
const chunks: any[] = []
|
||||
for await (const chunk of handler.createMessage(systemPrompt, baseMessages)) {
|
||||
chunks.push(chunk)
|
||||
}
|
||||
|
||||
const statusChunks = chunks.filter((c) => c.type === "status")
|
||||
expect(statusChunks).toEqual([
|
||||
{ type: "status", mode: "background", status: "queued", responseId: "resp_bg" },
|
||||
{ type: "status", mode: "background", status: "in_progress" },
|
||||
{ type: "status", mode: "background", status: "completed", responseId: "resp_bg" },
|
||||
])
|
||||
})
|
||||
|
||||
it("emits background status chunks for Responses events (SSE fallback)", async () => {
|
||||
// Force fallback by making SDK return non-iterable
|
||||
mockResponsesCreate.mockResolvedValueOnce({})
|
||||
|
||||
const encoder = new TextEncoder()
|
||||
const sseStream = new ReadableStream({
|
||||
start(controller) {
|
||||
controller.enqueue(encoder.encode('data: {"type":"response.queued","response":{"id":"resp_bg2"}}\n\n'))
|
||||
controller.enqueue(encoder.encode('data: {"type":"response.in_progress"}\n\n'))
|
||||
controller.enqueue(encoder.encode('data: {"type":"response.text.delta","delta":"Hi"}\n\n'))
|
||||
controller.enqueue(
|
||||
encoder.encode(
|
||||
'data: {"type":"response.done","response":{"id":"resp_bg2","usage":{"input_tokens":1,"output_tokens":1}}}\n\n',
|
||||
),
|
||||
)
|
||||
controller.enqueue(encoder.encode("data: [DONE]\n\n"))
|
||||
controller.close()
|
||||
},
|
||||
})
|
||||
|
||||
const mockFetch = vitest.fn().mockResolvedValue(
|
||||
new Response(sseStream, {
|
||||
status: 200,
|
||||
headers: { "Content-Type": "text/event-stream" },
|
||||
}),
|
||||
)
|
||||
global.fetch = mockFetch as any
|
||||
|
||||
const handler = new OpenAiNativeHandler({
|
||||
apiModelId: "gpt-5-pro-2025-10-06",
|
||||
openAiNativeApiKey: "test",
|
||||
openAiNativeBackgroundMode: true,
|
||||
})
|
||||
|
||||
const chunks: any[] = []
|
||||
for await (const chunk of handler.createMessage(systemPrompt, baseMessages)) {
|
||||
chunks.push(chunk)
|
||||
}
|
||||
|
||||
const statusChunks = chunks.filter((c) => c.type === "status")
|
||||
expect(statusChunks).toEqual([
|
||||
{ type: "status", mode: "background", status: "queued", responseId: "resp_bg2" },
|
||||
{ type: "status", mode: "background", status: "in_progress" },
|
||||
{ type: "status", mode: "background", status: "completed", responseId: "resp_bg2" },
|
||||
])
|
||||
|
||||
// Clean up fetch
|
||||
delete (global as any).fetch
|
||||
})
|
||||
})
|
||||
|
||||
describe("OpenAI Native streaming metadata tracking", () => {
|
||||
beforeEach(() => {
|
||||
mockResponsesCreate.mockClear()
|
||||
})
|
||||
|
||||
it("tracks sequence_number from streaming events and exposes via getLastSequenceNumber", async () => {
|
||||
const handler = new OpenAiNativeHandler({
|
||||
apiModelId: "gpt-5-pro-2025-10-06",
|
||||
openAiNativeApiKey: "test",
|
||||
})
|
||||
|
||||
const createSequenceIterable = () => ({
|
||||
async *[Symbol.asyncIterator]() {
|
||||
yield { type: "response.text.delta", delta: "A", sequence_number: 1 }
|
||||
yield { type: "response.reasoning.delta", delta: "B", sequence_number: 2 }
|
||||
yield {
|
||||
type: "response.done",
|
||||
sequence_number: 3,
|
||||
response: { id: "resp_123", usage: { input_tokens: 1, output_tokens: 2 } },
|
||||
}
|
||||
},
|
||||
})
|
||||
|
||||
mockResponsesCreate.mockResolvedValueOnce(createSequenceIterable())
|
||||
|
||||
const chunks: any[] = []
|
||||
for await (const chunk of handler.createMessage("System", [{ role: "user", content: "hi" }])) {
|
||||
chunks.push(chunk)
|
||||
}
|
||||
|
||||
expect(chunks).toContainEqual({ type: "text", text: "A" })
|
||||
expect(chunks).toContainEqual({ type: "reasoning", text: "B" })
|
||||
expect(handler.getLastSequenceNumber()).toBe(3)
|
||||
expect(handler.getLastResponseId()).toBe("resp_123")
|
||||
})
|
||||
})
|
||||
|
||||
// Added plumbing test for openAiNativeBackgroundMode
|
||||
describe("OpenAI Native background mode setting (plumbing)", () => {
|
||||
it("should surface openAiNativeBackgroundMode in handler options when provided", () => {
|
||||
const handler = new OpenAiNativeHandler({
|
||||
apiModelId: "gpt-4.1",
|
||||
openAiNativeApiKey: "test-api-key",
|
||||
openAiNativeBackgroundMode: true,
|
||||
} as ApiHandlerOptions)
|
||||
|
||||
// Access protected options via runtime cast to verify pass-through
|
||||
expect((handler as any).options.openAiNativeBackgroundMode).toBe(true)
|
||||
})
|
||||
})
|
||||
|
||||
describe("OpenAI Native background auto-resume and polling", () => {
|
||||
const systemPrompt = "System prompt"
|
||||
const messages: Anthropic.Messages.MessageParam[] = [{ role: "user", content: "hello" }]
|
||||
|
||||
beforeEach(() => {
|
||||
mockResponsesCreate.mockClear()
|
||||
if ((global as any).fetch) {
|
||||
delete (global as any).fetch
|
||||
}
|
||||
})
|
||||
|
||||
it("resumes background stream on drop and emits no duplicate deltas", async () => {
|
||||
const handler = new OpenAiNativeHandler({
|
||||
apiModelId: "gpt-5-pro-2025-10-06",
|
||||
openAiNativeApiKey: "test",
|
||||
openAiNativeBackgroundMode: true,
|
||||
})
|
||||
|
||||
const dropIterable = {
|
||||
async *[Symbol.asyncIterator]() {
|
||||
yield { type: "response.queued", response: { id: "resp_resume" }, sequence_number: 0 }
|
||||
yield { type: "response.in_progress", sequence_number: 1 }
|
||||
yield { type: "response.text.delta", delta: "Hello", sequence_number: 2 }
|
||||
throw new Error("network drop")
|
||||
},
|
||||
}
|
||||
mockResponsesCreate.mockResolvedValueOnce(dropIterable as any)
|
||||
|
||||
const encoder = new TextEncoder()
|
||||
const sseStream = new ReadableStream({
|
||||
start(controller) {
|
||||
controller.enqueue(
|
||||
encoder.encode(
|
||||
'data: {"type":"response.output_item.added","item":{"type":"text","text":"SHOULD_SKIP"},"sequence_number":2}\n\n',
|
||||
),
|
||||
)
|
||||
controller.enqueue(
|
||||
encoder.encode(
|
||||
'data: {"type":"response.output_item.added","item":{"type":"text","text":" world"},"sequence_number":3}\n\n',
|
||||
),
|
||||
)
|
||||
controller.enqueue(
|
||||
encoder.encode(
|
||||
'data: {"type":"response.done","response":{"id":"resp_resume","usage":{"input_tokens":10,"output_tokens":5}},"sequence_number":4}\n\n',
|
||||
),
|
||||
)
|
||||
controller.enqueue(encoder.encode("data: [DONE]\n\n"))
|
||||
controller.close()
|
||||
},
|
||||
})
|
||||
;(global as any).fetch = vitest
|
||||
.fn()
|
||||
.mockResolvedValue(
|
||||
new Response(sseStream, { status: 200, headers: { "Content-Type": "text/event-stream" } }),
|
||||
)
|
||||
|
||||
const stream = handler.createMessage(systemPrompt, messages)
|
||||
const chunks: any[] = []
|
||||
for await (const c of stream) {
|
||||
chunks.push(c)
|
||||
}
|
||||
|
||||
const statusChunks = chunks.filter((c) => c.type === "status")
|
||||
const statusNames = statusChunks.map((s: any) => s.status)
|
||||
const reconnectIdx = statusNames.indexOf("reconnecting")
|
||||
const inProgIdx = statusNames.findIndex((s, i) => s === "in_progress" && i > reconnectIdx)
|
||||
expect(reconnectIdx).toBeGreaterThanOrEqual(0)
|
||||
expect(inProgIdx).toBeGreaterThan(reconnectIdx)
|
||||
|
||||
const fullText = chunks
|
||||
.filter((c) => c.type === "text")
|
||||
.map((c: any) => c.text)
|
||||
.join("")
|
||||
expect(fullText).toBe("Hello world")
|
||||
expect(fullText).not.toContain("SHOULD_SKIP")
|
||||
|
||||
const usageChunks = chunks.filter((c) => c.type === "usage")
|
||||
expect(usageChunks).toHaveLength(1)
|
||||
})
|
||||
|
||||
it("falls back to polling after failed resume and yields final output/usage", async () => {
|
||||
const handler = new OpenAiNativeHandler({
|
||||
apiModelId: "gpt-5-pro-2025-10-06",
|
||||
openAiNativeApiKey: "test",
|
||||
openAiNativeBackgroundMode: true,
|
||||
openAiNativeBackgroundResumeMaxRetries: 1,
|
||||
openAiNativeBackgroundResumeBaseDelayMs: 0,
|
||||
openAiNativeBackgroundPollIntervalMs: 1,
|
||||
openAiNativeBackgroundPollMaxMinutes: 1,
|
||||
} as ApiHandlerOptions)
|
||||
|
||||
const dropIterable = {
|
||||
async *[Symbol.asyncIterator]() {
|
||||
yield { type: "response.queued", response: { id: "resp_poll" }, sequence_number: 0 }
|
||||
yield { type: "response.in_progress", sequence_number: 1 }
|
||||
throw new Error("network drop")
|
||||
},
|
||||
}
|
||||
mockResponsesCreate.mockResolvedValueOnce(dropIterable as any)
|
||||
|
||||
let pollStep = 0
|
||||
;(global as any).fetch = vitest.fn().mockImplementation((url: string) => {
|
||||
if (url.includes("?stream=true")) {
|
||||
return Promise.resolve({
|
||||
ok: false,
|
||||
status: 500,
|
||||
text: async () => "resume failed",
|
||||
} as any)
|
||||
}
|
||||
// polling path
|
||||
const payloads = [
|
||||
{ response: { id: "resp_poll", status: "queued" } },
|
||||
{ response: { id: "resp_poll", status: "in_progress" } },
|
||||
{
|
||||
response: {
|
||||
id: "resp_poll",
|
||||
status: "completed",
|
||||
output: [{ type: "message", content: [{ type: "output_text", text: "Polled result" }] }],
|
||||
usage: { input_tokens: 7, output_tokens: 3 },
|
||||
},
|
||||
},
|
||||
]
|
||||
const payload = payloads[Math.min(pollStep++, payloads.length - 1)]
|
||||
return Promise.resolve(
|
||||
new Response(JSON.stringify(payload), { status: 200, headers: { "Content-Type": "application/json" } }),
|
||||
)
|
||||
})
|
||||
|
||||
const stream = handler.createMessage(systemPrompt, messages)
|
||||
const chunks: any[] = []
|
||||
for await (const c of stream) {
|
||||
chunks.push(c)
|
||||
}
|
||||
|
||||
const statusNames = chunks.filter((c) => c.type === "status").map((s: any) => s.status)
|
||||
const idxReconnect = statusNames.indexOf("reconnecting")
|
||||
const idxPolling = statusNames.indexOf("polling")
|
||||
const idxQueued = statusNames.indexOf("queued")
|
||||
const idxInProgress = statusNames.indexOf("in_progress")
|
||||
const idxCompleted = statusNames.indexOf("completed")
|
||||
expect(idxReconnect).toBeGreaterThanOrEqual(0)
|
||||
expect(idxPolling).toBeGreaterThan(idxReconnect)
|
||||
|
||||
const idxQueuedAfterPolling = statusNames.findIndex((s, i) => s === "queued" && i > idxPolling)
|
||||
const idxInProgressAfterQueued = statusNames.findIndex(
|
||||
(s, i) => s === "in_progress" && i > idxQueuedAfterPolling,
|
||||
)
|
||||
const idxCompletedAfterInProgress = statusNames.findIndex(
|
||||
(s, i) => s === "completed" && i > idxInProgressAfterQueued,
|
||||
)
|
||||
|
||||
expect(idxQueuedAfterPolling).toBeGreaterThan(idxPolling)
|
||||
expect(idxInProgressAfterQueued).toBeGreaterThan(idxQueuedAfterPolling)
|
||||
expect(idxCompletedAfterInProgress).toBeGreaterThan(idxInProgressAfterQueued)
|
||||
|
||||
const finalText = chunks
|
||||
.filter((c) => c.type === "text")
|
||||
.map((c: any) => c.text)
|
||||
.join("")
|
||||
expect(finalText).toBe("Polled result")
|
||||
|
||||
const usageChunks = chunks.filter((c) => c.type === "usage")
|
||||
expect(usageChunks).toHaveLength(1)
|
||||
expect(usageChunks[0]).toMatchObject({ type: "usage", inputTokens: 7, outputTokens: 3 })
|
||||
})
|
||||
|
||||
it("does not attempt resume when not in background mode", async () => {
|
||||
const handler = new OpenAiNativeHandler({
|
||||
apiModelId: "gpt-4.1",
|
||||
openAiNativeApiKey: "test",
|
||||
openAiNativeBackgroundMode: false,
|
||||
})
|
||||
|
||||
const dropIterable = {
|
||||
async *[Symbol.asyncIterator]() {
|
||||
yield { type: "response.text.delta", delta: "Hi", sequence_number: 1 }
|
||||
throw new Error("drop")
|
||||
},
|
||||
}
|
||||
mockResponsesCreate.mockResolvedValueOnce(dropIterable as any)
|
||||
;(global as any).fetch = vitest.fn().mockRejectedValue(new Error("SSE fallback failed"))
|
||||
|
||||
const stream = handler.createMessage(systemPrompt, messages)
|
||||
|
||||
const chunks: any[] = []
|
||||
await expect(async () => {
|
||||
for await (const c of stream) {
|
||||
chunks.push(c)
|
||||
}
|
||||
}).rejects.toThrow()
|
||||
|
||||
const statuses = chunks.filter((c) => c.type === "status").map((s: any) => s.status)
|
||||
expect(statuses).not.toContain("reconnecting")
|
||||
expect(statuses).not.toContain("polling")
|
||||
})
|
||||
})
|
||||
|
|
|
|||
|
|
@ -36,6 +36,12 @@ export class OpenAiNativeHandler extends BaseProvider implements SingleCompletio
|
|||
private lastResponseId: string | undefined
|
||||
// Abort controller for cancelling ongoing requests
|
||||
private abortController?: AbortController
|
||||
// Sequence number for background mode stream resumption
|
||||
private lastSequenceNumber: number | undefined
|
||||
// Track whether current request is in background mode for status chunk annotation
|
||||
private currentRequestIsBackground?: boolean
|
||||
// Cutoff sequence for filtering stale events during resume
|
||||
private resumeCutoffSequence?: number
|
||||
|
||||
// Event types handled by the shared event processor to avoid duplication
|
||||
private readonly coreHandledEventTypes = new Set<string>([
|
||||
|
|
@ -241,6 +247,7 @@ export class OpenAiNativeHandler extends BaseProvider implements SingleCompletio
|
|||
}>
|
||||
tool_choice?: any
|
||||
parallel_tool_calls?: boolean
|
||||
background?: boolean
|
||||
}
|
||||
|
||||
// Validate requested tier against model support; if not supported, omit.
|
||||
|
|
@ -312,6 +319,13 @@ export class OpenAiNativeHandler extends BaseProvider implements SingleCompletio
|
|||
body.text = { verbosity: (verbosity || "medium") as VerbosityLevel }
|
||||
}
|
||||
|
||||
// Enable background mode when either explicitly opted in or required by model metadata
|
||||
if (this.options.openAiNativeBackgroundMode === true || model.info.backgroundMode === true) {
|
||||
body.background = true
|
||||
body.stream = true
|
||||
body.store = true
|
||||
}
|
||||
|
||||
return body
|
||||
}
|
||||
|
||||
|
|
@ -325,6 +339,15 @@ export class OpenAiNativeHandler extends BaseProvider implements SingleCompletio
|
|||
// Create AbortController for cancellation
|
||||
this.abortController = new AbortController()
|
||||
|
||||
// Annotate if this request uses background mode (used for status chunks)
|
||||
this.currentRequestIsBackground = !!requestBody?.background
|
||||
|
||||
const canAttemptResume = () =>
|
||||
this.currentRequestIsBackground &&
|
||||
(this.options.openAiNativeBackgroundAutoResume ?? true) &&
|
||||
!!this.lastResponseId &&
|
||||
typeof this.lastSequenceNumber === "number"
|
||||
|
||||
try {
|
||||
// Use the official SDK
|
||||
const stream = (await (this.client as any).responses.create(requestBody, {
|
||||
|
|
@ -337,21 +360,53 @@ export class OpenAiNativeHandler extends BaseProvider implements SingleCompletio
|
|||
)
|
||||
}
|
||||
|
||||
for await (const event of stream) {
|
||||
// Check if request was aborted
|
||||
if (this.abortController.signal.aborted) {
|
||||
break
|
||||
}
|
||||
try {
|
||||
for await (const event of stream) {
|
||||
// Check if request was aborted
|
||||
if (this.abortController?.signal.aborted) {
|
||||
break
|
||||
}
|
||||
|
||||
for await (const outChunk of this.processEvent(event, model)) {
|
||||
yield outChunk
|
||||
for await (const outChunk of this.processEvent(event, model)) {
|
||||
yield outChunk
|
||||
}
|
||||
}
|
||||
} catch (iterErr) {
|
||||
// Stream dropped mid-flight; attempt resume for background requests
|
||||
if (canAttemptResume()) {
|
||||
for await (const chunk of this.attemptResumeOrPoll(
|
||||
this.lastResponseId!,
|
||||
this.lastSequenceNumber!,
|
||||
model,
|
||||
)) {
|
||||
yield chunk
|
||||
}
|
||||
return
|
||||
}
|
||||
throw iterErr
|
||||
}
|
||||
} catch (sdkErr: any) {
|
||||
// For errors, fallback to manual SSE via fetch
|
||||
yield* this.makeResponsesApiRequest(requestBody, model, metadata, systemPrompt, messages)
|
||||
try {
|
||||
yield* this.makeResponsesApiRequest(requestBody, model, metadata, systemPrompt, messages)
|
||||
} catch (fallbackErr) {
|
||||
// If SSE fallback fails mid-stream and we can resume, try that
|
||||
if (canAttemptResume()) {
|
||||
for await (const chunk of this.attemptResumeOrPoll(
|
||||
this.lastResponseId!,
|
||||
this.lastSequenceNumber!,
|
||||
model,
|
||||
)) {
|
||||
yield chunk
|
||||
}
|
||||
return
|
||||
}
|
||||
throw fallbackErr
|
||||
}
|
||||
} finally {
|
||||
this.abortController = undefined
|
||||
// Always clear background flag at end of request lifecycle
|
||||
this.currentRequestIsBackground = undefined
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -590,6 +645,20 @@ export class OpenAiNativeHandler extends BaseProvider implements SingleCompletio
|
|||
try {
|
||||
const parsed = JSON.parse(data)
|
||||
|
||||
// Skip stale events when resuming a dropped background stream
|
||||
if (
|
||||
typeof parsed?.sequence_number === "number" &&
|
||||
this.resumeCutoffSequence !== undefined &&
|
||||
parsed.sequence_number <= this.resumeCutoffSequence
|
||||
) {
|
||||
continue
|
||||
}
|
||||
|
||||
// Record sequence number for cursor tracking
|
||||
if (typeof parsed?.sequence_number === "number") {
|
||||
this.lastSequenceNumber = parsed.sequence_number
|
||||
}
|
||||
|
||||
// Capture resolved service tier if present
|
||||
if (parsed.response?.service_tier) {
|
||||
this.lastServiceTier = parsed.response.service_tier as ServiceTier
|
||||
|
|
@ -879,14 +948,31 @@ export class OpenAiNativeHandler extends BaseProvider implements SingleCompletio
|
|||
}
|
||||
// Handle queued event
|
||||
else if (parsed.type === "response.queued") {
|
||||
// Response is queued
|
||||
yield {
|
||||
type: "status",
|
||||
mode: this.currentRequestIsBackground ? "background" : undefined,
|
||||
status: "queued",
|
||||
...(parsed.response?.id ? { responseId: parsed.response.id } : {}),
|
||||
}
|
||||
}
|
||||
// Handle in_progress event
|
||||
else if (parsed.type === "response.in_progress") {
|
||||
// Response is being processed
|
||||
yield {
|
||||
type: "status",
|
||||
mode: this.currentRequestIsBackground ? "background" : undefined,
|
||||
status: "in_progress",
|
||||
...(parsed.response?.id ? { responseId: parsed.response.id } : {}),
|
||||
}
|
||||
}
|
||||
// Handle failed event
|
||||
else if (parsed.type === "response.failed") {
|
||||
// Emit failed status for UI lifecycle
|
||||
yield {
|
||||
type: "status",
|
||||
mode: this.currentRequestIsBackground ? "background" : undefined,
|
||||
status: "failed",
|
||||
...(parsed.response?.id ? { responseId: parsed.response.id } : {}),
|
||||
}
|
||||
// Response failed
|
||||
if (parsed.error || parsed.message) {
|
||||
throw new Error(
|
||||
|
|
@ -907,6 +993,16 @@ export class OpenAiNativeHandler extends BaseProvider implements SingleCompletio
|
|||
this.lastResponseOutput = parsed.response.output
|
||||
}
|
||||
|
||||
// Emit completed status for UI lifecycle
|
||||
yield {
|
||||
type: "status",
|
||||
mode: this.currentRequestIsBackground ? "background" : undefined,
|
||||
status: "completed",
|
||||
...(parsed.response?.id ? { responseId: parsed.response.id } : {}),
|
||||
}
|
||||
// Clear background marker on completion
|
||||
this.currentRequestIsBackground = undefined
|
||||
|
||||
// Check if the done event contains the complete output (as a fallback)
|
||||
if (
|
||||
!hasContent &&
|
||||
|
|
@ -1022,6 +1118,196 @@ export class OpenAiNativeHandler extends BaseProvider implements SingleCompletio
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Attempt to resume a dropped background stream; if resume fails, fall back to polling.
|
||||
*/
|
||||
private async *attemptResumeOrPoll(responseId: string, lastSeq: number, model: OpenAiNativeModel): ApiStream {
|
||||
// Emit reconnecting status
|
||||
yield {
|
||||
type: "status",
|
||||
mode: "background",
|
||||
status: "reconnecting",
|
||||
responseId,
|
||||
}
|
||||
|
||||
const apiKey = this.options.openAiNativeApiKey ?? "not-provided"
|
||||
const baseUrl = this.options.openAiNativeBaseUrl || "https://api.openai.com"
|
||||
const resumeMaxRetries = this.options.openAiNativeBackgroundResumeMaxRetries ?? 3
|
||||
const resumeBaseDelayMs = this.options.openAiNativeBackgroundResumeBaseDelayMs ?? 1000
|
||||
|
||||
// Try streaming resume with exponential backoff
|
||||
for (let attempt = 0; attempt < resumeMaxRetries; attempt++) {
|
||||
try {
|
||||
const resumeUrl = `${baseUrl}/v1/responses/${responseId}?stream=true&starting_after=${lastSeq}`
|
||||
const res = await fetch(resumeUrl, {
|
||||
method: "GET",
|
||||
headers: {
|
||||
Authorization: `Bearer ${apiKey}`,
|
||||
Accept: "text/event-stream",
|
||||
},
|
||||
})
|
||||
|
||||
if (!res.ok || !res.body) {
|
||||
throw new Error(`Resume request failed (${res.status})`)
|
||||
}
|
||||
|
||||
this.resumeCutoffSequence = lastSeq
|
||||
|
||||
let emittedInProgress = false
|
||||
try {
|
||||
for await (const chunk of this.handleStreamResponse(res.body, model)) {
|
||||
// After the handshake and first accepted chunk, emit in_progress once
|
||||
if (!emittedInProgress) {
|
||||
emittedInProgress = true
|
||||
yield {
|
||||
type: "status",
|
||||
mode: "background",
|
||||
status: "in_progress",
|
||||
responseId,
|
||||
}
|
||||
}
|
||||
// Avoid double-emitting in_progress if the inner handler surfaces it
|
||||
if (chunk.type === "status" && (chunk as any).status === "in_progress") {
|
||||
continue
|
||||
}
|
||||
yield chunk
|
||||
}
|
||||
// Successful resume
|
||||
this.resumeCutoffSequence = undefined
|
||||
return
|
||||
} catch (e) {
|
||||
// Resume stream failed mid-flight; reset and throw to retry
|
||||
this.resumeCutoffSequence = undefined
|
||||
throw e
|
||||
}
|
||||
} catch {
|
||||
// Wait with backoff before next attempt
|
||||
const delay = resumeBaseDelayMs * Math.pow(2, attempt)
|
||||
if (delay > 0) {
|
||||
await new Promise((r) => setTimeout(r, delay))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Resume failed - begin polling fallback
|
||||
yield {
|
||||
type: "status",
|
||||
mode: "background",
|
||||
status: "polling",
|
||||
responseId,
|
||||
}
|
||||
|
||||
const pollIntervalMs = this.options.openAiNativeBackgroundPollIntervalMs ?? 2000
|
||||
const pollMaxMinutes = this.options.openAiNativeBackgroundPollMaxMinutes ?? 20
|
||||
const deadline = Date.now() + pollMaxMinutes * 60_000
|
||||
|
||||
let lastEmittedStatus: "queued" | "in_progress" | "completed" | "failed" | "canceled" | undefined = undefined
|
||||
|
||||
while (Date.now() <= deadline) {
|
||||
try {
|
||||
const pollRes = await fetch(`${baseUrl}/v1/responses/${responseId}`, {
|
||||
method: "GET",
|
||||
headers: {
|
||||
Authorization: `Bearer ${apiKey}`,
|
||||
},
|
||||
})
|
||||
|
||||
if (!pollRes.ok) {
|
||||
// transient; wait and retry
|
||||
await new Promise((r) => setTimeout(r, pollIntervalMs))
|
||||
continue
|
||||
}
|
||||
|
||||
let raw: any
|
||||
try {
|
||||
raw = await pollRes.json()
|
||||
} catch {
|
||||
await new Promise((r) => setTimeout(r, pollIntervalMs))
|
||||
continue
|
||||
}
|
||||
|
||||
const resp = raw?.response ?? raw
|
||||
const status: string | undefined = resp?.status
|
||||
const respId: string | undefined = resp?.id ?? responseId
|
||||
|
||||
// Capture resolved service tier if present
|
||||
if (resp?.service_tier) {
|
||||
this.lastServiceTier = resp.service_tier as ServiceTier
|
||||
}
|
||||
|
||||
// Emit status transitions
|
||||
if (
|
||||
status &&
|
||||
(status === "queued" ||
|
||||
status === "in_progress" ||
|
||||
status === "completed" ||
|
||||
status === "failed" ||
|
||||
status === "canceled")
|
||||
) {
|
||||
if (status !== lastEmittedStatus) {
|
||||
yield {
|
||||
type: "status",
|
||||
mode: "background",
|
||||
status: status as any,
|
||||
...(respId ? { responseId: respId } : {}),
|
||||
}
|
||||
lastEmittedStatus = status as any
|
||||
}
|
||||
}
|
||||
|
||||
if (status === "completed") {
|
||||
// Synthesize final output
|
||||
const output = resp?.output ?? raw?.output
|
||||
if (Array.isArray(output)) {
|
||||
for (const outputItem of output) {
|
||||
if (outputItem.type === "text" && Array.isArray(outputItem.content)) {
|
||||
for (const content of outputItem.content) {
|
||||
if (content?.type === "text" && typeof content.text === "string") {
|
||||
yield { type: "text", text: content.text }
|
||||
}
|
||||
}
|
||||
} else if (outputItem.type === "message" && Array.isArray(outputItem.content)) {
|
||||
for (const content of outputItem.content) {
|
||||
if (
|
||||
(content?.type === "output_text" || content?.type === "text") &&
|
||||
typeof content.text === "string"
|
||||
) {
|
||||
yield { type: "text", text: content.text }
|
||||
}
|
||||
}
|
||||
} else if (outputItem.type === "reasoning" && Array.isArray(outputItem.summary)) {
|
||||
for (const summary of outputItem.summary) {
|
||||
if (summary?.type === "summary_text" && typeof summary.text === "string") {
|
||||
yield { type: "reasoning", text: summary.text }
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Synthesize usage
|
||||
const usage = resp?.usage ?? raw?.usage
|
||||
const usageData = this.normalizeUsage(usage, model)
|
||||
if (usageData) {
|
||||
yield usageData
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
if (status === "failed" || status === "canceled") {
|
||||
throw new Error(`Response ${status}: ${respId || responseId}`)
|
||||
}
|
||||
} catch {
|
||||
// ignore transient poll errors
|
||||
}
|
||||
|
||||
await new Promise((r) => setTimeout(r, pollIntervalMs))
|
||||
}
|
||||
|
||||
throw new Error(`Background response polling timed out for ${responseId}`)
|
||||
}
|
||||
|
||||
/**
|
||||
* Shared processor for Responses API events.
|
||||
*/
|
||||
|
|
@ -1038,6 +1324,34 @@ export class OpenAiNativeHandler extends BaseProvider implements SingleCompletio
|
|||
if (event?.response?.id) {
|
||||
this.lastResponseId = event.response.id as string
|
||||
}
|
||||
// Record sequence number for cursor tracking
|
||||
if (typeof event?.sequence_number === "number") {
|
||||
this.lastSequenceNumber = event.sequence_number
|
||||
}
|
||||
|
||||
// Map lifecycle events to status chunks
|
||||
const statusMap: Record<string, "queued" | "in_progress" | "completed" | "failed" | "canceled"> = {
|
||||
"response.queued": "queued",
|
||||
"response.in_progress": "in_progress",
|
||||
"response.completed": "completed",
|
||||
"response.done": "completed",
|
||||
"response.failed": "failed",
|
||||
"response.canceled": "canceled",
|
||||
}
|
||||
const mappedStatus = statusMap[event?.type as string]
|
||||
if (mappedStatus) {
|
||||
yield {
|
||||
type: "status",
|
||||
mode: this.currentRequestIsBackground ? "background" : undefined,
|
||||
status: mappedStatus,
|
||||
...(event?.response?.id ? { responseId: event.response.id } : {}),
|
||||
}
|
||||
// Clear background flag for terminal statuses
|
||||
if (mappedStatus === "completed" || mappedStatus === "failed" || mappedStatus === "canceled") {
|
||||
this.currentRequestIsBackground = undefined
|
||||
}
|
||||
// Do not return; allow further handling (e.g., usage on done/completed)
|
||||
}
|
||||
|
||||
// Handle known streaming text deltas
|
||||
if (event?.type === "response.text.delta" || event?.type === "response.output_text.delta") {
|
||||
|
|
@ -1252,6 +1566,23 @@ export class OpenAiNativeHandler extends BaseProvider implements SingleCompletio
|
|||
return this.lastResponseId
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the last sequence number observed from streaming events.
|
||||
* @returns The sequence number, or undefined if not available yet
|
||||
*/
|
||||
getLastSequenceNumber(): number | undefined {
|
||||
return this.lastSequenceNumber
|
||||
}
|
||||
|
||||
/**
|
||||
* Sets the last response ID for conversation continuity.
|
||||
* Typically only used in tests or special flows.
|
||||
* @param responseId The response ID to store
|
||||
*/
|
||||
setResponseId(responseId: string): void {
|
||||
this.lastResponseId = responseId
|
||||
}
|
||||
|
||||
async completePrompt(prompt: string): Promise<string> {
|
||||
// Create AbortController for cancellation
|
||||
this.abortController = new AbortController()
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ export type ApiStreamChunk =
|
|||
| ApiStreamToolCallDeltaChunk
|
||||
| ApiStreamToolCallEndChunk
|
||||
| ApiStreamToolCallPartialChunk
|
||||
| ApiStreamStatusChunk
|
||||
| ApiStreamError
|
||||
|
||||
export interface ApiStreamError {
|
||||
|
|
@ -85,3 +86,10 @@ export interface GroundingSource {
|
|||
url: string
|
||||
snippet?: string
|
||||
}
|
||||
|
||||
export interface ApiStreamStatusChunk {
|
||||
type: "status"
|
||||
mode?: "background"
|
||||
status: "queued" | "in_progress" | "completed" | "failed" | "canceled" | "reconnecting" | "polling"
|
||||
responseId?: string
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2612,6 +2612,24 @@ export class Task extends EventEmitter<TaskEvents> implements TaskLike {
|
|||
presentAssistantMessage(this)
|
||||
break
|
||||
}
|
||||
|
||||
case "status": {
|
||||
try {
|
||||
const apiReqMsg = this.clineMessages[lastApiReqIndex]
|
||||
if (apiReqMsg && apiReqMsg.type === "say" && apiReqMsg.say === "api_req_started") {
|
||||
;(apiReqMsg as any).metadata = (apiReqMsg as any).metadata || {}
|
||||
if (chunk.mode === "background") {
|
||||
;(apiReqMsg as any).metadata.background = true
|
||||
}
|
||||
;(apiReqMsg as any).metadata.backgroundStatus = chunk.status
|
||||
if (chunk.responseId) {
|
||||
;(apiReqMsg as any).metadata.responseId = chunk.responseId
|
||||
}
|
||||
await this.updateClineMessage(apiReqMsg)
|
||||
}
|
||||
} catch {}
|
||||
break
|
||||
}
|
||||
case "text": {
|
||||
assistantMessage += chunk.text
|
||||
|
||||
|
|
|
|||
|
|
@ -24,6 +24,20 @@ export type ApiHandlerOptions = Omit<ProviderSettings, "apiProvider"> & {
|
|||
* When undefined, Ollama will use the model's default num_ctx from the Modelfile.
|
||||
*/
|
||||
ollamaNumCtx?: number
|
||||
/**
|
||||
* Opt-in for OpenAI Responses background mode when using apiProvider=openai-native.
|
||||
* Defaults to false when omitted.
|
||||
*/
|
||||
openAiNativeBackgroundMode?: boolean
|
||||
/**
|
||||
* Auto-resume/poll configuration for OpenAI Responses background mode.
|
||||
* These are plumbed-only (no UI). Defaults are resolved in the handler.
|
||||
*/
|
||||
openAiNativeBackgroundAutoResume?: boolean
|
||||
openAiNativeBackgroundResumeMaxRetries?: number
|
||||
openAiNativeBackgroundResumeBaseDelayMs?: number
|
||||
openAiNativeBackgroundPollIntervalMs?: number
|
||||
openAiNativeBackgroundPollMaxMinutes?: number
|
||||
}
|
||||
|
||||
// RouterName
|
||||
|
|
|
|||
|
|
@ -15,6 +15,9 @@ import { useExtensionState } from "@src/context/ExtensionStateContext"
|
|||
import { findMatchingResourceOrTemplate } from "@src/utils/mcp"
|
||||
import { vscode } from "@src/utils/vscode"
|
||||
import { formatPathTooltip } from "@src/utils/formatPathTooltip"
|
||||
import { removeLeadingNonAlphanumeric } from "@src/utils/removeLeadingNonAlphanumeric"
|
||||
import { getLanguageFromPath } from "@src/utils/getLanguageFromPath"
|
||||
import { labelForBackgroundStatus } from "@src/utils/backgroundStatus"
|
||||
|
||||
import { ToolUseBlock, ToolUseBlockHeader } from "../common/ToolUseBlock"
|
||||
import UpdateTodoListToolBlock from "./UpdateTodoListToolBlock"
|
||||
|
|
@ -313,6 +316,21 @@ export const ChatRowContent = ({
|
|||
/>
|
||||
</div>
|
||||
)
|
||||
// Background mode UI label/icon handling
|
||||
const meta: any = message.metadata
|
||||
const isBackground = meta?.background === true
|
||||
const bgStatus = meta?.backgroundStatus as
|
||||
| "queued"
|
||||
| "in_progress"
|
||||
| "reconnecting"
|
||||
| "polling"
|
||||
| "completed"
|
||||
| "failed"
|
||||
| "canceled"
|
||||
| undefined
|
||||
const bgDone =
|
||||
isBackground && (bgStatus === "completed" || bgStatus === "failed" || bgStatus === "canceled")
|
||||
const label = isBackground ? labelForBackgroundStatus(bgStatus) : undefined
|
||||
return [
|
||||
apiReqCancelReason !== null && apiReqCancelReason !== undefined ? (
|
||||
apiReqCancelReason === "user_cancelled" ? (
|
||||
|
|
@ -320,6 +338,16 @@ export const ChatRowContent = ({
|
|||
) : (
|
||||
getIconSpan("error", errorColor)
|
||||
)
|
||||
) : bgDone ? (
|
||||
bgStatus === "completed" ? (
|
||||
isExpanded ? (
|
||||
<ChevronDown className="w-4 shrink-0" />
|
||||
) : (
|
||||
<ChevronRight className="w-4 shrink-0" />
|
||||
)
|
||||
) : (
|
||||
getIconSpan("error", bgStatus === "canceled" ? cancelledColor : errorColor)
|
||||
)
|
||||
) : cost !== null && cost !== undefined ? (
|
||||
getIconSpan("arrow-swap", normalColor)
|
||||
) : apiRequestFailedMessage ? (
|
||||
|
|
@ -337,6 +365,8 @@ export const ChatRowContent = ({
|
|||
{t("chat:apiRequest.streamingFailed")}
|
||||
</span>
|
||||
)
|
||||
) : label ? (
|
||||
<span style={{ color: normalColor }}>{label}</span>
|
||||
) : cost !== null && cost !== undefined ? (
|
||||
<span style={{ color: normalColor }}>{t("chat:apiRequest.title")}</span>
|
||||
) : apiRequestFailedMessage ? (
|
||||
|
|
@ -1066,8 +1096,14 @@ export const ChatRowContent = ({
|
|||
)
|
||||
case "api_req_started":
|
||||
// Determine if the API request is in progress
|
||||
const bgMeta: any = message.metadata
|
||||
const bgStatus = bgMeta?.background === true ? bgMeta?.backgroundStatus : undefined
|
||||
const bgDone = bgStatus === "completed" || bgStatus === "failed" || bgStatus === "canceled"
|
||||
const isApiRequestInProgress =
|
||||
apiReqCancelReason === undefined && apiRequestFailedMessage === undefined && cost === undefined
|
||||
apiReqCancelReason === undefined &&
|
||||
apiRequestFailedMessage === undefined &&
|
||||
cost === undefined &&
|
||||
!bgDone
|
||||
|
||||
return (
|
||||
<>
|
||||
|
|
|
|||
35
webview-ui/src/utils/__tests__/backgroundStatus.spec.ts
Normal file
35
webview-ui/src/utils/__tests__/backgroundStatus.spec.ts
Normal file
|
|
@ -0,0 +1,35 @@
|
|||
import { labelForBackgroundStatus } from "@src/utils/backgroundStatus"
|
||||
|
||||
describe("labelForBackgroundStatus()", () => {
|
||||
it("maps queued", () => {
|
||||
expect(labelForBackgroundStatus("queued")).toBe("API Request: background mode (queued)…")
|
||||
})
|
||||
|
||||
it("maps in_progress", () => {
|
||||
expect(labelForBackgroundStatus("in_progress")).toBe("API Request: background mode (in progress)…")
|
||||
})
|
||||
|
||||
it("maps reconnecting", () => {
|
||||
expect(labelForBackgroundStatus("reconnecting")).toBe("API Request: background mode (reconnecting…)")
|
||||
})
|
||||
|
||||
it("maps polling", () => {
|
||||
expect(labelForBackgroundStatus("polling")).toBe("API Request: background mode (polling…)")
|
||||
})
|
||||
|
||||
it("maps completed", () => {
|
||||
expect(labelForBackgroundStatus("completed")).toBe("API Request: background mode (completed)")
|
||||
})
|
||||
|
||||
it("maps failed", () => {
|
||||
expect(labelForBackgroundStatus("failed")).toBe("API Request: background mode (failed)")
|
||||
})
|
||||
|
||||
it("maps canceled", () => {
|
||||
expect(labelForBackgroundStatus("canceled")).toBe("API Request: background mode (canceled)")
|
||||
})
|
||||
|
||||
it("maps undefined to generic label", () => {
|
||||
expect(labelForBackgroundStatus(undefined)).toBe("API Request: background mode")
|
||||
})
|
||||
})
|
||||
35
webview-ui/src/utils/__tests__/backgroundStatus.test.ts
Normal file
35
webview-ui/src/utils/__tests__/backgroundStatus.test.ts
Normal file
|
|
@ -0,0 +1,35 @@
|
|||
import { labelForBackgroundStatus } from "@src/utils/backgroundStatus"
|
||||
|
||||
describe("labelForBackgroundStatus()", () => {
|
||||
it("maps queued", () => {
|
||||
expect(labelForBackgroundStatus("queued")).toBe("API Request: background mode (queued)…")
|
||||
})
|
||||
|
||||
it("maps in_progress", () => {
|
||||
expect(labelForBackgroundStatus("in_progress")).toBe("API Request: background mode (in progress)…")
|
||||
})
|
||||
|
||||
it("maps reconnecting", () => {
|
||||
expect(labelForBackgroundStatus("reconnecting")).toBe("API Request: background mode (reconnecting…)")
|
||||
})
|
||||
|
||||
it("maps polling", () => {
|
||||
expect(labelForBackgroundStatus("polling")).toBe("API Request: background mode (polling…)")
|
||||
})
|
||||
|
||||
it("maps completed", () => {
|
||||
expect(labelForBackgroundStatus("completed")).toBe("API Request: background mode (completed)")
|
||||
})
|
||||
|
||||
it("maps failed", () => {
|
||||
expect(labelForBackgroundStatus("failed")).toBe("API Request: background mode (failed)")
|
||||
})
|
||||
|
||||
it("maps canceled", () => {
|
||||
expect(labelForBackgroundStatus("canceled")).toBe("API Request: background mode (canceled)")
|
||||
})
|
||||
|
||||
it("maps undefined to generic label", () => {
|
||||
expect(labelForBackgroundStatus(undefined)).toBe("API Request: background mode")
|
||||
})
|
||||
})
|
||||
29
webview-ui/src/utils/backgroundStatus.ts
Normal file
29
webview-ui/src/utils/backgroundStatus.ts
Normal file
|
|
@ -0,0 +1,29 @@
|
|||
export type BackgroundStatus =
|
||||
| "queued"
|
||||
| "in_progress"
|
||||
| "completed"
|
||||
| "failed"
|
||||
| "canceled"
|
||||
| "reconnecting"
|
||||
| "polling"
|
||||
|
||||
export function labelForBackgroundStatus(s?: BackgroundStatus): string {
|
||||
switch (s) {
|
||||
case "queued":
|
||||
return "API Request: background mode (queued)…"
|
||||
case "in_progress":
|
||||
return "API Request: background mode (in progress)…"
|
||||
case "reconnecting":
|
||||
return "API Request: background mode (reconnecting…)"
|
||||
case "polling":
|
||||
return "API Request: background mode (polling…)"
|
||||
case "completed":
|
||||
return "API Request: background mode (completed)"
|
||||
case "failed":
|
||||
return "API Request: background mode (failed)"
|
||||
case "canceled":
|
||||
return "API Request: background mode (canceled)"
|
||||
default:
|
||||
return "API Request: background mode"
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Reference in a new issue