diff --git a/src/api/providers/openai.ts b/src/api/providers/openai.ts index df502c0442..a8308adf53 100644 --- a/src/api/providers/openai.ts +++ b/src/api/providers/openai.ts @@ -19,7 +19,13 @@ import { convertToR1Format } from "../transform/r1-format" import { convertToSimpleMessages } from "../transform/simple-format" import { ApiStream, ApiStreamUsageChunk } from "../transform/stream" import { getModelParams } from "../transform/model-params" -import { addArkCaching, extractArkResponseId, getArkCachedTokens } from "../transform/caching/ark" +import { + addArkCaching, + extractArkResponseId, + getArkCachedTokens, + ArkChatCompletionCreateParamsStreaming, + ArkChatCompletionCreateParamsNonStreaming, +} from "../transform/caching/ark" import { DEFAULT_HEADERS } from "./constants" import { BaseProvider } from "./base-provider" @@ -151,7 +157,7 @@ export class OpenAiHandler extends BaseProvider implements SingleCompletionHandl const isGrokXAI = this._isGrokXAI(this.options.openAiBaseUrl) - const requestOptions: OpenAI.Chat.Completions.ChatCompletionCreateParamsStreaming = { + const requestOptions: ArkChatCompletionCreateParamsStreaming = { model: modelId, temperature: this.options.modelTemperature ?? (deepseekReasoner ? DEEP_SEEK_DEFAULT_TEMPERATURE : 0), messages: convertedMessages, @@ -232,7 +238,7 @@ export class OpenAiHandler extends BaseProvider implements SingleCompletionHandl content: systemPrompt, } - const requestOptions: OpenAI.Chat.Completions.ChatCompletionCreateParamsNonStreaming = { + const requestOptions: ArkChatCompletionCreateParamsNonStreaming = { model: modelId, messages: deepseekReasoner ? convertToR1Format([{ role: "user", content: systemPrompt }, ...messages]) @@ -304,7 +310,7 @@ export class OpenAiHandler extends BaseProvider implements SingleCompletionHandl const model = this.getModel() const modelInfo = model.info - const requestOptions: OpenAI.Chat.Completions.ChatCompletionCreateParamsNonStreaming = { + const requestOptions: ArkChatCompletionCreateParamsNonStreaming = { model: model.id, messages: [{ role: "user", content: prompt }], } @@ -340,7 +346,7 @@ export class OpenAiHandler extends BaseProvider implements SingleCompletionHandl if (this.options.openAiStreamingEnabled ?? true) { const isGrokXAI = this._isGrokXAI(this.options.openAiBaseUrl) - const requestOptions: OpenAI.Chat.Completions.ChatCompletionCreateParamsStreaming = { + const requestOptions: ArkChatCompletionCreateParamsStreaming = { model: modelId, messages: [ { @@ -375,7 +381,7 @@ export class OpenAiHandler extends BaseProvider implements SingleCompletionHandl yield* this.handleStreamResponse(stream, ark) } else { - const requestOptions: OpenAI.Chat.Completions.ChatCompletionCreateParamsNonStreaming = { + const requestOptions: ArkChatCompletionCreateParamsNonStreaming = { model: modelId, messages: [ { @@ -482,9 +488,7 @@ export class OpenAiHandler extends BaseProvider implements SingleCompletionHandl * O3 family models handle max_tokens separately in handleO3FamilyMessage */ private addMaxTokensIfNeeded( - requestOptions: - | OpenAI.Chat.Completions.ChatCompletionCreateParamsStreaming - | OpenAI.Chat.Completions.ChatCompletionCreateParamsNonStreaming, + requestOptions: ArkChatCompletionCreateParamsStreaming | ArkChatCompletionCreateParamsNonStreaming, modelInfo: ModelInfo, ): void { // Only add max_completion_tokens if includeMaxTokens is true diff --git a/src/api/transform/caching/__tests__/ark.spec.ts b/src/api/transform/caching/__tests__/ark.spec.ts index 4a50664463..8db178c189 100644 --- a/src/api/transform/caching/__tests__/ark.spec.ts +++ b/src/api/transform/caching/__tests__/ark.spec.ts @@ -1,12 +1,19 @@ // npx vitest run api/transform/caching/__tests__/ark.spec.ts -import { addArkCaching, extractArkResponseId, getArkCachedTokens, hasArkCachedTokens } from "../ark" +import { + addArkCaching, + extractArkResponseId, + getArkCachedTokens, + hasArkCachedTokens, + ArkChatCompletionCreateParamsStreaming, + ArkChatCompletionCreateParamsNonStreaming, +} from "../ark" import OpenAI from "openai" describe("Ark Context Caching", () => { describe("addArkCaching", () => { it("should add basic caching configuration", () => { - const requestOptions: OpenAI.Chat.Completions.ChatCompletionCreateParamsStreaming = { + const requestOptions: ArkChatCompletionCreateParamsStreaming = { model: "doubao-pro-4k", messages: [ { role: "system", content: "You are a helpful assistant." }, @@ -28,7 +35,7 @@ describe("Ark Context Caching", () => { }) it("should add previous response ID when provided", () => { - const requestOptions: OpenAI.Chat.Completions.ChatCompletionCreateParamsStreaming = { + const requestOptions: ArkChatCompletionCreateParamsStreaming = { model: "doubao-pro-4k", messages: [{ role: "user", content: "Follow up question" }], stream: true, @@ -47,7 +54,7 @@ describe("Ark Context Caching", () => { }) it("should add cache TTL when provided", () => { - const requestOptions: OpenAI.Chat.Completions.ChatCompletionCreateParamsStreaming = { + const requestOptions: ArkChatCompletionCreateParamsStreaming = { model: "doubao-pro-4k", messages: [{ role: "user", content: "Hello!" }], stream: true, @@ -66,7 +73,7 @@ describe("Ark Context Caching", () => { }) it("should add both previous response ID and cache TTL", () => { - const requestOptions: OpenAI.Chat.Completions.ChatCompletionCreateParamsStreaming = { + const requestOptions: ArkChatCompletionCreateParamsStreaming = { model: "doubao-pro-4k", messages: [{ role: "user", content: "Hello!" }], stream: true, @@ -87,7 +94,7 @@ describe("Ark Context Caching", () => { }) it("should work with non-streaming requests", () => { - const requestOptions: OpenAI.Chat.Completions.ChatCompletionCreateParamsNonStreaming = { + const requestOptions: ArkChatCompletionCreateParamsNonStreaming = { model: "doubao-pro-4k", messages: [{ role: "user", content: "Hello!" }], } @@ -107,7 +114,7 @@ describe("Ark Context Caching", () => { }) it("should not add optional fields when not provided", () => { - const requestOptions: OpenAI.Chat.Completions.ChatCompletionCreateParamsStreaming = { + const requestOptions: ArkChatCompletionCreateParamsStreaming = { model: "doubao-pro-4k", messages: [{ role: "user", content: "Hello!" }], stream: true, @@ -323,7 +330,7 @@ describe("Ark Context Caching", () => { describe("integration scenarios", () => { it("should handle complete caching workflow", () => { // First request - no previous response ID - const firstRequest: OpenAI.Chat.Completions.ChatCompletionCreateParamsStreaming = { + const firstRequest: ArkChatCompletionCreateParamsStreaming = { model: "doubao-pro-4k", messages: [{ role: "user", content: "Hello!" }], stream: true, @@ -364,7 +371,7 @@ describe("Ark Context Caching", () => { expect(hasArkCachedTokens(firstResponse.usage)).toBe(false) // Second request - with previous response ID - const secondRequest: OpenAI.Chat.Completions.ChatCompletionCreateParamsStreaming = { + const secondRequest: ArkChatCompletionCreateParamsStreaming = { model: "doubao-pro-4k", messages: [ { role: "user", content: "Hello!" }, diff --git a/src/api/transform/caching/ark.ts b/src/api/transform/caching/ark.ts index 14c80c9a56..6b7edfc47d 100644 --- a/src/api/transform/caching/ark.ts +++ b/src/api/transform/caching/ark.ts @@ -19,6 +19,39 @@ export interface ArkCacheOptions { cacheTtl?: number } +/** + * Ark-specific caching configuration + */ +export interface ArkCachingConfig { + type: "enabled" +} + +/** + * Extended OpenAI request parameters with Ark-specific caching support + */ +export interface ArkChatCompletionCreateParamsStreaming + extends OpenAI.Chat.Completions.ChatCompletionCreateParamsStreaming { + /** Ark caching configuration */ + caching?: ArkCachingConfig + /** Previous response ID for context continuation */ + previous_response_id?: string + /** Cache TTL in seconds */ + cache_ttl?: number +} + +/** + * Extended OpenAI request parameters with Ark-specific caching support (non-streaming) + */ +export interface ArkChatCompletionCreateParamsNonStreaming + extends OpenAI.Chat.Completions.ChatCompletionCreateParamsNonStreaming { + /** Ark caching configuration */ + caching?: ArkCachingConfig + /** Previous response ID for context continuation */ + previous_response_id?: string + /** Cache TTL in seconds */ + cache_ttl?: number +} + /** * Add context caching support for Ark/Volcengine using the Responses API * @@ -26,24 +59,22 @@ export interface ArkCacheOptions { * @param cacheOptions - Ark-specific caching options */ export function addArkCaching( - requestOptions: - | OpenAI.Chat.Completions.ChatCompletionCreateParamsStreaming - | OpenAI.Chat.Completions.ChatCompletionCreateParamsNonStreaming, + requestOptions: ArkChatCompletionCreateParamsStreaming | ArkChatCompletionCreateParamsNonStreaming, cacheOptions: ArkCacheOptions = {}, ): void { // Enable caching for this request - ;(requestOptions as any).caching = { + requestOptions.caching = { type: "enabled", } // If we have a previous response ID, reference it for context continuation if (cacheOptions.previousResponseId) { - ;(requestOptions as any).previous_response_id = cacheOptions.previousResponseId + requestOptions.previous_response_id = cacheOptions.previousResponseId } // Set cache TTL (default to 1 hour as recommended in the issue) if (cacheOptions.cacheTtl) { - ;(requestOptions as any).cache_ttl = cacheOptions.cacheTtl + requestOptions.cache_ttl = cacheOptions.cacheTtl } }