From aec03d71c36e938822cff825927b78fc1f96b9d2 Mon Sep 17 00:00:00 2001 From: Tom X Nguyen Date: Mon, 3 Mar 2025 07:47:12 +0700 Subject: [PATCH 1/4] feat: add virtual id for extended output capabilities --- src/api/providers/anthropic.ts | 101 ++++++++++++++++++++++++++------- src/shared/api.ts | 12 ++++ 2 files changed, 92 insertions(+), 21 deletions(-) diff --git a/src/api/providers/anthropic.ts b/src/api/providers/anthropic.ts index a23f99261e..6cd57704c0 100644 --- a/src/api/providers/anthropic.ts +++ b/src/api/providers/anthropic.ts @@ -29,7 +29,7 @@ export class AnthropicHandler extends BaseProvider implements SingleCompletionHa async *createMessage(systemPrompt: string, messages: Anthropic.Messages.MessageParam[]): ApiStream { let stream: AnthropicStream const cacheControl: CacheControlEphemeral = { type: "ephemeral" } - let { id: modelId, maxTokens, thinking, temperature } = this.getModel() + let { id: modelId, maxTokens, thinking, temperature, virtualId } = this.getModel() switch (modelId) { case "claude-3-7-sonnet-20250219": @@ -82,6 +82,15 @@ export class AnthropicHandler extends BaseProvider implements SingleCompletionHa // prompt caching: https://x.com/alexalbert__/status/1823751995901272068 // https://github.com/anthropics/anthropic-sdk-typescript?tab=readme-ov-file#default-headers // https://github.com/anthropics/anthropic-sdk-typescript/commit/c920b77fc67bd839bfeb6716ceab9d7c9bbe7393 + + // Check for the thinking-128k variant first + if (virtualId === "claude-3-7-sonnet-20250219:thinking-128k") { + return { + headers: { "anthropic-beta": "output-128k-2025-02-19" }, + } + } + + // Then check for models that support prompt caching switch (modelId) { case "claude-3-5-sonnet-20241022": case "claude-3-5-haiku-20241022": @@ -184,31 +193,58 @@ export class AnthropicHandler extends BaseProvider implements SingleCompletionHa let id = modelId && modelId in anthropicModels ? (modelId as AnthropicModelId) : anthropicDefaultModelId const info: ModelInfo = anthropicModels[id] + // Track the original model ID for special variant handling + const virtualId = id + // The `:thinking` variant is a virtual identifier for the // `claude-3-7-sonnet-20250219` model with a thinking budget. // We can handle this more elegantly in the future. - if (id === "claude-3-7-sonnet-20250219:thinking") { + if (id === "claude-3-7-sonnet-20250219:thinking" || id === "claude-3-7-sonnet-20250219:thinking-128k") { id = "claude-3-7-sonnet-20250219" } return { id, info, + virtualId, // Include the original ID to use for header selection ...getModelParams({ options: this.options, model: info, defaultMaxTokens: ANTHROPIC_DEFAULT_MAX_TOKENS }), } } async completePrompt(prompt: string) { - let { id: modelId, maxTokens, thinking, temperature } = this.getModel() + let { id: modelId, maxTokens, thinking, temperature, virtualId } = this.getModel() - const message = await this.client.messages.create({ - model: modelId, - max_tokens: maxTokens ?? ANTHROPIC_DEFAULT_MAX_TOKENS, - thinking, - temperature, - messages: [{ role: "user", content: prompt }], - stream: false, - }) + const message = await this.client.messages.create( + { + model: modelId, + max_tokens: maxTokens ?? ANTHROPIC_DEFAULT_MAX_TOKENS, + thinking, + temperature, + messages: [{ role: "user", content: prompt }], + stream: false, + }, + (() => { + // Check for the thinking-128k variant first + if (virtualId === "claude-3-7-sonnet-20250219:thinking-128k") { + return { + headers: { "anthropic-beta": "output-128k-2025-02-19" }, + } + } + + // Then check for models that support prompt caching + switch (modelId) { + case "claude-3-5-sonnet-20241022": + case "claude-3-5-haiku-20241022": + case "claude-3-opus-20240229": + case "claude-3-haiku-20240307": + return { + headers: { "anthropic-beta": "prompt-caching-2024-07-31" }, + } + default: + return undefined + } + })(), + ) const content = message.content.find(({ type }) => type === "text") return content?.type === "text" ? content.text : "" @@ -223,17 +259,40 @@ export class AnthropicHandler extends BaseProvider implements SingleCompletionHa override async countTokens(content: Array): Promise { try { // Use the current model - const actualModelId = this.getModel().id + const { id: actualModelId, virtualId } = this.getModel() - const response = await this.client.messages.countTokens({ - model: actualModelId, - messages: [ - { - role: "user", - content: content, - }, - ], - }) + const response = await this.client.messages.countTokens( + { + model: actualModelId, + messages: [ + { + role: "user", + content: content, + }, + ], + }, + (() => { + // Check for the thinking-128k variant first + if (virtualId === "claude-3-7-sonnet-20250219:thinking-128k") { + return { + headers: { "anthropic-beta": "output-128k-2025-02-19" }, + } + } + + // Then check for models that support prompt caching + switch (actualModelId) { + case "claude-3-5-sonnet-20241022": + case "claude-3-5-haiku-20241022": + case "claude-3-opus-20240229": + case "claude-3-haiku-20240307": + return { + headers: { "anthropic-beta": "prompt-caching-2024-07-31" }, + } + default: + return undefined + } + })(), + ) return response.input_tokens } catch (error) { diff --git a/src/shared/api.ts b/src/shared/api.ts index b16e5142a0..f6085aba98 100644 --- a/src/shared/api.ts +++ b/src/shared/api.ts @@ -98,6 +98,18 @@ export interface ModelInfo { export type AnthropicModelId = keyof typeof anthropicModels export const anthropicDefaultModelId: AnthropicModelId = "claude-3-7-sonnet-20250219" export const anthropicModels = { + "claude-3-7-sonnet-20250219:thinking-128k": { + maxTokens: 128_000, + contextWindow: 200_000, + supportsImages: true, + supportsComputerUse: true, + supportsPromptCache: true, + inputPrice: 3.0, // $3 per million input tokens + outputPrice: 15.0, // $15 per million output tokens + cacheWritesPrice: 3.75, // $3.75 per million tokens + cacheReadsPrice: 0.3, // $0.30 per million tokens + thinking: true, + }, "claude-3-7-sonnet-20250219:thinking": { maxTokens: 64_000, contextWindow: 200_000, From d2b4a836059e86c33ded35145635c332a3087d97 Mon Sep 17 00:00:00 2001 From: Tom X Nguyen Date: Mon, 3 Mar 2025 08:33:35 +0700 Subject: [PATCH 2/4] fix: simplify logic and allow for multiple betas --- src/api/providers/anthropic.ts | 93 +++++++++------------------------- 1 file changed, 24 insertions(+), 69 deletions(-) diff --git a/src/api/providers/anthropic.ts b/src/api/providers/anthropic.ts index 6cd57704c0..4df53244c8 100644 --- a/src/api/providers/anthropic.ts +++ b/src/api/providers/anthropic.ts @@ -83,11 +83,11 @@ export class AnthropicHandler extends BaseProvider implements SingleCompletionHa // https://github.com/anthropics/anthropic-sdk-typescript?tab=readme-ov-file#default-headers // https://github.com/anthropics/anthropic-sdk-typescript/commit/c920b77fc67bd839bfeb6716ceab9d7c9bbe7393 + const betas = [] + // Check for the thinking-128k variant first if (virtualId === "claude-3-7-sonnet-20250219:thinking-128k") { - return { - headers: { "anthropic-beta": "output-128k-2025-02-19" }, - } + betas.push("output-128k-2025-02-19") } // Then check for models that support prompt caching @@ -96,8 +96,9 @@ export class AnthropicHandler extends BaseProvider implements SingleCompletionHa case "claude-3-5-haiku-20241022": case "claude-3-opus-20240229": case "claude-3-haiku-20240307": + betas.push("prompt-caching-2024-07-31") return { - headers: { "anthropic-beta": "prompt-caching-2024-07-31" }, + headers: { "anthropic-beta": betas.join(",") }, } default: return undefined @@ -212,39 +213,16 @@ export class AnthropicHandler extends BaseProvider implements SingleCompletionHa } async completePrompt(prompt: string) { - let { id: modelId, maxTokens, thinking, temperature, virtualId } = this.getModel() + let { id: modelId, maxTokens, thinking, temperature } = this.getModel() - const message = await this.client.messages.create( - { - model: modelId, - max_tokens: maxTokens ?? ANTHROPIC_DEFAULT_MAX_TOKENS, - thinking, - temperature, - messages: [{ role: "user", content: prompt }], - stream: false, - }, - (() => { - // Check for the thinking-128k variant first - if (virtualId === "claude-3-7-sonnet-20250219:thinking-128k") { - return { - headers: { "anthropic-beta": "output-128k-2025-02-19" }, - } - } - - // Then check for models that support prompt caching - switch (modelId) { - case "claude-3-5-sonnet-20241022": - case "claude-3-5-haiku-20241022": - case "claude-3-opus-20240229": - case "claude-3-haiku-20240307": - return { - headers: { "anthropic-beta": "prompt-caching-2024-07-31" }, - } - default: - return undefined - } - })(), - ) + const message = await this.client.messages.create({ + model: modelId, + max_tokens: maxTokens ?? ANTHROPIC_DEFAULT_MAX_TOKENS, + thinking, + temperature, + messages: [{ role: "user", content: prompt }], + stream: false, + }) const content = message.content.find(({ type }) => type === "text") return content?.type === "text" ? content.text : "" @@ -259,40 +237,17 @@ export class AnthropicHandler extends BaseProvider implements SingleCompletionHa override async countTokens(content: Array): Promise { try { // Use the current model - const { id: actualModelId, virtualId } = this.getModel() + const actualModelId = this.getModel().id - const response = await this.client.messages.countTokens( - { - model: actualModelId, - messages: [ - { - role: "user", - content: content, - }, - ], - }, - (() => { - // Check for the thinking-128k variant first - if (virtualId === "claude-3-7-sonnet-20250219:thinking-128k") { - return { - headers: { "anthropic-beta": "output-128k-2025-02-19" }, - } - } - - // Then check for models that support prompt caching - switch (actualModelId) { - case "claude-3-5-sonnet-20241022": - case "claude-3-5-haiku-20241022": - case "claude-3-opus-20240229": - case "claude-3-haiku-20240307": - return { - headers: { "anthropic-beta": "prompt-caching-2024-07-31" }, - } - default: - return undefined - } - })(), - ) + const response = await this.client.messages.countTokens({ + model: actualModelId, + messages: [ + { + role: "user", + content: content, + }, + ], + }) return response.input_tokens } catch (error) { From 38cf054391da259465ee8238cf26ebbdb6ad47e7 Mon Sep 17 00:00:00 2001 From: Tom X Nguyen Date: Mon, 3 Mar 2025 09:11:18 +0700 Subject: [PATCH 3/4] chore: add missing 3.7 sonnet modelid --- src/api/providers/anthropic.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/api/providers/anthropic.ts b/src/api/providers/anthropic.ts index 4df53244c8..53d28593e9 100644 --- a/src/api/providers/anthropic.ts +++ b/src/api/providers/anthropic.ts @@ -92,6 +92,7 @@ export class AnthropicHandler extends BaseProvider implements SingleCompletionHa // Then check for models that support prompt caching switch (modelId) { + case "claude-3-7-sonnet-20250219": case "claude-3-5-sonnet-20241022": case "claude-3-5-haiku-20241022": case "claude-3-opus-20240229": From ef93f5d4220cecfee4527c8733f8801e85148577 Mon Sep 17 00:00:00 2001 From: Tom X Nguyen Date: Mon, 3 Mar 2025 11:17:01 +0700 Subject: [PATCH 4/4] refactor: always enable 128k for simplification --- src/api/providers/anthropic.ts | 4 ++-- src/shared/api.ts | 14 +------------- 2 files changed, 3 insertions(+), 15 deletions(-) diff --git a/src/api/providers/anthropic.ts b/src/api/providers/anthropic.ts index 53d28593e9..173550dc58 100644 --- a/src/api/providers/anthropic.ts +++ b/src/api/providers/anthropic.ts @@ -86,7 +86,7 @@ export class AnthropicHandler extends BaseProvider implements SingleCompletionHa const betas = [] // Check for the thinking-128k variant first - if (virtualId === "claude-3-7-sonnet-20250219:thinking-128k") { + if (virtualId === "claude-3-7-sonnet-20250219:thinking") { betas.push("output-128k-2025-02-19") } @@ -201,7 +201,7 @@ export class AnthropicHandler extends BaseProvider implements SingleCompletionHa // The `:thinking` variant is a virtual identifier for the // `claude-3-7-sonnet-20250219` model with a thinking budget. // We can handle this more elegantly in the future. - if (id === "claude-3-7-sonnet-20250219:thinking" || id === "claude-3-7-sonnet-20250219:thinking-128k") { + if (id === "claude-3-7-sonnet-20250219:thinking") { id = "claude-3-7-sonnet-20250219" } diff --git a/src/shared/api.ts b/src/shared/api.ts index f6085aba98..462afc46cd 100644 --- a/src/shared/api.ts +++ b/src/shared/api.ts @@ -98,20 +98,8 @@ export interface ModelInfo { export type AnthropicModelId = keyof typeof anthropicModels export const anthropicDefaultModelId: AnthropicModelId = "claude-3-7-sonnet-20250219" export const anthropicModels = { - "claude-3-7-sonnet-20250219:thinking-128k": { - maxTokens: 128_000, - contextWindow: 200_000, - supportsImages: true, - supportsComputerUse: true, - supportsPromptCache: true, - inputPrice: 3.0, // $3 per million input tokens - outputPrice: 15.0, // $15 per million output tokens - cacheWritesPrice: 3.75, // $3.75 per million tokens - cacheReadsPrice: 0.3, // $0.30 per million tokens - thinking: true, - }, "claude-3-7-sonnet-20250219:thinking": { - maxTokens: 64_000, + maxTokens: 128_000, contextWindow: 200_000, supportsImages: true, supportsComputerUse: true,