mirror of
https://github.com/RooVetGit/Roo-Code.git
synced 2026-09-07 08:26:51 +00:00
feat: migrate anthropic provider to AI SDK
This commit is contained in:
daniel-lxs
parent
06b25185e2
commit
83561622cc
8 changed files with 1054 additions and 884 deletions
15
pnpm-lock.yaml
generated
15
pnpm-lock.yaml
generated
|
|
@ -746,6 +746,9 @@ importers:
|
|||
|
||||
src:
|
||||
dependencies:
|
||||
'@ai-sdk/anthropic':
|
||||
specifier: ^3.0.37
|
||||
version: 3.0.37(zod@3.25.76)
|
||||
'@ai-sdk/cerebras':
|
||||
specifier: ^1.0.0
|
||||
version: 1.0.35(zod@3.25.76)
|
||||
|
|
@ -1423,6 +1426,12 @@ packages:
|
|||
peerDependencies:
|
||||
zod: 3.25.76
|
||||
|
||||
'@ai-sdk/anthropic@3.0.37':
|
||||
resolution: {integrity: sha512-tEgcJPw+a6obbF+SHrEiZsx3DNxOHqeY8bK4IpiNsZ8YPZD141R34g3lEAaQnmNN5mGsEJ8SXoEDabuzi8wFJQ==}
|
||||
engines: {node: '>=18'}
|
||||
peerDependencies:
|
||||
zod: 3.25.76
|
||||
|
||||
'@ai-sdk/cerebras@1.0.35':
|
||||
resolution: {integrity: sha512-JrNdMYptrOUjNthibgBeAcBjZ/H+fXb49sSrWhOx5Aq8eUcrYvwQ2DtSAi8VraHssZu78NAnBMrgFWSUOTXFxw==}
|
||||
engines: {node: '>=18'}
|
||||
|
|
@ -11136,6 +11145,12 @@ snapshots:
|
|||
'@ai-sdk/provider-utils': 3.0.20(zod@3.25.76)
|
||||
zod: 3.25.76
|
||||
|
||||
'@ai-sdk/anthropic@3.0.37(zod@3.25.76)':
|
||||
dependencies:
|
||||
'@ai-sdk/provider': 3.0.7
|
||||
'@ai-sdk/provider-utils': 4.0.13(zod@3.25.76)
|
||||
zod: 3.25.76
|
||||
|
||||
'@ai-sdk/cerebras@1.0.35(zod@3.25.76)':
|
||||
dependencies:
|
||||
'@ai-sdk/openai-compatible': 1.0.31(zod@3.25.76)
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -1,7 +1,6 @@
|
|||
import { Anthropic } from "@anthropic-ai/sdk"
|
||||
import { Stream as AnthropicStream } from "@anthropic-ai/sdk/streaming"
|
||||
import { CacheControlEphemeral } from "@anthropic-ai/sdk/resources"
|
||||
import OpenAI from "openai"
|
||||
import type { Anthropic } from "@anthropic-ai/sdk"
|
||||
import { createAnthropic, type AnthropicProvider } from "@ai-sdk/anthropic"
|
||||
import { streamText, generateText, ToolSet } from "ai"
|
||||
|
||||
import {
|
||||
type ModelInfo,
|
||||
|
|
@ -15,34 +14,67 @@ import { TelemetryService } from "@roo-code/telemetry"
|
|||
|
||||
import type { ApiHandlerOptions } from "../../shared/api"
|
||||
|
||||
import { ApiStream } from "../transform/stream"
|
||||
import { getModelParams } from "../transform/model-params"
|
||||
import { filterNonAnthropicBlocks } from "../transform/anthropic-filter"
|
||||
import { handleProviderError } from "./utils/error-handler"
|
||||
|
||||
import { BaseProvider } from "./base-provider"
|
||||
import type { SingleCompletionHandler, ApiHandlerCreateMessageMetadata } from "../index"
|
||||
import { calculateApiCostAnthropic } from "../../shared/cost"
|
||||
import {
|
||||
convertOpenAIToolsToAnthropic,
|
||||
convertOpenAIToolChoiceToAnthropic,
|
||||
} from "../../core/prompts/tools/native-tools/converters"
|
||||
convertToAiSdkMessages,
|
||||
convertToolsForAiSdk,
|
||||
processAiSdkStreamPart,
|
||||
mapToolChoice,
|
||||
} from "../transform/ai-sdk"
|
||||
import type { ApiStream, ApiStreamUsageChunk } from "../transform/stream"
|
||||
import { getModelParams } from "../transform/model-params"
|
||||
import { addAiSdkAnthropicCacheBreakpoints } from "../transform/caching/ai-sdk-anthropic"
|
||||
|
||||
import type { SingleCompletionHandler, ApiHandlerCreateMessageMetadata } from "../index"
|
||||
import { BaseProvider } from "./base-provider"
|
||||
import { DEFAULT_HEADERS } from "./constants"
|
||||
import { calculateApiCostAnthropic } from "../../shared/cost"
|
||||
|
||||
/**
|
||||
* Models that support Anthropic prompt caching.
|
||||
* These models require the `prompt-caching-2024-07-31` beta header.
|
||||
*/
|
||||
const CACHE_SUPPORTED_MODELS = new Set<string>([
|
||||
"claude-sonnet-4-5",
|
||||
"claude-sonnet-4-20250514",
|
||||
"claude-opus-4-6",
|
||||
"claude-opus-4-5-20251101",
|
||||
"claude-opus-4-1-20250805",
|
||||
"claude-opus-4-20250514",
|
||||
"claude-3-7-sonnet-20250219",
|
||||
"claude-3-5-sonnet-20241022",
|
||||
"claude-3-5-haiku-20241022",
|
||||
"claude-3-opus-20240229",
|
||||
"claude-haiku-4-5-20251001",
|
||||
"claude-3-haiku-20240307",
|
||||
])
|
||||
|
||||
/**
|
||||
* Models that support the 1M context beta.
|
||||
*/
|
||||
const CONTEXT_1M_MODELS = new Set<string>(["claude-sonnet-4-20250514", "claude-sonnet-4-5", "claude-opus-4-6"])
|
||||
|
||||
export class AnthropicHandler extends BaseProvider implements SingleCompletionHandler {
|
||||
private options: ApiHandlerOptions
|
||||
private client: Anthropic
|
||||
private provider: AnthropicProvider
|
||||
private readonly providerName = "Anthropic"
|
||||
private lastThoughtSignature: string | undefined
|
||||
private lastRedactedBlocks: Array<{ type: "redacted_thinking"; data: string }> | undefined
|
||||
|
||||
constructor(options: ApiHandlerOptions) {
|
||||
super()
|
||||
this.options = options
|
||||
|
||||
const apiKeyFieldName =
|
||||
this.options.anthropicBaseUrl && this.options.anthropicUseAuthToken ? "authToken" : "apiKey"
|
||||
const useAuthToken = !!(this.options.anthropicBaseUrl && this.options.anthropicUseAuthToken)
|
||||
|
||||
this.client = new Anthropic({
|
||||
const headers: Record<string, string> = { ...DEFAULT_HEADERS }
|
||||
if (useAuthToken && this.options.apiKey) {
|
||||
headers["Authorization"] = `Bearer ${this.options.apiKey}`
|
||||
}
|
||||
|
||||
this.provider = createAnthropic({
|
||||
apiKey: useAuthToken ? "" : (this.options.apiKey ?? "not-provided"),
|
||||
baseURL: this.options.anthropicBaseUrl || undefined,
|
||||
[apiKeyFieldName]: this.options.apiKey,
|
||||
headers,
|
||||
})
|
||||
}
|
||||
|
||||
|
|
@ -51,9 +83,7 @@ export class AnthropicHandler extends BaseProvider implements SingleCompletionHa
|
|||
messages: Anthropic.Messages.MessageParam[],
|
||||
metadata?: ApiHandlerCreateMessageMetadata,
|
||||
): ApiStream {
|
||||
let stream: AnthropicStream<Anthropic.Messages.RawMessageStreamEvent>
|
||||
const cacheControl: CacheControlEphemeral = { type: "ephemeral" }
|
||||
let {
|
||||
const {
|
||||
id: modelId,
|
||||
betas = ["fine-grained-tool-streaming-2025-05-14"],
|
||||
maxTokens,
|
||||
|
|
@ -61,271 +91,120 @@ export class AnthropicHandler extends BaseProvider implements SingleCompletionHa
|
|||
reasoning: thinking,
|
||||
} = this.getModel()
|
||||
|
||||
// Filter out non-Anthropic blocks (reasoning, thoughtSignature, etc.) before sending to the API
|
||||
const sanitizedMessages = filterNonAnthropicBlocks(messages)
|
||||
// Build beta headers
|
||||
const betaHeaders = [...betas]
|
||||
|
||||
// Add 1M context beta flag if enabled for supported models (Claude Sonnet 4/4.5, Opus 4.6)
|
||||
if (
|
||||
(modelId === "claude-sonnet-4-20250514" ||
|
||||
modelId === "claude-sonnet-4-5" ||
|
||||
modelId === "claude-opus-4-6") &&
|
||||
this.options.anthropicBeta1MContext
|
||||
) {
|
||||
betas.push("context-1m-2025-08-07")
|
||||
if (CACHE_SUPPORTED_MODELS.has(modelId)) {
|
||||
betaHeaders.push("prompt-caching-2024-07-31")
|
||||
}
|
||||
|
||||
const nativeToolParams = {
|
||||
tools: convertOpenAIToolsToAnthropic(metadata?.tools ?? []),
|
||||
tool_choice: convertOpenAIToolChoiceToAnthropic(metadata?.tool_choice, metadata?.parallelToolCalls),
|
||||
if (CONTEXT_1M_MODELS.has(modelId) && this.options.anthropicBeta1MContext) {
|
||||
betaHeaders.push("context-1m-2025-08-07")
|
||||
}
|
||||
|
||||
switch (modelId) {
|
||||
case "claude-sonnet-4-5":
|
||||
case "claude-sonnet-4-20250514":
|
||||
case "claude-opus-4-6":
|
||||
case "claude-opus-4-5-20251101":
|
||||
case "claude-opus-4-1-20250805":
|
||||
case "claude-opus-4-20250514":
|
||||
case "claude-3-7-sonnet-20250219":
|
||||
case "claude-3-5-sonnet-20241022":
|
||||
case "claude-3-5-haiku-20241022":
|
||||
case "claude-3-opus-20240229":
|
||||
case "claude-haiku-4-5-20251001":
|
||||
case "claude-3-haiku-20240307": {
|
||||
/**
|
||||
* The latest message will be the new user message, one before
|
||||
* will be the assistant message from a previous request, and
|
||||
* the user message before that will be a previously cached user
|
||||
* message. So we need to mark the latest user message as
|
||||
* ephemeral to cache it for the next request, and mark the
|
||||
* second to last user message as ephemeral to let the server
|
||||
* know the last message to retrieve from the cache for the
|
||||
* current request.
|
||||
*/
|
||||
const userMsgIndices = sanitizedMessages.reduce(
|
||||
(acc, msg, index) => (msg.role === "user" ? [...acc, index] : acc),
|
||||
[] as number[],
|
||||
)
|
||||
// Convert messages to AI SDK format (handles filtering of reasoning/thinking/etc. blocks)
|
||||
const aiSdkMessages = convertToAiSdkMessages(messages)
|
||||
|
||||
const lastUserMsgIndex = userMsgIndices[userMsgIndices.length - 1] ?? -1
|
||||
const secondLastMsgUserIndex = userMsgIndices[userMsgIndices.length - 2] ?? -1
|
||||
// Add cache breakpoints to the last 2 user messages
|
||||
const useCache = CACHE_SUPPORTED_MODELS.has(modelId)
|
||||
const cachedMessages = useCache ? addAiSdkAnthropicCacheBreakpoints(aiSdkMessages) : aiSdkMessages
|
||||
|
||||
try {
|
||||
stream = await this.client.messages.create(
|
||||
{
|
||||
model: modelId,
|
||||
max_tokens: maxTokens ?? ANTHROPIC_DEFAULT_MAX_TOKENS,
|
||||
temperature,
|
||||
thinking,
|
||||
// Setting cache breakpoint for system prompt so new tasks can reuse it.
|
||||
system: [{ text: systemPrompt, type: "text", cache_control: cacheControl }],
|
||||
messages: sanitizedMessages.map((message, index) => {
|
||||
if (index === lastUserMsgIndex || index === secondLastMsgUserIndex) {
|
||||
return {
|
||||
...message,
|
||||
content:
|
||||
typeof message.content === "string"
|
||||
? [{ type: "text", text: message.content, cache_control: cacheControl }]
|
||||
: message.content.map((content, contentIndex) =>
|
||||
contentIndex === message.content.length - 1
|
||||
? { ...content, cache_control: cacheControl }
|
||||
: content,
|
||||
),
|
||||
}
|
||||
// Convert tools to AI SDK format
|
||||
const openAiTools = this.convertToolsForOpenAI(metadata?.tools)
|
||||
const aiSdkTools = convertToolsForAiSdk(openAiTools) as ToolSet | undefined
|
||||
const toolChoice = mapToolChoice(metadata?.tool_choice)
|
||||
|
||||
// Map Anthropic thinking config from snake_case to camelCase for AI SDK
|
||||
const thinkingProviderOptions = thinking
|
||||
? {
|
||||
thinking:
|
||||
thinking.type === "enabled"
|
||||
? {
|
||||
type: "enabled" as const,
|
||||
budgetTokens: (thinking as { budget_tokens: number }).budget_tokens,
|
||||
}
|
||||
return message
|
||||
}),
|
||||
stream: true,
|
||||
...nativeToolParams,
|
||||
},
|
||||
(() => {
|
||||
// prompt caching: https://x.com/alexalbert__/status/1823751995901272068
|
||||
// https://github.com/anthropics/anthropic-sdk-typescript?tab=readme-ov-file#default-headers
|
||||
// https://github.com/anthropics/anthropic-sdk-typescript/commit/c920b77fc67bd839bfeb6716ceab9d7c9bbe7393
|
||||
: thinking,
|
||||
}
|
||||
: undefined
|
||||
|
||||
// Then check for models that support prompt caching
|
||||
switch (modelId) {
|
||||
case "claude-sonnet-4-5":
|
||||
case "claude-sonnet-4-20250514":
|
||||
case "claude-opus-4-6":
|
||||
case "claude-opus-4-5-20251101":
|
||||
case "claude-opus-4-1-20250805":
|
||||
case "claude-opus-4-20250514":
|
||||
case "claude-3-7-sonnet-20250219":
|
||||
case "claude-3-5-sonnet-20241022":
|
||||
case "claude-3-5-haiku-20241022":
|
||||
case "claude-3-opus-20240229":
|
||||
case "claude-haiku-4-5-20251001":
|
||||
case "claude-3-haiku-20240307":
|
||||
betas.push("prompt-caching-2024-07-31")
|
||||
return { headers: { "anthropic-beta": betas.join(",") } }
|
||||
default:
|
||||
return undefined
|
||||
}
|
||||
})(),
|
||||
)
|
||||
} catch (error) {
|
||||
TelemetryService.instance.captureException(
|
||||
new ApiProviderError(
|
||||
error instanceof Error ? error.message : String(error),
|
||||
this.providerName,
|
||||
modelId,
|
||||
"createMessage",
|
||||
),
|
||||
)
|
||||
throw error
|
||||
}
|
||||
break
|
||||
}
|
||||
default: {
|
||||
try {
|
||||
stream = (await this.client.messages.create({
|
||||
model: modelId,
|
||||
max_tokens: maxTokens ?? ANTHROPIC_DEFAULT_MAX_TOKENS,
|
||||
temperature,
|
||||
system: [{ text: systemPrompt, type: "text" }],
|
||||
messages: sanitizedMessages,
|
||||
stream: true,
|
||||
...nativeToolParams,
|
||||
})) as any
|
||||
} catch (error) {
|
||||
TelemetryService.instance.captureException(
|
||||
new ApiProviderError(
|
||||
error instanceof Error ? error.message : String(error),
|
||||
this.providerName,
|
||||
modelId,
|
||||
"createMessage",
|
||||
),
|
||||
)
|
||||
throw error
|
||||
}
|
||||
break
|
||||
}
|
||||
// Build system prompt — with cache control for supported models
|
||||
// Cast to any to bypass strict typing: the AI SDK Anthropic provider accepts
|
||||
// text parts with providerOptions at runtime for system prompt caching.
|
||||
const system: any = useCache
|
||||
? [
|
||||
{
|
||||
type: "text" as const,
|
||||
text: systemPrompt,
|
||||
providerOptions: { anthropic: { cacheControl: { type: "ephemeral" } } },
|
||||
},
|
||||
]
|
||||
: systemPrompt
|
||||
|
||||
// Build the request options
|
||||
const requestOptions: Parameters<typeof streamText>[0] = {
|
||||
model: this.provider(modelId),
|
||||
system,
|
||||
messages: cachedMessages,
|
||||
maxOutputTokens: maxTokens ?? ANTHROPIC_DEFAULT_MAX_TOKENS,
|
||||
temperature,
|
||||
tools: aiSdkTools,
|
||||
toolChoice,
|
||||
headers: { "anthropic-beta": betaHeaders.join(",") },
|
||||
...(thinkingProviderOptions && {
|
||||
providerOptions: { anthropic: thinkingProviderOptions } as any,
|
||||
}),
|
||||
}
|
||||
|
||||
let inputTokens = 0
|
||||
let outputTokens = 0
|
||||
let cacheWriteTokens = 0
|
||||
let cacheReadTokens = 0
|
||||
try {
|
||||
// Reset reasoning state for this request
|
||||
this.lastThoughtSignature = undefined
|
||||
this.lastRedactedBlocks = undefined
|
||||
|
||||
for await (const chunk of stream) {
|
||||
switch (chunk.type) {
|
||||
case "message_start": {
|
||||
// Tells us cache reads/writes/input/output.
|
||||
const {
|
||||
input_tokens = 0,
|
||||
output_tokens = 0,
|
||||
cache_creation_input_tokens,
|
||||
cache_read_input_tokens,
|
||||
} = chunk.message.usage
|
||||
const result = streamText(requestOptions)
|
||||
|
||||
yield {
|
||||
type: "usage",
|
||||
inputTokens: input_tokens,
|
||||
outputTokens: output_tokens,
|
||||
cacheWriteTokens: cache_creation_input_tokens || undefined,
|
||||
cacheReadTokens: cache_read_input_tokens || undefined,
|
||||
}
|
||||
|
||||
inputTokens += input_tokens
|
||||
outputTokens += output_tokens
|
||||
cacheWriteTokens += cache_creation_input_tokens || 0
|
||||
cacheReadTokens += cache_read_input_tokens || 0
|
||||
|
||||
break
|
||||
// Process the full stream
|
||||
for await (const part of result.fullStream) {
|
||||
for (const chunk of processAiSdkStreamPart(part)) {
|
||||
yield chunk
|
||||
}
|
||||
case "message_delta":
|
||||
// Tells us stop_reason, stop_sequence, and output tokens
|
||||
// along the way and at the end of the message.
|
||||
yield {
|
||||
type: "usage",
|
||||
inputTokens: 0,
|
||||
outputTokens: chunk.usage.output_tokens || 0,
|
||||
}
|
||||
|
||||
break
|
||||
case "message_stop":
|
||||
// No usage data, just an indicator that the message is done.
|
||||
break
|
||||
case "content_block_start":
|
||||
switch (chunk.content_block.type) {
|
||||
case "thinking":
|
||||
// We may receive multiple text blocks, in which
|
||||
// case just insert a line break between them.
|
||||
if (chunk.index > 0) {
|
||||
yield { type: "reasoning", text: "\n" }
|
||||
}
|
||||
|
||||
yield { type: "reasoning", text: chunk.content_block.thinking }
|
||||
break
|
||||
case "text":
|
||||
// We may receive multiple text blocks, in which
|
||||
// case just insert a line break between them.
|
||||
if (chunk.index > 0) {
|
||||
yield { type: "text", text: "\n" }
|
||||
}
|
||||
|
||||
yield { type: "text", text: chunk.content_block.text }
|
||||
break
|
||||
case "tool_use": {
|
||||
// Emit initial tool call partial with id and name
|
||||
yield {
|
||||
type: "tool_call_partial",
|
||||
index: chunk.index,
|
||||
id: chunk.content_block.id,
|
||||
name: chunk.content_block.name,
|
||||
arguments: undefined,
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
break
|
||||
case "content_block_delta":
|
||||
switch (chunk.delta.type) {
|
||||
case "thinking_delta":
|
||||
yield { type: "reasoning", text: chunk.delta.thinking }
|
||||
break
|
||||
case "text_delta":
|
||||
yield { type: "text", text: chunk.delta.text }
|
||||
break
|
||||
case "input_json_delta": {
|
||||
// Emit tool call partial chunks as arguments stream in
|
||||
yield {
|
||||
type: "tool_call_partial",
|
||||
index: chunk.index,
|
||||
id: undefined,
|
||||
name: undefined,
|
||||
arguments: chunk.delta.partial_json,
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
break
|
||||
case "content_block_stop":
|
||||
// Block complete - no action needed for now.
|
||||
// NativeToolCallParser handles tool call completion
|
||||
// Note: Signature for multi-turn thinking would require using stream.finalMessage()
|
||||
// after iteration completes, which requires restructuring the streaming approach.
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if (inputTokens > 0 || outputTokens > 0 || cacheWriteTokens > 0 || cacheReadTokens > 0) {
|
||||
const { totalCost } = calculateApiCostAnthropic(
|
||||
this.getModel().info,
|
||||
inputTokens,
|
||||
outputTokens,
|
||||
cacheWriteTokens,
|
||||
cacheReadTokens,
|
||||
// After stream completes, capture reasoning data for signatures and redacted thinking
|
||||
const reasoning = await result.reasoning
|
||||
if (reasoning && Array.isArray(reasoning)) {
|
||||
for (const entry of reasoning) {
|
||||
// The AI SDK types reasoning parts as { type: "reasoning" } but the
|
||||
// Anthropic provider returns richer types at runtime including "text"
|
||||
// (with signature) and "redacted" (with data). Use any cast.
|
||||
const entryAny = entry as any
|
||||
if (entryAny.type === "text" && entryAny.signature) {
|
||||
this.lastThoughtSignature = entryAny.signature
|
||||
}
|
||||
if (entryAny.type === "redacted" && entryAny.data) {
|
||||
if (!this.lastRedactedBlocks) {
|
||||
this.lastRedactedBlocks = []
|
||||
}
|
||||
this.lastRedactedBlocks.push({
|
||||
type: "redacted_thinking",
|
||||
data: entryAny.data,
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Yield usage metrics at the end
|
||||
const usage = await result.usage
|
||||
const providerMetadata = await result.providerMetadata
|
||||
|
||||
if (usage) {
|
||||
yield this.processUsageMetrics(usage, this.getModel().info, providerMetadata)
|
||||
}
|
||||
} catch (error) {
|
||||
const errorMessage = error instanceof Error ? error.message : String(error)
|
||||
TelemetryService.instance.captureException(
|
||||
new ApiProviderError(errorMessage, this.providerName, modelId, "createMessage"),
|
||||
)
|
||||
|
||||
yield {
|
||||
type: "usage",
|
||||
inputTokens: 0,
|
||||
outputTokens: 0,
|
||||
totalCost,
|
||||
}
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -335,11 +214,7 @@ export class AnthropicHandler extends BaseProvider implements SingleCompletionHa
|
|||
let info: ModelInfo = anthropicModels[id]
|
||||
|
||||
// If 1M context beta is enabled for supported models, update the model info
|
||||
if (
|
||||
(id === "claude-sonnet-4-20250514" || id === "claude-sonnet-4-5" || id === "claude-opus-4-6") &&
|
||||
this.options.anthropicBeta1MContext
|
||||
) {
|
||||
// Use the tier pricing for 1M context
|
||||
if (CONTEXT_1M_MODELS.has(id) && this.options.anthropicBeta1MContext) {
|
||||
const tier = info.tiers?.[0]
|
||||
if (tier) {
|
||||
info = {
|
||||
|
|
@ -373,31 +248,87 @@ export class AnthropicHandler extends BaseProvider implements SingleCompletionHa
|
|||
}
|
||||
|
||||
async completePrompt(prompt: string) {
|
||||
let { id: model, temperature } = this.getModel()
|
||||
const { id: model, temperature } = this.getModel()
|
||||
|
||||
let message
|
||||
try {
|
||||
message = await this.client.messages.create({
|
||||
model,
|
||||
max_tokens: ANTHROPIC_DEFAULT_MAX_TOKENS,
|
||||
thinking: undefined,
|
||||
const result = await generateText({
|
||||
model: this.provider(model),
|
||||
prompt,
|
||||
maxOutputTokens: ANTHROPIC_DEFAULT_MAX_TOKENS,
|
||||
temperature,
|
||||
messages: [{ role: "user", content: prompt }],
|
||||
stream: false,
|
||||
})
|
||||
|
||||
return result.text ?? ""
|
||||
} catch (error) {
|
||||
const errorMessage = error instanceof Error ? error.message : String(error)
|
||||
TelemetryService.instance.captureException(
|
||||
new ApiProviderError(
|
||||
error instanceof Error ? error.message : String(error),
|
||||
this.providerName,
|
||||
model,
|
||||
"completePrompt",
|
||||
),
|
||||
new ApiProviderError(errorMessage, this.providerName, model, "completePrompt"),
|
||||
)
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
const content = message.content.find(({ type }) => type === "text")
|
||||
return content?.type === "text" ? content.text : ""
|
||||
/**
|
||||
* Process usage metrics from the AI SDK response.
|
||||
* Handles Anthropic-specific cache tokens from providerMetadata.
|
||||
*/
|
||||
private processUsageMetrics(
|
||||
usage: {
|
||||
inputTokens?: number
|
||||
outputTokens?: number
|
||||
details?: {
|
||||
cachedInputTokens?: number
|
||||
reasoningTokens?: number
|
||||
}
|
||||
},
|
||||
info: ModelInfo,
|
||||
providerMetadata?: Record<string, unknown>,
|
||||
): ApiStreamUsageChunk {
|
||||
const inputTokens = usage.inputTokens || 0
|
||||
const outputTokens = usage.outputTokens || 0
|
||||
const cacheReadTokens = usage.details?.cachedInputTokens
|
||||
|
||||
// Cache write tokens come from Anthropic-specific provider metadata
|
||||
const anthropicMeta = providerMetadata?.anthropic as { cacheCreationInputTokens?: number } | undefined
|
||||
const cacheWriteTokens = anthropicMeta?.cacheCreationInputTokens
|
||||
|
||||
const { totalCost } = calculateApiCostAnthropic(
|
||||
info,
|
||||
inputTokens,
|
||||
outputTokens,
|
||||
cacheWriteTokens,
|
||||
cacheReadTokens,
|
||||
)
|
||||
|
||||
return {
|
||||
type: "usage",
|
||||
inputTokens,
|
||||
outputTokens,
|
||||
cacheWriteTokens,
|
||||
cacheReadTokens,
|
||||
totalCost,
|
||||
}
|
||||
}
|
||||
|
||||
override isAiSdkProvider(): boolean {
|
||||
return true
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the thought signature captured from the last Anthropic response.
|
||||
* Anthropic extended thinking returns a signature on thinking blocks
|
||||
* that must be round-tripped for tool use continuations.
|
||||
*/
|
||||
getThoughtSignature(): string | undefined {
|
||||
return this.lastThoughtSignature
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns redacted thinking blocks from the last Anthropic response.
|
||||
* These blocks are returned when safety filters trigger on reasoning content
|
||||
* and must be passed back verbatim for proper reasoning continuity.
|
||||
*/
|
||||
getRedactedThinkingBlocks(): Array<{ type: "redacted_thinking"; data: string }> | undefined {
|
||||
return this.lastRedactedBlocks
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -349,7 +349,7 @@ describe("AI SDK conversion utilities", () => {
|
|||
expect(result[0]).toEqual({
|
||||
role: "assistant",
|
||||
content: [
|
||||
{ type: "reasoning", text: "Deep thought" },
|
||||
{ type: "reasoning", text: "Deep thought", signature: "sig" },
|
||||
{ type: "text", text: "OK" },
|
||||
],
|
||||
})
|
||||
|
|
|
|||
|
|
@ -126,7 +126,8 @@ export function convertToAiSdkMessages(
|
|||
}
|
||||
} else if (message.role === "assistant") {
|
||||
const textParts: string[] = []
|
||||
const reasoningParts: string[] = []
|
||||
const reasoningEntries: Array<{ text: string; signature?: string }> = []
|
||||
const redactedReasoningEntries: Array<{ data: string }> = []
|
||||
const reasoningContent = (() => {
|
||||
const maybe = (message as unknown as { reasoning_content?: unknown }).reasoning_content
|
||||
return typeof maybe === "string" && maybe.length > 0 ? maybe : undefined
|
||||
|
|
@ -188,7 +189,7 @@ export function convertToAiSdkMessages(
|
|||
|
||||
const text = (part as unknown as { text?: string }).text
|
||||
if (typeof text === "string" && text.length > 0) {
|
||||
reasoningParts.push(text)
|
||||
reasoningEntries.push({ text })
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
|
@ -196,16 +197,31 @@ export function convertToAiSdkMessages(
|
|||
if ((part as unknown as { type?: string }).type === "thinking") {
|
||||
if (reasoningContent) continue
|
||||
|
||||
const thinking = (part as unknown as { thinking?: string }).thinking
|
||||
const partAny2 = part as unknown as { thinking?: string; signature?: string }
|
||||
const thinking = partAny2.thinking
|
||||
if (typeof thinking === "string" && thinking.length > 0) {
|
||||
reasoningParts.push(thinking)
|
||||
reasoningEntries.push({
|
||||
text: thinking,
|
||||
...(partAny2.signature ? { signature: partAny2.signature } : {}),
|
||||
})
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
// Anthropic redacted_thinking blocks must be round-tripped verbatim.
|
||||
// The AI SDK represents these as { type: "redacted-reasoning", data: "..." }.
|
||||
if ((part as unknown as { type?: string }).type === "redacted_thinking") {
|
||||
const data = (part as unknown as { data?: string }).data
|
||||
if (typeof data === "string") {
|
||||
redactedReasoningEntries.push({ data })
|
||||
}
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
const content: Array<
|
||||
| { type: "reasoning"; text: string }
|
||||
| { type: "reasoning"; text: string; signature?: string }
|
||||
| { type: "redacted-reasoning"; data: string }
|
||||
| { type: "text"; text: string }
|
||||
| {
|
||||
type: "tool-call"
|
||||
|
|
@ -218,8 +234,26 @@ export function convertToAiSdkMessages(
|
|||
|
||||
if (reasoningContent) {
|
||||
content.push({ type: "reasoning", text: reasoningContent })
|
||||
} else if (reasoningParts.length > 0) {
|
||||
content.push({ type: "reasoning", text: reasoningParts.join("") })
|
||||
} else {
|
||||
// When any entry carries a signature (Anthropic extended thinking),
|
||||
// keep entries separate so signatures are preserved for round-tripping.
|
||||
const hasSignatures = reasoningEntries.some((e) => e.signature)
|
||||
|
||||
if (hasSignatures) {
|
||||
for (const entry of reasoningEntries) {
|
||||
content.push({
|
||||
type: "reasoning",
|
||||
text: entry.text,
|
||||
...(entry.signature ? { signature: entry.signature } : {}),
|
||||
})
|
||||
}
|
||||
} else if (reasoningEntries.length > 0) {
|
||||
content.push({ type: "reasoning", text: reasoningEntries.map((e) => e.text).join("") })
|
||||
}
|
||||
|
||||
for (const entry of redactedReasoningEntries) {
|
||||
content.push({ type: "redacted-reasoning", data: entry.data })
|
||||
}
|
||||
}
|
||||
|
||||
if (textParts.length > 0) {
|
||||
|
|
@ -416,6 +450,16 @@ export function* processAiSdkStreamPart(part: ExtendedStreamPart): Generator<Api
|
|||
}
|
||||
break
|
||||
|
||||
case "reasoning-end": {
|
||||
// Anthropic extended thinking: reasoning-end may carry a signature
|
||||
// needed for tool use continuations. Emit as thinking_complete.
|
||||
const signature = (part as any).signature
|
||||
if (signature) {
|
||||
yield { type: "thinking_complete", signature }
|
||||
}
|
||||
break
|
||||
}
|
||||
|
||||
// Ignore lifecycle events that don't need to yield chunks.
|
||||
// Note: tool-call is intentionally ignored because tool-input-start/delta/end already
|
||||
// provide complete tool call information. Emitting tool-call would cause duplicate
|
||||
|
|
@ -423,7 +467,6 @@ export function* processAiSdkStreamPart(part: ExtendedStreamPart): Generator<Api
|
|||
case "text-start":
|
||||
case "text-end":
|
||||
case "reasoning-start":
|
||||
case "reasoning-end":
|
||||
case "start-step":
|
||||
case "finish-step":
|
||||
case "start":
|
||||
|
|
|
|||
152
src/api/transform/caching/__tests__/ai-sdk-anthropic.spec.ts
Normal file
152
src/api/transform/caching/__tests__/ai-sdk-anthropic.spec.ts
Normal file
|
|
@ -0,0 +1,152 @@
|
|||
// npx vitest run src/api/transform/caching/__tests__/ai-sdk-anthropic.spec.ts
|
||||
|
||||
import type { ModelMessage } from "ai"
|
||||
|
||||
import { addAiSdkAnthropicCacheBreakpoints } from "../ai-sdk-anthropic"
|
||||
|
||||
const CACHE_CONTROL = { anthropic: { cacheControl: { type: "ephemeral" } } }
|
||||
|
||||
describe("addAiSdkAnthropicCacheBreakpoints", () => {
|
||||
it("should return messages unchanged when there are no user messages", () => {
|
||||
const messages: ModelMessage[] = [{ role: "assistant", content: [{ type: "text", text: "Hello" }] }]
|
||||
|
||||
const result = addAiSdkAnthropicCacheBreakpoints(messages)
|
||||
expect(result).toEqual(messages)
|
||||
})
|
||||
|
||||
it("should add cache breakpoint to a single user message with string content", () => {
|
||||
const messages: ModelMessage[] = [
|
||||
{ role: "user", content: "Hello" },
|
||||
{ role: "assistant", content: [{ type: "text", text: "Hi" }] },
|
||||
]
|
||||
|
||||
const result = addAiSdkAnthropicCacheBreakpoints(messages)
|
||||
|
||||
expect(result[0]).toEqual({
|
||||
role: "user",
|
||||
content: [{ type: "text", text: "Hello", providerOptions: CACHE_CONTROL }],
|
||||
})
|
||||
})
|
||||
|
||||
it("should add cache breakpoints to the last two user messages", () => {
|
||||
const messages: ModelMessage[] = [
|
||||
{ role: "user", content: "First" },
|
||||
{ role: "assistant", content: [{ type: "text", text: "Response 1" }] },
|
||||
{ role: "user", content: "Second" },
|
||||
{ role: "assistant", content: [{ type: "text", text: "Response 2" }] },
|
||||
{ role: "user", content: "Third" },
|
||||
]
|
||||
|
||||
const result = addAiSdkAnthropicCacheBreakpoints(messages)
|
||||
|
||||
// First user message should NOT have cache control
|
||||
expect(result[0]).toEqual({ role: "user", content: "First" })
|
||||
|
||||
// Second user message should have cache control
|
||||
expect(result[2]).toEqual({
|
||||
role: "user",
|
||||
content: [{ type: "text", text: "Second", providerOptions: CACHE_CONTROL }],
|
||||
})
|
||||
|
||||
// Third user message should have cache control
|
||||
expect(result[4]).toEqual({
|
||||
role: "user",
|
||||
content: [{ type: "text", text: "Third", providerOptions: CACHE_CONTROL }],
|
||||
})
|
||||
})
|
||||
|
||||
it("should add cache breakpoint to the last text part of array content", () => {
|
||||
const messages: ModelMessage[] = [
|
||||
{
|
||||
role: "user",
|
||||
content: [
|
||||
{ type: "text", text: "First part" },
|
||||
{ type: "image", image: "data:image/png;base64,..." },
|
||||
{ type: "text", text: "Last text part" },
|
||||
],
|
||||
},
|
||||
]
|
||||
|
||||
const result = addAiSdkAnthropicCacheBreakpoints(messages)
|
||||
|
||||
expect((result[0] as any).content).toEqual([
|
||||
{ type: "text", text: "First part" },
|
||||
{ type: "image", image: "data:image/png;base64,..." },
|
||||
{ type: "text", text: "Last text part", providerOptions: CACHE_CONTROL },
|
||||
])
|
||||
})
|
||||
|
||||
it("should add placeholder text part when no text parts exist in array content", () => {
|
||||
const messages: ModelMessage[] = [
|
||||
{
|
||||
role: "user",
|
||||
content: [{ type: "image", image: "data:image/png;base64,..." }],
|
||||
},
|
||||
]
|
||||
|
||||
const result = addAiSdkAnthropicCacheBreakpoints(messages)
|
||||
|
||||
expect((result[0] as any).content).toEqual([
|
||||
{ type: "image", image: "data:image/png;base64,..." },
|
||||
{ type: "text", text: "...", providerOptions: CACHE_CONTROL },
|
||||
])
|
||||
})
|
||||
|
||||
it("should not mutate the original messages", () => {
|
||||
const messages: ModelMessage[] = [
|
||||
{ role: "user", content: "Hello" },
|
||||
{ role: "assistant", content: [{ type: "text", text: "Hi" }] },
|
||||
]
|
||||
|
||||
const original = JSON.parse(JSON.stringify(messages))
|
||||
addAiSdkAnthropicCacheBreakpoints(messages)
|
||||
|
||||
expect(messages).toEqual(original)
|
||||
})
|
||||
|
||||
it("should handle both user messages when only two exist", () => {
|
||||
const messages: ModelMessage[] = [
|
||||
{ role: "user", content: "First" },
|
||||
{ role: "assistant", content: [{ type: "text", text: "Response" }] },
|
||||
{ role: "user", content: "Second" },
|
||||
]
|
||||
|
||||
const result = addAiSdkAnthropicCacheBreakpoints(messages)
|
||||
|
||||
expect(result[0]).toEqual({
|
||||
role: "user",
|
||||
content: [{ type: "text", text: "First", providerOptions: CACHE_CONTROL }],
|
||||
})
|
||||
expect(result[2]).toEqual({
|
||||
role: "user",
|
||||
content: [{ type: "text", text: "Second", providerOptions: CACHE_CONTROL }],
|
||||
})
|
||||
})
|
||||
|
||||
it("should not modify assistant or tool messages", () => {
|
||||
const assistantMsg: ModelMessage = { role: "assistant", content: [{ type: "text", text: "Response" }] }
|
||||
const toolMsg: ModelMessage = {
|
||||
role: "tool",
|
||||
content: [
|
||||
{
|
||||
type: "tool-result",
|
||||
toolCallId: "call1",
|
||||
toolName: "test",
|
||||
output: { type: "text", value: "result" },
|
||||
},
|
||||
],
|
||||
} as ModelMessage
|
||||
|
||||
const messages: ModelMessage[] = [
|
||||
{ role: "user", content: "Hello" },
|
||||
assistantMsg,
|
||||
toolMsg,
|
||||
{ role: "user", content: "Continue" },
|
||||
]
|
||||
|
||||
const result = addAiSdkAnthropicCacheBreakpoints(messages)
|
||||
|
||||
expect(result[1]).toEqual(assistantMsg)
|
||||
expect(result[2]).toEqual(toolMsg)
|
||||
})
|
||||
})
|
||||
90
src/api/transform/caching/ai-sdk-anthropic.ts
Normal file
90
src/api/transform/caching/ai-sdk-anthropic.ts
Normal file
|
|
@ -0,0 +1,90 @@
|
|||
import type { ModelMessage } from "ai"
|
||||
|
||||
const ANTHROPIC_CACHE_CONTROL = {
|
||||
anthropic: { cacheControl: { type: "ephemeral" } },
|
||||
}
|
||||
|
||||
/**
|
||||
* Add Anthropic cache breakpoints to AI SDK ModelMessage array.
|
||||
* Adds `providerOptions.anthropic.cacheControl` to the last text content part
|
||||
* of the last 2 user messages, enabling prompt caching for Anthropic models
|
||||
* via the AI SDK.
|
||||
*
|
||||
* Note: System prompt caching is handled separately at the streamText call level
|
||||
* by passing the system prompt as an array with providerOptions.
|
||||
*
|
||||
* @param messages - Array of AI SDK ModelMessage objects
|
||||
* @returns New array with cache breakpoints added (does not mutate input)
|
||||
*/
|
||||
export function addAiSdkAnthropicCacheBreakpoints(messages: ModelMessage[]): ModelMessage[] {
|
||||
const userMsgIndices = messages.reduce(
|
||||
(acc, msg, index) => (msg.role === "user" ? [...acc, index] : acc),
|
||||
[] as number[],
|
||||
)
|
||||
|
||||
const targetIndices = new Set(userMsgIndices.slice(-2))
|
||||
|
||||
if (targetIndices.size === 0) {
|
||||
return messages
|
||||
}
|
||||
|
||||
return messages.map((message, index) => {
|
||||
if (!targetIndices.has(index)) {
|
||||
return message
|
||||
}
|
||||
|
||||
if (typeof message.content === "string") {
|
||||
return {
|
||||
...message,
|
||||
content: [
|
||||
{
|
||||
type: "text" as const,
|
||||
text: message.content,
|
||||
providerOptions: ANTHROPIC_CACHE_CONTROL,
|
||||
},
|
||||
],
|
||||
} as ModelMessage
|
||||
}
|
||||
|
||||
if (Array.isArray(message.content)) {
|
||||
// Find the index of the last text part
|
||||
let lastTextIndex = -1
|
||||
for (let i = message.content.length - 1; i >= 0; i--) {
|
||||
if ((message.content[i] as { type: string }).type === "text") {
|
||||
lastTextIndex = i
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if (lastTextIndex === -1) {
|
||||
// No text part found — add a placeholder
|
||||
return {
|
||||
...message,
|
||||
content: [
|
||||
...message.content,
|
||||
{
|
||||
type: "text" as const,
|
||||
text: "...",
|
||||
providerOptions: ANTHROPIC_CACHE_CONTROL,
|
||||
},
|
||||
],
|
||||
} as ModelMessage
|
||||
}
|
||||
|
||||
return {
|
||||
...message,
|
||||
content: message.content.map((part, i) => {
|
||||
if (i === lastTextIndex) {
|
||||
return {
|
||||
...(part as Record<string, unknown>),
|
||||
providerOptions: ANTHROPIC_CACHE_CONTROL,
|
||||
}
|
||||
}
|
||||
return part
|
||||
}),
|
||||
} as ModelMessage
|
||||
}
|
||||
|
||||
return message
|
||||
})
|
||||
}
|
||||
|
|
@ -450,6 +450,7 @@
|
|||
"clean": "rimraf README.md CHANGELOG.md LICENSE dist logs mock .turbo"
|
||||
},
|
||||
"dependencies": {
|
||||
"@ai-sdk/anthropic": "^3.0.37",
|
||||
"@ai-sdk/cerebras": "^1.0.0",
|
||||
"@ai-sdk/deepseek": "^2.0.14",
|
||||
"@ai-sdk/fireworks": "^2.0.26",
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue