diff --git a/.changeset/warm-pans-perform.md b/.changeset/warm-pans-perform.md new file mode 100644 index 0000000000..bb62d72c6b --- /dev/null +++ b/.changeset/warm-pans-perform.md @@ -0,0 +1,5 @@ +--- +"claude-dev": minor +--- + +Better enterprise support for providers (Bedrock, Vertex) diff --git a/package-lock.json b/package-lock.json index 9d6304649e..19fc5a334a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,8 +9,8 @@ "version": "3.4.9", "license": "Apache-2.0", "dependencies": { - "@anthropic-ai/bedrock-sdk": "^0.10.2", - "@anthropic-ai/sdk": "^0.26.0", + "@anthropic-ai/bedrock-sdk": "^0.12.4", + "@anthropic-ai/sdk": "^0.36.3", "@anthropic-ai/vertex-sdk": "^0.6.4", "@google/generative-ai": "^0.18.0", "@mistralai/mistralai": "^1.5.0", @@ -78,11 +78,12 @@ } }, "node_modules/@anthropic-ai/bedrock-sdk": { - "version": "0.10.2", - "resolved": "https://registry.npmjs.org/@anthropic-ai/bedrock-sdk/-/bedrock-sdk-0.10.2.tgz", - "integrity": "sha512-sGmTzKJQHVwfXexe+yfzPU3rJmUMCygC+GNPkmMsPX/Jr+WKtJ0M71nGyHONr6vcHwUpUWA6o0MRH/oHaE54KA==", + "version": "0.12.4", + "resolved": "https://registry.npmjs.org/@anthropic-ai/bedrock-sdk/-/bedrock-sdk-0.12.4.tgz", + "integrity": "sha512-kraOgWWyVO/Wef3wYbpws77pCZubg/hCzXQ7RGrLRJsRpbTIT+ms+MigGu/0b1qn3o5WuIrumlT3Qtu3esHpzw==", + "license": "MIT", "dependencies": { - "@anthropic-ai/sdk": "^0", + "@anthropic-ai/sdk": ">=0.36 <1", "@aws-crypto/sha256-js": "^4.0.0", "@aws-sdk/client-bedrock-runtime": "^3.423.0", "@aws-sdk/credential-providers": "^3.341.0", @@ -96,9 +97,10 @@ } }, "node_modules/@anthropic-ai/sdk": { - "version": "0.26.0", - "resolved": "https://registry.npmjs.org/@anthropic-ai/sdk/-/sdk-0.26.0.tgz", - "integrity": "sha512-vNbZ2rnnMfk8Bf4OdeVy6GA4EXao8tGC0tLEoSAl1NZrip9oOxnEGUkXl3FsPQgeBM5hmpGE1tSLuu9HEVJiHg==", + "version": "0.36.3", + "resolved": "https://registry.npmjs.org/@anthropic-ai/sdk/-/sdk-0.36.3.tgz", + "integrity": "sha512-+c0mMLxL/17yFZ4P5+U6bTWiCSFZUKJddrv01ud2aFBWnTPLdRncYV76D3q1tqfnL7aCnhRtykFnoCFzvr4U3Q==", + "license": "MIT", "dependencies": { "@types/node": "^18.11.18", "@types/node-fetch": "^2.6.4", diff --git a/package.json b/package.json index 64c5ce2b5e..12183e8680 100644 --- a/package.json +++ b/package.json @@ -262,9 +262,9 @@ "typescript": "^5.4.5" }, "dependencies": { - "@anthropic-ai/bedrock-sdk": "^0.10.2", - "@anthropic-ai/sdk": "^0.26.0", "@anthropic-ai/vertex-sdk": "^0.6.4", + "@anthropic-ai/bedrock-sdk": "^0.12.4", + "@anthropic-ai/sdk": "^0.36.3", "@google/generative-ai": "^0.18.0", "@mistralai/mistralai": "^1.5.0", "@modelcontextprotocol/sdk": "^1.0.1", @@ -305,4 +305,4 @@ "web-tree-sitter": "^0.22.6", "zod": "^3.24.2" } -} +} \ No newline at end of file diff --git a/src/api/index.ts b/src/api/index.ts index 4681ec03af..b9ce780e9c 100644 --- a/src/api/index.ts +++ b/src/api/index.ts @@ -18,8 +18,8 @@ import { MistralHandler } from "./providers/mistral" import { VsCodeLmHandler } from "./providers/vscode-lm" import { LiteLlmHandler } from "./providers/litellm" -export interface ApiHandler { - createMessage(systemPrompt: string, messages: Anthropic.Messages.MessageParam[]): ApiStream +export interface ApiHandler { + createMessage(systemPrompt: string, messages: MessageType[]): ApiStream getModel(): { id: string; info: ModelInfo } } diff --git a/src/api/providers/anthropic.ts b/src/api/providers/anthropic.ts index 959916e9a8..55840528b7 100644 --- a/src/api/providers/anthropic.ts +++ b/src/api/providers/anthropic.ts @@ -1,185 +1,74 @@ import { Anthropic } from "@anthropic-ai/sdk" import { Stream as AnthropicStream } from "@anthropic-ai/sdk/streaming" -import { withRetry } from "../retry" -import { anthropicDefaultModelId, AnthropicModelId, anthropicModels, ApiHandlerOptions, ModelInfo } from "../../shared/api" -import { ApiHandler } from "../index" +import { anthropicDefaultModelId, AnthropicModelId, anthropicModels, ModelInfo } from "../../shared/api" import { ApiStream } from "../transform/stream" +import { ClaudeStreamingHandler } from "./claude-streaming" -export class AnthropicHandler implements ApiHandler { - private options: ApiHandlerOptions - private client: Anthropic - - constructor(options: ApiHandlerOptions) { - this.options = options - this.client = new Anthropic({ +/** + * Handles interactions with the Anthropic service. + */ +export class AnthropicHandler extends ClaudeStreamingHandler { + getClient() { + return new Anthropic({ apiKey: this.options.apiKey, - baseURL: this.options.anthropicBaseUrl || undefined, + baseURL: this.options.anthropicBaseUrl || null, // default baseURL: https://api.anthropic.com }) } - @withRetry() - async *createMessage(systemPrompt: string, messages: Anthropic.Messages.MessageParam[]): ApiStream { + override async *createStreamingMessage(systemPrompt: string, messages: Anthropic.Messages.MessageParam[]): ApiStream { const model = this.getModel() - let stream: AnthropicStream const modelId = model.id - switch (modelId) { - // 'latest' alias does not support cache_control - case "claude-3-7-sonnet-20250219": - case "claude-3-5-sonnet-20241022": - case "claude-3-5-haiku-20241022": - case "claude-3-opus-20240229": - case "claude-3-haiku-20240307": { - /* - The latest message will be the new user message, one before will be the assistant message from a previous request, and the user message before that will be a previously cached user message. So we need to mark the latest user message as ephemeral to cache it for the next request, and mark the second to last user message as ephemeral to let the server know the last message to retrieve from the cache for the current request.. - */ - const userMsgIndices = messages.reduce( - (acc, msg, index) => (msg.role === "user" ? [...acc, index] : acc), - [] as number[], - ) - const lastUserMsgIndex = userMsgIndices[userMsgIndices.length - 1] ?? -1 - const secondLastMsgUserIndex = userMsgIndices[userMsgIndices.length - 2] ?? -1 - stream = await this.client.beta.promptCaching.messages.create( - { - model: modelId, - max_tokens: model.info.maxTokens || 8192, - temperature: 0, - system: [ - { - text: systemPrompt, - type: "text", - cache_control: { type: "ephemeral" }, - }, - ], // setting cache breakpoint for system prompt so new tasks can reuse it - messages: messages.map((message, index) => { - if (index === lastUserMsgIndex || index === secondLastMsgUserIndex) { - return { - ...message, - content: - typeof message.content === "string" - ? [ - { - type: "text", - text: message.content, - cache_control: { - type: "ephemeral", - }, - }, - ] - : message.content.map((content, contentIndex) => - contentIndex === message.content.length - 1 - ? { - ...content, - cache_control: { - type: "ephemeral", - }, - } - : content, - ), - } - } - return message - }), - // tools, // cache breakpoints go from tools > system > messages, and since tools dont change, we can just set the breakpoint at the end of system (this avoids having to set a breakpoint at the end of tools which by itself does not meet min requirements for haiku caching) - // tool_choice: { type: "auto" }, - // tools: tools, - stream: true, - }, - (() => { - // prompt caching: https://x.com/alexalbert__/status/1823751995901272068 - // https://github.com/anthropics/anthropic-sdk-typescript?tab=readme-ov-file#default-headers - // https://github.com/anthropics/anthropic-sdk-typescript/commit/c920b77fc67bd839bfeb6716ceab9d7c9bbe7393 - switch (modelId) { - case "claude-3-7-sonnet-20250219": - case "claude-3-5-sonnet-20241022": - case "claude-3-5-haiku-20241022": - case "claude-3-opus-20240229": - case "claude-3-haiku-20240307": - return { - headers: { - "anthropic-beta": "prompt-caching-2024-07-31", - }, - } - default: - return undefined - } - })(), - ) - break - } - default: { - stream = (await this.client.messages.create({ - model: modelId, - max_tokens: model.info.maxTokens || 8192, - temperature: 0, - system: [{ text: systemPrompt, type: "text" }], - messages, - // tools, - // tool_choice: { type: "auto" }, - stream: true, - })) as any - break - } + + let stream: AnthropicStream + + if (Object.keys(anthropicModels).includes(modelId)) { + stream = await this.createModelStream( + systemPrompt, + messages, + modelId, + model.info.maxTokens ?? AnthropicHandler.DEFAULT_TOKEN_SIZE, + ) + } else { + throw new Error(`Invalid model ID: ${modelId}`) } - for await (const chunk of stream) { - switch (chunk.type) { - case "message_start": - // tells us cache reads/writes/input/output - const usage = chunk.message.usage - yield { - type: "usage", - inputTokens: usage.input_tokens || 0, - outputTokens: usage.output_tokens || 0, - cacheWriteTokens: usage.cache_creation_input_tokens || undefined, - cacheReadTokens: usage.cache_read_input_tokens || undefined, - } - break - case "message_delta": - // tells us stop_reason, stop_sequence, and output tokens along the way and at the end of the message - - yield { - type: "usage", - inputTokens: 0, - outputTokens: chunk.usage.output_tokens || 0, - } - break - case "message_stop": - // no usage data, just an indicator that the message is done - break - case "content_block_start": - switch (chunk.content_block.type) { - case "text": - // we may receive multiple text blocks, in which case just insert a line break between them - if (chunk.index > 0) { - yield { - type: "text", - text: "\n", - } - } - yield { - type: "text", - text: chunk.content_block.text, - } - break - } - break - case "content_block_delta": - switch (chunk.delta.type) { - case "text_delta": - yield { - type: "text", - text: chunk.delta.text, - } - break - } - break - case "content_block_stop": - break - } - } + yield* this.processStream(stream) } - getModel(): { id: AnthropicModelId; info: ModelInfo } { + override async createModelStream( + systemPrompt: string, + messages: Anthropic.Messages.MessageParam[], + modelId: string, + maxTokens: number, + ): Promise> { + /* + The latest message will be the new user message, one before will be the assistant message from a previous request, and the user message before that will be a previously cached user message. So we need to mark the latest user message as ephemeral to cache it for the next request, and mark the second to last user message as ephemeral to let the server know the last message to retrieve from the cache for the current request.. + */ + const userMsgIndices = messages.reduce((acc, msg, index) => (msg.role === "user" ? [...acc, index] : acc), [] as number[]) + const lastUserMsgIndex = userMsgIndices[userMsgIndices.length - 1] ?? -1 + const secondLastMsgUserIndex = userMsgIndices[userMsgIndices.length - 2] ?? -1 + return await this.client.messages.create({ + model: modelId, + max_tokens: maxTokens || AnthropicHandler.DEFAULT_TOKEN_SIZE, + temperature: AnthropicHandler.DEFAULT_TEMPERATURE, + system: [ + { + text: systemPrompt, + type: "text", + cache_control: { type: "ephemeral" }, + }, + ], // setting cache breakpoint for system prompt so new tasks can reuse it + messages: messages.map((message, index) => + this.transformMessage(message, index, lastUserMsgIndex, secondLastMsgUserIndex), + ), + // tools, // cache breakpoints go from tools > system > messages, and since tools dont change, we can just set the breakpoint at the end of system (this avoids having to set a breakpoint at the end of tools which by itself does not meet min requirements for haiku caching) + // tool_choice: { type: "auto" }, + // tools: tools, + stream: true, + }) + } + + override getModel(): { id: AnthropicModelId; info: ModelInfo } { const modelId = this.options.apiModelId if (modelId && modelId in anthropicModels) { const id = modelId as AnthropicModelId diff --git a/src/api/providers/bedrock.ts b/src/api/providers/bedrock.ts index 75a0c980b1..72fe097769 100644 --- a/src/api/providers/bedrock.ts +++ b/src/api/providers/bedrock.ts @@ -1,100 +1,128 @@ import AnthropicBedrock from "@anthropic-ai/bedrock-sdk" import { Anthropic } from "@anthropic-ai/sdk" -import { ApiHandler } from "../" -import { ApiHandlerOptions, bedrockDefaultModelId, BedrockModelId, bedrockModels, ModelInfo } from "../../shared/api" +import { Stream as AnthropicStream } from "@anthropic-ai/sdk/streaming" +import { bedrockDefaultModelId, BedrockModelId, bedrockModels, ModelInfo } from "../../shared/api" import { ApiStream } from "../transform/stream" -import { fromNodeProviderChain } from "@aws-sdk/credential-providers" +import { fromIni, fromNodeProviderChain } from "@aws-sdk/credential-providers" +import { ClaudeStreamingHandler } from "./claude-streaming" -// https://docs.anthropic.com/en/api/claude-on-amazon-bedrock -export class AwsBedrockHandler implements ApiHandler { - private options: ApiHandlerOptions +/** + * Handles interactions with the Anthropic Bedrock service using AWS credentials. + */ +export class AwsBedrockHandler extends ClaudeStreamingHandler { + async getClient() { + const clientConfig: any = { + awsRegion: this.options.awsRegion || "us-west-2", + } - constructor(options: ApiHandlerOptions) { - this.options = options + try { + this.saveClientToNodeProviderChain() + // Use AWS profile credentials if specified. + if (this.options.awsUseProfile) { + const credentials = await fromIni({ + profile: this.options.awsProfile || "default", + ignoreCache: true, + })() + clientConfig.awsAccessKey = credentials.accessKeyId + clientConfig.awsSecretKey = credentials.secretAccessKey + clientConfig.awsSessionToken = credentials.sessionToken + } + // Use provided AWS access key and secret key if specified. + else if (this.options.awsAccessKey && this.options.awsSecretKey) { + clientConfig.awsAccessKey = this.options.awsAccessKey + clientConfig.awsSecretKey = this.options.awsSecretKey + if (this.options.awsSessionToken) { + clientConfig.awsSessionToken = this.options.awsSessionToken + } + } + } catch (error) { + console.error("Failed to initialize Bedrock client:", error) + throw error + } + return new AnthropicBedrock(clientConfig) } - async *createMessage(systemPrompt: string, messages: Anthropic.Messages.MessageParam[]): ApiStream { - // cross region inference requires prefixing the model id with the region - let modelId = await this.getModelId() + async *createStreamingMessage(systemPrompt: string, messages: Anthropic.Messages.MessageParam[]): ApiStream { + const model = this.getModel() + const modelId = this.getModelId() - // create anthropic client, using sessions created or renewed after this handler's - // initialization, and allowing for session renewal if necessary as well - let client = await this.getClient() + let stream: AnthropicStream - const stream = await client.messages.create({ + if (Object.keys(bedrockModels).includes(modelId)) { + stream = await this.createModelStream( + systemPrompt, + messages, + modelId, + model.info.maxTokens ?? ClaudeStreamingHandler.DEFAULT_TOKEN_SIZE, + ) + } else { + stream = await this.client.messages.create({ + model: modelId, + max_tokens: model.info.maxTokens || ClaudeStreamingHandler.DEFAULT_TOKEN_SIZE, + temperature: ClaudeStreamingHandler.DEFAULT_TEMPERATURE, + system: systemPrompt, + messages, + stream: true, + }) + } + + yield* this.processStream(stream) + } + + async createModelStream( + systemPrompt: string, + messages: Anthropic.Messages.MessageParam[], + modelId: string, + maxTokens: number, + ): Promise> { + const userMsgIndices = messages.reduce((acc, msg, index) => (msg.role === "user" ? [...acc, index] : acc), [] as number[]) + + return await this.client.messages.create({ model: modelId, - max_tokens: this.getModel().info.maxTokens || 8192, - temperature: 0, - system: systemPrompt, + max_tokens: maxTokens || ClaudeStreamingHandler.DEFAULT_TOKEN_SIZE, + temperature: ClaudeStreamingHandler.DEFAULT_TEMPERATURE, + system: [ + { + text: systemPrompt, + type: "text", + cache_control: { type: "ephemeral" }, + }, + ], // setting cache breakpoint for system prompt so new tasks can reuse it messages, stream: true, }) - for await (const chunk of stream) { - switch (chunk.type) { - case "message_start": - const usage = chunk.message.usage - yield { - type: "usage", - inputTokens: usage.input_tokens || 0, - outputTokens: usage.output_tokens || 0, - } - break - case "message_delta": - yield { - type: "usage", - inputTokens: 0, - outputTokens: chunk.usage.output_tokens || 0, - } - break + } - case "content_block_start": - switch (chunk.content_block.type) { - case "text": - if (chunk.index > 0) { - yield { - type: "text", - text: "\n", - } - } - yield { - type: "text", - text: chunk.content_block.text, - } - break - } - break - case "content_block_delta": - switch (chunk.delta.type) { - case "text_delta": - yield { - type: "text", - text: chunk.delta.text, - } - break - } - break + protected getModelId(): string { + if (this.options.awsUseCrossRegionInference) { + const regionPrefix = (this.options.awsRegion || "").slice(0, 3) + switch (regionPrefix) { + case "us-": + return `us.${this.getModel().id}` + case "eu-": + return `eu.${this.getModel().id}` + default: + return this.getModel().id } } + return this.getModel().id } getModel(): { id: BedrockModelId; info: ModelInfo } { const modelId = this.options.apiModelId if (modelId && modelId in bedrockModels) { - const id = modelId as BedrockModelId - return { id, info: bedrockModels[id] } - } - return { - id: bedrockDefaultModelId, - info: bedrockModels[bedrockDefaultModelId], + // Return the model information for the specified model ID + return { id: modelId as BedrockModelId, info: bedrockModels[modelId as BedrockModelId] } } + return { id: bedrockDefaultModelId, info: bedrockModels[bedrockDefaultModelId] } } - private async getClient(): Promise { + private async saveClientToNodeProviderChain() { // Create AWS credentials by executing a an AWS provider chain exactly as the // Anthropic SDK does it, by wrapping the default chain into a temporary process // environment. const providerChain = fromNodeProviderChain() - const credentials = await AwsBedrockHandler.withTempEnv( + await AwsBedrockHandler.withTempEnv( () => { AwsBedrockHandler.setEnv("AWS_REGION", this.options.awsRegion) AwsBedrockHandler.setEnv("AWS_ACCESS_KEY_ID", this.options.awsAccessKey) @@ -104,36 +132,6 @@ export class AwsBedrockHandler implements ApiHandler { }, () => providerChain(), ) - - // Return an AnthropicBedrock client with the resolved/assumed credentials. - // - // When AnthropicBedrock creates its AWS client, the chain will execute very - // fast as the access/secret keys will already be already provided, and have - // a higher precedence than the profiles. - return new AnthropicBedrock({ - awsAccessKey: credentials.accessKeyId, - awsSecretKey: credentials.secretAccessKey, - awsSessionToken: credentials.sessionToken, - awsRegion: this.options.awsRegion || "us-east-1", - }) - } - - private async getModelId(): Promise { - if (this.options.awsUseCrossRegionInference) { - let regionPrefix = (this.options.awsRegion || "").slice(0, 3) - switch (regionPrefix) { - case "us-": - return `us.${this.getModel().id}` - case "eu-": - return `eu.${this.getModel().id}` - break - default: - // cross region inference is not supported in this region, falling back to default model - return this.getModel().id - break - } - } - return this.getModel().id } private static async withTempEnv(updateEnv: () => void, fn: () => Promise): Promise { diff --git a/src/api/providers/claude-streaming.ts b/src/api/providers/claude-streaming.ts new file mode 100644 index 0000000000..b7ca7337af --- /dev/null +++ b/src/api/providers/claude-streaming.ts @@ -0,0 +1,211 @@ +import { Anthropic } from "@anthropic-ai/sdk" +import { ApiHandler } from "../" +import { ApiHandlerOptions, ModelInfo } from "../../shared/api" +import { ApiStream } from "../transform/stream" +import { withRetry } from "../retry" +import { Stream as AnthropicStream } from "@anthropic-ai/sdk/streaming" +import { AnthropicVertex } from "@anthropic-ai/vertex-sdk" +import AnthropicBedrock from "@anthropic-ai/bedrock-sdk" + +/** + * Abstract base class for Claude-based streaming providers. + * + * This class provides a standardized framework for handling Claude-based API interactions, + * ensuring consistency and reusability. It enforces a contract for subclasses to implement + * specific methods, promoting a clear and maintainable architecture. + * + * The use of generators and yielding allows efficient handling of asynchronous data streams, + * enabling real-time data processing without blocking the main thread. + * + * Generics provide flexibility and type safety for different enterprise clients. + * + * Caching improves performance by storing frequently accessed data, reducing latency and load + * on external services, and optimizing data handling. + * + * @template ClientType - The type of client. Must be one of Anthropic, AnthropicVertex, or AnthropicBedrock. + * @implements ApiHandler + */ + +export abstract class ClaudeStreamingHandler + implements ApiHandler +{ + static readonly DEFAULT_TOKEN_SIZE: number = 8192 // The default token size for message generation. + static readonly DEFAULT_TEMPERATURE: number = 0 // The default temperature for message generation. + protected options: ApiHandlerOptions // The options for the handler. + protected cache: Map // A cache of message streams. + protected client!: ClientType // The client. + + /** + * Creates a new handler. + * @param options - The options for the handler. + */ + constructor(options: ApiHandlerOptions) { + this.options = options + this.cache = new Map() + this._initialize() + } + + /** + * Initializes the client. + */ + private _initialize() { + const client = this.getClient() + if (client instanceof Promise) { + client + .then((resolvedClient) => { + this.client = resolvedClient + }) + .catch((error) => { + throw new Error("Failed to initialize client: " + error) + }) + } else { + this.client = client + } + } + + /** + * Creates a message stream to a Claude model. + * @param systemPrompt - The system prompt to initialize the conversation. + * @param messages - An array of message parameters. + * @returns An asynchronous generator yielding ApiStream events. + */ + protected abstract createModelStream( + systemPrompt: string, + messages: Anthropic.Messages.MessageParam[], + modelId: string, + maxTokens: number, + ): Promise> + + /** + * Initializes the client. + * This method must be implemented by subclasses. + */ + protected abstract getClient(): ClientType | Promise + + /** + * Creates a message stream. + * @param systemPrompt - The system prompt to initialize the conversation. + * @param messages - An array of message parameters. + * @returns An asynchronous generator yielding ApiStream events. + */ + @withRetry() + async *createMessage(systemPrompt: string, messages: Anthropic.Messages.MessageParam[]): ApiStream { + try { + yield* this.createStreamingMessage(systemPrompt, messages) + } catch (error) { + this.handleMessageStreamError(error) + } + } + + /** + * Handles an error that occurs during message stream creation. Override to handle provider-specific errors. + * @param error - The error that threw during message stream creation. + * @throws An error with a message indicating the failure. + */ + protected handleMessageStreamError(error: any) { + throw new Error(`Failed to create message stream: ${error instanceof Error ? error.message : "Unknown error"}`) + } + + /** + * Creates a streaming message stream. + * This method must be implemented by subclasses. + * @param systemPrompt - The system prompt to initialize the conversation. + * @param messages - An array of message parameters. + * @returns An asynchronous generator yielding ApiStream events. + */ + protected abstract createStreamingMessage(systemPrompt: string, messages: Anthropic.Messages.MessageParam[]): ApiStream + + /** + * Processes a stream of raw message events. + * @param stream - A stream of raw message events. + * @returns An asynchronous generator yielding ApiStream events. + */ + protected async *processStream(stream: AnthropicStream): ApiStream { + for await (const chunk of stream) { + yield* this.processChunk(chunk) + } + } + + /** + * Processes each chunk of the stream and yields the appropriate ApiStream events. + * @param chunk - The chunk of data received from the stream. + * @returns An asynchronous generator yielding ApiStream events. + */ + protected async *processChunk(chunk: any): ApiStream { + switch (chunk.type) { + case "message_start": + // tells us cache reads/writes/input/output + yield { + type: "usage", + inputTokens: chunk.message.usage.input_tokens || 0, + outputTokens: chunk.message.usage.output_tokens || 0, + cacheWriteTokens: chunk.message.usage.cache_creation_input_tokens || undefined, + cacheReadTokens: chunk.message.usage.cache_read_input_tokens || undefined, + } + break + case "message_delta": + // tells us stop_reason, stop_sequence, and output tokens along the way and at the end of the message + + yield { + type: "usage", + inputTokens: 0, + outputTokens: chunk.usage.output_tokens || 0, + } + break + case "message_stop": + // no usage data, just an indicator that the message is done + break + case "content_block_start": + if (chunk.content_block.type === "text") { + // we may receive multiple text blocks, in which case just insert a line break between them + if (chunk.index > 0) { + yield { type: "text", text: "\n" } + } + yield { type: "text", text: chunk.content_block.text } + } + break + case "content_block_delta": + if (chunk.delta.type === "text_delta") { + yield { type: "text", text: chunk.delta.text } + } + break + } + } + + /** + * Transforms a message based on its index and user message indices. + * @param message - The message to transform. + * @param index - The index of the message in the array. + * @param lastUserMsgIndex - The index of the last user message. + * @param secondLastMsgUserIndex - The index of the second last user message. + * @returns The transformed message. + */ + protected transformMessage( + message: Anthropic.Messages.MessageParam, + index: number, + lastUserMsgIndex: number, + secondLastMsgUserIndex: number, + ): Anthropic.Messages.MessageParam { + if (index === lastUserMsgIndex || index === secondLastMsgUserIndex) { + return { + ...message, + content: + typeof message.content === "string" + ? [{ type: "text", text: message.content, cache_control: { type: "ephemeral" } }] + : message.content.map((content, contentIndex) => + contentIndex === message.content.length - 1 + ? { ...content, cache_control: { type: "ephemeral" } } + : content, + ), + } + } + return message + } + + /** + * Gets the model ID and info for the handler. + * This method must be implemented by subclasses. + * @returns The model ID and info. + */ + abstract getModel(): { id: string; info: ModelInfo } +} diff --git a/src/api/providers/vertex.ts b/src/api/providers/vertex.ts index 314095ac3f..c692b2ab4e 100644 --- a/src/api/providers/vertex.ts +++ b/src/api/providers/vertex.ts @@ -1,186 +1,88 @@ import { Anthropic } from "@anthropic-ai/sdk" import { AnthropicVertex } from "@anthropic-ai/vertex-sdk" -import { withRetry } from "../retry" -import { ApiHandler } from "../" -import { ApiHandlerOptions, ModelInfo, vertexDefaultModelId, VertexModelId, vertexModels } from "../../shared/api" +import { Stream as AnthropicStream } from "@anthropic-ai/sdk/streaming" +import { ClaudeStreamingHandler } from "./claude-streaming" +import { ModelInfo, vertexDefaultModelId, VertexModelId, vertexModels } from "../../shared/api" import { ApiStream } from "../transform/stream" -// https://docs.anthropic.com/en/api/claude-on-vertex-ai -export class VertexHandler implements ApiHandler { - private options: ApiHandlerOptions - private client: AnthropicVertex - - constructor(options: ApiHandlerOptions) { - this.options = options - this.client = new AnthropicVertex({ +/** + * Handles interactions with the Anthropic Vertex service. + */ +export class VertexHandler extends ClaudeStreamingHandler { + getClient() { + return new AnthropicVertex({ projectId: this.options.vertexProjectId, - // https://cloud.google.com/vertex-ai/generative-ai/docs/partner-models/use-claude#regions region: this.options.vertexRegion, }) } - @withRetry() - async *createMessage(systemPrompt: string, messages: Anthropic.Messages.MessageParam[]): ApiStream { + async *createStreamingMessage(systemPrompt: string, messages: Anthropic.Messages.MessageParam[]): ApiStream { const model = this.getModel() const modelId = model.id + let stream: AnthropicStream - let stream - switch (modelId) { - case "claude-3-7-sonnet@20250219": - case "claude-3-5-sonnet-v2@20241022": - case "claude-3-5-sonnet@20240620": - case "claude-3-5-haiku@20241022": - case "claude-3-opus@20240229": - case "claude-3-haiku@20240307": { - // Find indices of user messages for cache control - const userMsgIndices = messages.reduce( - (acc, msg, index) => (msg.role === "user" ? [...acc, index] : acc), - [] as number[], - ) - const lastUserMsgIndex = userMsgIndices[userMsgIndices.length - 1] ?? -1 - const secondLastMsgUserIndex = userMsgIndices[userMsgIndices.length - 2] ?? -1 + if (Object.keys(vertexModels).includes(modelId)) { + stream = await this.createModelStream( + systemPrompt, + messages, + modelId, + model.info.maxTokens ?? ClaudeStreamingHandler.DEFAULT_TOKEN_SIZE, + ) + } else { + stream = this.client.messages.create({ + model: modelId, + max_tokens: model.info.maxTokens || ClaudeStreamingHandler.DEFAULT_TOKEN_SIZE, + temperature: ClaudeStreamingHandler.DEFAULT_TEMPERATURE, + system: [{ text: systemPrompt, type: "text" }], + messages, + stream: true, + }) as any + } - stream = await this.client.beta.messages.create( - { - model: modelId, - max_tokens: model.info.maxTokens || 8192, - temperature: 0, - system: [ - { - text: systemPrompt, - type: "text", - cache_control: { type: "ephemeral" }, - }, - ], - messages: messages.map((message, index) => { - if (index === lastUserMsgIndex || index === secondLastMsgUserIndex) { - return { - ...message, - content: - typeof message.content === "string" - ? [ - { - type: "text", - text: message.content, - cache_control: { - type: "ephemeral", - }, - }, - ] - : message.content.map((content, contentIndex) => - contentIndex === message.content.length - 1 - ? { - ...content, - cache_control: { - type: "ephemeral", - }, - } - : content, - ), - } - } - return { - ...message, - content: - typeof message.content === "string" - ? [ - { - type: "text", - text: message.content, - }, - ] - : message.content, - } - }), - stream: true, - }, - { - headers: {}, - }, - ) - break - } - default: { - stream = await this.client.beta.messages.create({ - model: modelId, - max_tokens: model.info.maxTokens || 8192, - temperature: 0, - system: [ - { - text: systemPrompt, - type: "text", - }, - ], - messages: messages.map((message) => ({ - ...message, - content: - typeof message.content === "string" - ? [ - { - type: "text", - text: message.content, - }, - ] - : message.content, - })), - stream: true, - }) - break - } - } - for await (const chunk of stream) { - switch (chunk.type) { - case "message_start": - const usage = chunk.message.usage - yield { - type: "usage", - inputTokens: usage.input_tokens || 0, - outputTokens: usage.output_tokens || 0, - cacheWriteTokens: usage.cache_creation_input_tokens || undefined, - cacheReadTokens: usage.cache_read_input_tokens || undefined, - } - break - case "message_delta": - yield { - type: "usage", - inputTokens: 0, - outputTokens: chunk.usage.output_tokens || 0, - } - break - case "message_stop": - break - case "content_block_start": - switch (chunk.content_block.type) { - case "text": - if (chunk.index > 0) { - yield { - type: "text", - text: "\n", - } - } - yield { - type: "text", - text: chunk.content_block.text, - } - break - } - break - case "content_block_delta": - switch (chunk.delta.type) { - case "text_delta": - yield { - type: "text", - text: chunk.delta.text, - } - break - } - break - case "content_block_stop": - break - } + yield* this.processStream(stream) + } + + protected override handleMessageStreamError(error: any): void { + if (error.error === "invalid_grant" && error.error_subtype === "invalid_rapt") { + // Handle reauthentication-related error (invalid_rapt) + console.error("Reauthentication-related error (invalid_rapt): ", error.error_description) + console.info( + `To resolve this issue, please visit the following link for reauthentication instructions: ${error.error_uri}, or execute \`gcloud auth application-default login\`.`, + ) + throw new Error(`Invalid grant: Please visit ${error.error_uri} or reauthenticate via the gcloud CLI to proceed.`) + } else { + super.handleMessageStreamError(error) } } + async createModelStream( + systemPrompt: string, + messages: Anthropic.Messages.MessageParam[], + modelId: string, + maxTokens: number, + ): Promise> { + const userMsgIndices = messages.reduce((acc, msg, index) => (msg.role === "user" ? [...acc, index] : acc), [] as number[]) + const lastUserMsgIndex = userMsgIndices[userMsgIndices.length - 1] ?? -1 + const secondLastMsgUserIndex = userMsgIndices[userMsgIndices.length - 2] ?? -1 + + return this.client.messages.create({ + model: modelId, + max_tokens: maxTokens || ClaudeStreamingHandler.DEFAULT_TOKEN_SIZE, + temperature: ClaudeStreamingHandler.DEFAULT_TEMPERATURE, + system: [ + { + text: systemPrompt, + type: "text", + cache_control: { type: "ephemeral" }, + }, + ], // setting cache breakpoint for system prompt so new tasks can reuse it + messages: messages.map((message, index) => + this.transformMessage(message, index, lastUserMsgIndex, secondLastMsgUserIndex), + ), + stream: true, + }) + } + getModel(): { id: VertexModelId; info: ModelInfo } { const modelId = this.options.apiModelId if (modelId && modelId in vertexModels) { diff --git a/src/api/transform/gemini-format.ts b/src/api/transform/gemini-format.ts index 0a783dc183..0e20fc58f6 100644 --- a/src/api/transform/gemini-format.ts +++ b/src/api/transform/gemini-format.ts @@ -19,6 +19,7 @@ export function convertAnthropicContentToGemini( | Anthropic.Messages.ImageBlockParam | Anthropic.Messages.ToolUseBlockParam | Anthropic.Messages.ToolResultBlockParam + | Anthropic.Messages.DocumentBlockParam >, ): Part[] { if (typeof content === "string") { @@ -38,6 +39,16 @@ export function convertAnthropicContentToGemini( mimeType: block.source.media_type, }, } as InlineDataPart + case "document": + if (block.source.type !== "base64") { + throw new Error("Unsupported document source type") + } + return { + inlineData: { + data: block.source.data, + mimeType: block.source.media_type, + }, + } as InlineDataPart case "tool_use": return { functionCall: { @@ -133,7 +144,7 @@ export function convertGeminiResponseToAnthropic(response: EnhancedGenerateConte // Add the main text response const text = response.text() if (text) { - content.push({ type: "text", text }) + content.push({ type: "text", text, citations: [] }) } // Add function calls as tool_use blocks @@ -183,6 +194,8 @@ export function convertGeminiResponseToAnthropic(response: EnhancedGenerateConte usage: { input_tokens: response.usageMetadata?.promptTokenCount ?? 0, output_tokens: response.usageMetadata?.candidatesTokenCount ?? 0, + cache_creation_input_tokens: null, + cache_read_input_tokens: null, }, } } diff --git a/src/api/transform/o1-format.ts b/src/api/transform/o1-format.ts index b7577cf903..d338bb9ce1 100644 --- a/src/api/transform/o1-format.ts +++ b/src/api/transform/o1-format.ts @@ -376,6 +376,7 @@ export function convertO1ResponseToAnthropicMessage( { type: "text", text: normalText, + citations: [], }, ], model: completion.model, @@ -396,6 +397,8 @@ export function convertO1ResponseToAnthropicMessage( usage: { input_tokens: completion.usage?.prompt_tokens || 0, output_tokens: completion.usage?.completion_tokens || 0, + cache_creation_input_tokens: null, + cache_read_input_tokens: null, }, } diff --git a/src/api/transform/openai-format.ts b/src/api/transform/openai-format.ts index 0b134ad4ac..68d8ff50f1 100644 --- a/src/api/transform/openai-format.ts +++ b/src/api/transform/openai-format.ts @@ -161,6 +161,7 @@ export function convertToAnthropicMessage(completion: OpenAI.Chat.Completions.Ch { type: "text", text: openAiMessage.content || "", + citations: [], // Add an empty array or appropriate citations }, ], model: completion.model, @@ -181,6 +182,8 @@ export function convertToAnthropicMessage(completion: OpenAI.Chat.Completions.Ch usage: { input_tokens: completion.usage?.prompt_tokens || 0, output_tokens: completion.usage?.completion_tokens || 0, + cache_creation_input_tokens: null, + cache_read_input_tokens: null, }, } diff --git a/src/api/transform/vscode-lm-format.ts b/src/api/transform/vscode-lm-format.ts index acec3656e1..69992ca7e8 100644 --- a/src/api/transform/vscode-lm-format.ts +++ b/src/api/transform/vscode-lm-format.ts @@ -175,6 +175,7 @@ export async function convertToAnthropicMessage( return { type: "text", text: part.value, + citations: [], // Add an empty array or appropriate citations here } } @@ -195,6 +196,8 @@ export async function convertToAnthropicMessage( usage: { input_tokens: 0, output_tokens: 0, + cache_creation_input_tokens: null, + cache_read_input_tokens: null, }, } } diff --git a/src/core/Cline.ts b/src/core/Cline.ts index 7471322133..a5be33b423 100644 --- a/src/core/Cline.ts +++ b/src/core/Cline.ts @@ -59,6 +59,7 @@ import { formatResponse } from "./prompts/responses" import { addUserInstructions, SYSTEM_PROMPT } from "./prompts/system" import { getNextTruncationRange, getTruncatedMessages } from "./sliding-window" import { ClineProvider, GlobalFileNames } from "./webview/ClineProvider" +import { ImageBlockParam, TextBlockParam, ToolResultBlockParam, ToolUseBlockParam } from "@anthropic-ai/sdk/resources/index.mjs" import { DEFAULT_LANGUAGE_SETTINGS, getLanguageKey, LanguageDisplay, LanguageKey } from "../shared/Languages" import { telemetryService } from "../services/telemetry/TelemetryService" @@ -959,7 +960,10 @@ export class Cline { existingApiConversationHistory[existingApiConversationHistory.length - 2] const existingUserContent: UserContent = Array.isArray(lastMessage.content) - ? lastMessage.content + ? lastMessage.content.filter( + (block): block is TextBlockParam | ImageBlockParam | ToolUseBlockParam | ToolResultBlockParam => + block.type !== "document", + ) : [{ type: "text", text: lastMessage.content }] if (previousAssistantMessage && previousAssistantMessage.role === "assistant") { const assistantContent = Array.isArray(previousAssistantMessage.content) diff --git a/src/integrations/misc/export-markdown.ts b/src/integrations/misc/export-markdown.ts index 76e95e36b7..3c8255c441 100644 --- a/src/integrations/misc/export-markdown.ts +++ b/src/integrations/misc/export-markdown.ts @@ -36,13 +36,18 @@ export async function downloadTask(dateTs: number, conversationHistory: Anthropi if (saveUri) { // Write content to the selected location - await vscode.workspace.fs.writeFile(saveUri, Buffer.from(markdownContent)) + await vscode.workspace.fs.writeFile(saveUri, new Uint8Array(Buffer.from(markdownContent))) vscode.window.showTextDocument(saveUri, { preview: true }) } } export function formatContentBlockToMarkdown( - block: Anthropic.TextBlockParam | Anthropic.ImageBlockParam | Anthropic.ToolUseBlockParam | Anthropic.ToolResultBlockParam, + block: + | Anthropic.TextBlockParam + | Anthropic.ImageBlockParam + | Anthropic.ToolUseBlockParam + | Anthropic.ToolResultBlockParam + | Anthropic.DocumentBlockParam, // messages: Anthropic.MessageParam[] ): string { switch (block.type) { diff --git a/src/shared/api.ts b/src/shared/api.ts index fc3d3a7a1f..80bbb928a0 100644 --- a/src/shared/api.ts +++ b/src/shared/api.ts @@ -245,7 +245,7 @@ export const vertexModels = { cacheWritesPrice: 3.75, cacheReadsPrice: 0.3, }, - "claude-3-5-sonnet@20240620": { + "claude-3-sonnet@20240229": { maxTokens: 8192, contextWindow: 200_000, supportsImages: true, diff --git a/webview-ui/src/components/chat/__tests__/Announcement.spec.tsx b/webview-ui/src/components/chat/__tests__/Announcement.spec.tsx index 563a705590..b7c3ab1fcc 100644 --- a/webview-ui/src/components/chat/__tests__/Announcement.spec.tsx +++ b/webview-ui/src/components/chat/__tests__/Announcement.spec.tsx @@ -24,16 +24,16 @@ describe("Announcement", () => { it("renders the mcp server improvements announcement", () => { render() - expect(screen.getByText(/MCP server improvements:/)).toBeInTheDocument() + expect(screen.getByText(/Introducing MCP Marketplace/)).toBeInTheDocument() }) it("renders the 'See new changes' button feature", () => { render() - expect(screen.getByText(/See it in action here./)).toBeInTheDocument() + expect(screen.getByText(/See a demo of the changes here!/)).toBeInTheDocument() }) it("renders the demo link", () => { render() - expect(screen.getByText(/See a demo here./)).toBeInTheDocument() + expect(screen.getByText(/Join us on/)).toBeInTheDocument() }) }) diff --git a/webview-ui/src/components/settings/ApiOptions.tsx b/webview-ui/src/components/settings/ApiOptions.tsx index 4929d43876..73dad3d581 100644 --- a/webview-ui/src/components/settings/ApiOptions.tsx +++ b/webview-ui/src/components/settings/ApiOptions.tsx @@ -603,17 +603,25 @@ const ApiOptions = ({ showModelOptions, apiErrorMessage, modelIdErrorMessage, is marginTop: "5px", color: "var(--vscode-descriptionForeground)", }}> - To use Google Cloud Vertex AI, you need to - - {"1) create a Google Cloud account › enable the Vertex AI API › enable the desired Claude models,"} - {" "} - - {"2) install the Google Cloud CLI › configure Application Default Credentials."} - + To use Google Cloud Vertex AI: +
    +
  1. + + { + "Create a Google Cloud account › enable the Vertex AI API › enable the desired Claude models." + } + +
    2. +
  2. + + {"Install the Google Cloud CLI › configure Application Default Credentials."} + +
    3. +

)} diff --git a/webview-ui/src/components/settings/__tests__/APIOptions.spec.tsx b/webview-ui/src/components/settings/__tests__/APIOptions.spec.tsx index 8cad54142b..a9c802b50e 100644 --- a/webview-ui/src/components/settings/__tests__/APIOptions.spec.tsx +++ b/webview-ui/src/components/settings/__tests__/APIOptions.spec.tsx @@ -1,123 +1,87 @@ -import { render, screen, fireEvent } from "@testing-library/react" -import { describe, it, expect, vi } from "vitest" +import { fireEvent, render, screen } from "@testing-library/react" +import userEvent from "@testing-library/user-event" import ApiOptions from "../ApiOptions" import { ExtensionStateContextProvider } from "../../../context/ExtensionStateContext" +import React from "react" -vi.mock("../../../context/ExtensionStateContext", async (importOriginal) => { - const actual = await importOriginal() - return { - ...actual, - // your mocked methods - useExtensionState: vi.fn(() => ({ - apiConfiguration: { - apiProvider: "requesty", - requestyApiKey: "", - requestyModelId: "", - }, - setApiConfiguration: vi.fn(), - uriScheme: "vscode", - })), - } -}) +vi.mock("../src/utils/vscode", () => ({ + vscode: { + postMessage: vi.fn(), + }, +})) -describe("ApiOptions Component", () => { - vi.clearAllMocks() - const mockPostMessage = vi.fn() - - beforeEach(() => { - global.vscode = { postMessage: mockPostMessage } as any - }) - - it("renders Requesty API Key input", () => { - render( - - - , - ) - const apiKeyInput = screen.getByPlaceholderText("Enter API Key...") - expect(apiKeyInput).toBeInTheDocument() - }) - - it("renders Requesty Model ID input", () => { - render( - - - , - ) - const modelIdInput = screen.getByPlaceholderText("Enter Model ID...") - expect(modelIdInput).toBeInTheDocument() - }) -}) - -vi.mock("../../../context/ExtensionStateContext", async (importOriginal) => { - const actual = await importOriginal() - return { - ...actual, - // your mocked methods - useExtensionState: vi.fn(() => ({ - apiConfiguration: { - apiProvider: "together", - requestyApiKey: "", - requestyModelId: "", - }, - setApiConfiguration: vi.fn(), - uriScheme: "vscode", - })), - } -}) - -describe("ApiOptions Component", () => { - vi.clearAllMocks() - const mockPostMessage = vi.fn() - - beforeEach(() => { - global.vscode = { postMessage: mockPostMessage } as any - }) - - it("renders Together API Key input", () => { - render( - - - , - ) - const apiKeyInput = screen.getByPlaceholderText("Enter API Key...") - expect(apiKeyInput).toBeInTheDocument() - }) - - it("renders Together Model ID input", () => { - render( - - - , - ) - const modelIdInput = screen.getByPlaceholderText("Enter Model ID...") - expect(modelIdInput).toBeInTheDocument() - }) -}) - -vi.mock("../../../context/ExtensionStateContext", async (importOriginal) => { - const actual = await importOriginal() - return { - ...actual, - // your mocked methods - useExtensionState: vi.fn(() => ({ - apiConfiguration: { - apiProvider: "openai", - requestyApiKey: "", - requestyModelId: "", - }, - setApiConfiguration: vi.fn(), - uriScheme: "vscode", - })), - } -}) - -describe("OpenApiInfoOptions", () => { - const mockPostMessage = vi.fn() +const renderComponent = (props = {}) => { + return render( + + + , + ) +} +describe("ApiOptions", () => { beforeEach(() => { vi.clearAllMocks() - global.vscode = { postMessage: mockPostMessage } + Element.prototype.scrollIntoView = vi.fn() + }) + + it("renders API Provider dropdown", () => { + renderComponent() + expect(screen.getByLabelText(/API Provider/i)).toBeInTheDocument() + }) + + it("renders OpenAI API Key input when OpenAI provider is selected", async () => { + renderComponent() + const dropdown = screen.getAllByRole("combobox")[0] + await userEvent.click(dropdown) + const option = screen.getByRole("option", { name: "OpenAI" }) + await userEvent.click(option) + expect(screen.getByPlaceholderText(/Enter API Key/i)).toBeInTheDocument() + }) + + it("renders error message when apiErrorMessage is provided", () => { + renderComponent({ apiErrorMessage: "API Error" }) + expect(screen.getByText(/API Error/i)).toBeInTheDocument() + }) + + it("renders model dropdown for selected provider", async () => { + renderComponent() + const dropdown = screen.getAllByRole("combobox")[0] + await userEvent.click(dropdown) + const option = screen.getByRole("option", { name: "Anthropic" }) + await userEvent.click(option) + expect(screen.getByPlaceholderText(/Enter API Key/i)).toBeInTheDocument() + }) + + it("renders OpenRouter API Key input when OpenRouter provider is selected", async () => { + renderComponent() + const dropdown = screen.getAllByRole("combobox")[0] + await userEvent.click(dropdown) + const option = screen.getByRole("option", { name: "OpenRouter" }) + await userEvent.click(option) + expect(screen.getByPlaceholderText(/Enter API Key/i)).toBeInTheDocument() + }) + + it("renders AWS credentials inputs when AWS Bedrock provider is selected", async () => { + renderComponent() + const dropdown = screen.getAllByRole("combobox")[0] + await userEvent.click(dropdown) + const option = screen.getByRole("option", { name: "AWS Bedrock" }) + await userEvent.click(option) + expect(screen.getByPlaceholderText(/Enter Access Key/i)).toBeInTheDocument() + expect(screen.getByPlaceholderText(/Enter Secret Key/i)).toBeInTheDocument() + expect(screen.getByPlaceholderText(/Enter Session Token/i)).toBeInTheDocument() + const toggle = screen.getByText("AWS Profile") + await userEvent.click(toggle) + expect(screen.getByPlaceholderText(/Enter profile name/i)).toBeInTheDocument() + }) + + it("renders Google Cloud Project ID input when Vertex provider is selected", async () => { + renderComponent() + const dropdown = screen.getAllByRole("combobox")[0] + await userEvent.click(dropdown) + const option = screen.getByRole("option", { name: "GCP Vertex AI" }) + await userEvent.click(option) + expect(screen.getByPlaceholderText(/Enter Project ID/i)).toBeInTheDocument() }) it("renders OpenAI Supports Images input", () => {