mirror of
https://github.com/RooVetGit/Roo-Code.git
synced 2026-08-28 05:27:24 +00:00
Better Streaming Support (#1980)
* feat: enterprise support * Update src/api/providers/enterprise.ts Co-authored-by: ellipsis-dev[bot] <65095814+ellipsis-dev[bot]@users.noreply.github.com> * refactor + anthropic * fix * update chunking * fix enterprise providers * minor refactor for chunking * comments * defaults * tests * remove imports * suggested fixes * decouple message type * upgrade libs, prompt caching no longer in beta for claude models * updates for tests * enterprise -> streaming provider * update tests * remove section from anthropic.ts * handle specific GCP Vertex invalid_grant error * finish comment * add caching stats to chunking --------- Co-authored-by: ellipsis-dev[bot] <65095814+ellipsis-dev[bot]@users.noreply.github.com>
This commit is contained in:
brownrw8
Trevor Hudson
parent
30a4a0e1f1
commit
9b5cefd69d
18 changed files with 590 additions and 580 deletions
5
.changeset/warm-pans-perform.md
Normal file
5
.changeset/warm-pans-perform.md
Normal file
|
|
@ -0,0 +1,5 @@
|
|||
---
|
||||
"claude-dev": minor
|
||||
---
|
||||
|
||||
Better enterprise support for providers (Bedrock, Vertex)
|
||||
20
package-lock.json
generated
20
package-lock.json
generated
|
|
@ -9,8 +9,8 @@
|
|||
"version": "3.4.9",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"@anthropic-ai/bedrock-sdk": "^0.10.2",
|
||||
"@anthropic-ai/sdk": "^0.26.0",
|
||||
"@anthropic-ai/bedrock-sdk": "^0.12.4",
|
||||
"@anthropic-ai/sdk": "^0.36.3",
|
||||
"@anthropic-ai/vertex-sdk": "^0.6.4",
|
||||
"@google/generative-ai": "^0.18.0",
|
||||
"@mistralai/mistralai": "^1.5.0",
|
||||
|
|
@ -78,11 +78,12 @@
|
|||
}
|
||||
},
|
||||
"node_modules/@anthropic-ai/bedrock-sdk": {
|
||||
"version": "0.10.2",
|
||||
"resolved": "https://registry.npmjs.org/@anthropic-ai/bedrock-sdk/-/bedrock-sdk-0.10.2.tgz",
|
||||
"integrity": "sha512-sGmTzKJQHVwfXexe+yfzPU3rJmUMCygC+GNPkmMsPX/Jr+WKtJ0M71nGyHONr6vcHwUpUWA6o0MRH/oHaE54KA==",
|
||||
"version": "0.12.4",
|
||||
"resolved": "https://registry.npmjs.org/@anthropic-ai/bedrock-sdk/-/bedrock-sdk-0.12.4.tgz",
|
||||
"integrity": "sha512-kraOgWWyVO/Wef3wYbpws77pCZubg/hCzXQ7RGrLRJsRpbTIT+ms+MigGu/0b1qn3o5WuIrumlT3Qtu3esHpzw==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@anthropic-ai/sdk": "^0",
|
||||
"@anthropic-ai/sdk": ">=0.36 <1",
|
||||
"@aws-crypto/sha256-js": "^4.0.0",
|
||||
"@aws-sdk/client-bedrock-runtime": "^3.423.0",
|
||||
"@aws-sdk/credential-providers": "^3.341.0",
|
||||
|
|
@ -96,9 +97,10 @@
|
|||
}
|
||||
},
|
||||
"node_modules/@anthropic-ai/sdk": {
|
||||
"version": "0.26.0",
|
||||
"resolved": "https://registry.npmjs.org/@anthropic-ai/sdk/-/sdk-0.26.0.tgz",
|
||||
"integrity": "sha512-vNbZ2rnnMfk8Bf4OdeVy6GA4EXao8tGC0tLEoSAl1NZrip9oOxnEGUkXl3FsPQgeBM5hmpGE1tSLuu9HEVJiHg==",
|
||||
"version": "0.36.3",
|
||||
"resolved": "https://registry.npmjs.org/@anthropic-ai/sdk/-/sdk-0.36.3.tgz",
|
||||
"integrity": "sha512-+c0mMLxL/17yFZ4P5+U6bTWiCSFZUKJddrv01ud2aFBWnTPLdRncYV76D3q1tqfnL7aCnhRtykFnoCFzvr4U3Q==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@types/node": "^18.11.18",
|
||||
"@types/node-fetch": "^2.6.4",
|
||||
|
|
|
|||
|
|
@ -262,9 +262,9 @@
|
|||
"typescript": "^5.4.5"
|
||||
},
|
||||
"dependencies": {
|
||||
"@anthropic-ai/bedrock-sdk": "^0.10.2",
|
||||
"@anthropic-ai/sdk": "^0.26.0",
|
||||
"@anthropic-ai/vertex-sdk": "^0.6.4",
|
||||
"@anthropic-ai/bedrock-sdk": "^0.12.4",
|
||||
"@anthropic-ai/sdk": "^0.36.3",
|
||||
"@google/generative-ai": "^0.18.0",
|
||||
"@mistralai/mistralai": "^1.5.0",
|
||||
"@modelcontextprotocol/sdk": "^1.0.1",
|
||||
|
|
@ -305,4 +305,4 @@
|
|||
"web-tree-sitter": "^0.22.6",
|
||||
"zod": "^3.24.2"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -18,8 +18,8 @@ import { MistralHandler } from "./providers/mistral"
|
|||
import { VsCodeLmHandler } from "./providers/vscode-lm"
|
||||
import { LiteLlmHandler } from "./providers/litellm"
|
||||
|
||||
export interface ApiHandler {
|
||||
createMessage(systemPrompt: string, messages: Anthropic.Messages.MessageParam[]): ApiStream
|
||||
export interface ApiHandler<MessageType = Anthropic.Messages.MessageParam> {
|
||||
createMessage(systemPrompt: string, messages: MessageType[]): ApiStream
|
||||
getModel(): { id: string; info: ModelInfo }
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,185 +1,74 @@
|
|||
import { Anthropic } from "@anthropic-ai/sdk"
|
||||
import { Stream as AnthropicStream } from "@anthropic-ai/sdk/streaming"
|
||||
import { withRetry } from "../retry"
|
||||
import { anthropicDefaultModelId, AnthropicModelId, anthropicModels, ApiHandlerOptions, ModelInfo } from "../../shared/api"
|
||||
import { ApiHandler } from "../index"
|
||||
import { anthropicDefaultModelId, AnthropicModelId, anthropicModels, ModelInfo } from "../../shared/api"
|
||||
import { ApiStream } from "../transform/stream"
|
||||
import { ClaudeStreamingHandler } from "./claude-streaming"
|
||||
|
||||
export class AnthropicHandler implements ApiHandler {
|
||||
private options: ApiHandlerOptions
|
||||
private client: Anthropic
|
||||
|
||||
constructor(options: ApiHandlerOptions) {
|
||||
this.options = options
|
||||
this.client = new Anthropic({
|
||||
/**
|
||||
* Handles interactions with the Anthropic service.
|
||||
*/
|
||||
export class AnthropicHandler extends ClaudeStreamingHandler<Anthropic> {
|
||||
getClient() {
|
||||
return new Anthropic({
|
||||
apiKey: this.options.apiKey,
|
||||
baseURL: this.options.anthropicBaseUrl || undefined,
|
||||
baseURL: this.options.anthropicBaseUrl || null, // default baseURL: https://api.anthropic.com
|
||||
})
|
||||
}
|
||||
|
||||
@withRetry()
|
||||
async *createMessage(systemPrompt: string, messages: Anthropic.Messages.MessageParam[]): ApiStream {
|
||||
override async *createStreamingMessage(systemPrompt: string, messages: Anthropic.Messages.MessageParam[]): ApiStream {
|
||||
const model = this.getModel()
|
||||
let stream: AnthropicStream<Anthropic.Beta.PromptCaching.Messages.RawPromptCachingBetaMessageStreamEvent>
|
||||
const modelId = model.id
|
||||
switch (modelId) {
|
||||
// 'latest' alias does not support cache_control
|
||||
case "claude-3-7-sonnet-20250219":
|
||||
case "claude-3-5-sonnet-20241022":
|
||||
case "claude-3-5-haiku-20241022":
|
||||
case "claude-3-opus-20240229":
|
||||
case "claude-3-haiku-20240307": {
|
||||
/*
|
||||
The latest message will be the new user message, one before will be the assistant message from a previous request, and the user message before that will be a previously cached user message. So we need to mark the latest user message as ephemeral to cache it for the next request, and mark the second to last user message as ephemeral to let the server know the last message to retrieve from the cache for the current request..
|
||||
*/
|
||||
const userMsgIndices = messages.reduce(
|
||||
(acc, msg, index) => (msg.role === "user" ? [...acc, index] : acc),
|
||||
[] as number[],
|
||||
)
|
||||
const lastUserMsgIndex = userMsgIndices[userMsgIndices.length - 1] ?? -1
|
||||
const secondLastMsgUserIndex = userMsgIndices[userMsgIndices.length - 2] ?? -1
|
||||
stream = await this.client.beta.promptCaching.messages.create(
|
||||
{
|
||||
model: modelId,
|
||||
max_tokens: model.info.maxTokens || 8192,
|
||||
temperature: 0,
|
||||
system: [
|
||||
{
|
||||
text: systemPrompt,
|
||||
type: "text",
|
||||
cache_control: { type: "ephemeral" },
|
||||
},
|
||||
], // setting cache breakpoint for system prompt so new tasks can reuse it
|
||||
messages: messages.map((message, index) => {
|
||||
if (index === lastUserMsgIndex || index === secondLastMsgUserIndex) {
|
||||
return {
|
||||
...message,
|
||||
content:
|
||||
typeof message.content === "string"
|
||||
? [
|
||||
{
|
||||
type: "text",
|
||||
text: message.content,
|
||||
cache_control: {
|
||||
type: "ephemeral",
|
||||
},
|
||||
},
|
||||
]
|
||||
: message.content.map((content, contentIndex) =>
|
||||
contentIndex === message.content.length - 1
|
||||
? {
|
||||
...content,
|
||||
cache_control: {
|
||||
type: "ephemeral",
|
||||
},
|
||||
}
|
||||
: content,
|
||||
),
|
||||
}
|
||||
}
|
||||
return message
|
||||
}),
|
||||
// tools, // cache breakpoints go from tools > system > messages, and since tools dont change, we can just set the breakpoint at the end of system (this avoids having to set a breakpoint at the end of tools which by itself does not meet min requirements for haiku caching)
|
||||
// tool_choice: { type: "auto" },
|
||||
// tools: tools,
|
||||
stream: true,
|
||||
},
|
||||
(() => {
|
||||
// prompt caching: https://x.com/alexalbert__/status/1823751995901272068
|
||||
// https://github.com/anthropics/anthropic-sdk-typescript?tab=readme-ov-file#default-headers
|
||||
// https://github.com/anthropics/anthropic-sdk-typescript/commit/c920b77fc67bd839bfeb6716ceab9d7c9bbe7393
|
||||
switch (modelId) {
|
||||
case "claude-3-7-sonnet-20250219":
|
||||
case "claude-3-5-sonnet-20241022":
|
||||
case "claude-3-5-haiku-20241022":
|
||||
case "claude-3-opus-20240229":
|
||||
case "claude-3-haiku-20240307":
|
||||
return {
|
||||
headers: {
|
||||
"anthropic-beta": "prompt-caching-2024-07-31",
|
||||
},
|
||||
}
|
||||
default:
|
||||
return undefined
|
||||
}
|
||||
})(),
|
||||
)
|
||||
break
|
||||
}
|
||||
default: {
|
||||
stream = (await this.client.messages.create({
|
||||
model: modelId,
|
||||
max_tokens: model.info.maxTokens || 8192,
|
||||
temperature: 0,
|
||||
system: [{ text: systemPrompt, type: "text" }],
|
||||
messages,
|
||||
// tools,
|
||||
// tool_choice: { type: "auto" },
|
||||
stream: true,
|
||||
})) as any
|
||||
break
|
||||
}
|
||||
|
||||
let stream: AnthropicStream<Anthropic.Messages.RawMessageStreamEvent>
|
||||
|
||||
if (Object.keys(anthropicModels).includes(modelId)) {
|
||||
stream = await this.createModelStream(
|
||||
systemPrompt,
|
||||
messages,
|
||||
modelId,
|
||||
model.info.maxTokens ?? AnthropicHandler.DEFAULT_TOKEN_SIZE,
|
||||
)
|
||||
} else {
|
||||
throw new Error(`Invalid model ID: ${modelId}`)
|
||||
}
|
||||
|
||||
for await (const chunk of stream) {
|
||||
switch (chunk.type) {
|
||||
case "message_start":
|
||||
// tells us cache reads/writes/input/output
|
||||
const usage = chunk.message.usage
|
||||
yield {
|
||||
type: "usage",
|
||||
inputTokens: usage.input_tokens || 0,
|
||||
outputTokens: usage.output_tokens || 0,
|
||||
cacheWriteTokens: usage.cache_creation_input_tokens || undefined,
|
||||
cacheReadTokens: usage.cache_read_input_tokens || undefined,
|
||||
}
|
||||
break
|
||||
case "message_delta":
|
||||
// tells us stop_reason, stop_sequence, and output tokens along the way and at the end of the message
|
||||
|
||||
yield {
|
||||
type: "usage",
|
||||
inputTokens: 0,
|
||||
outputTokens: chunk.usage.output_tokens || 0,
|
||||
}
|
||||
break
|
||||
case "message_stop":
|
||||
// no usage data, just an indicator that the message is done
|
||||
break
|
||||
case "content_block_start":
|
||||
switch (chunk.content_block.type) {
|
||||
case "text":
|
||||
// we may receive multiple text blocks, in which case just insert a line break between them
|
||||
if (chunk.index > 0) {
|
||||
yield {
|
||||
type: "text",
|
||||
text: "\n",
|
||||
}
|
||||
}
|
||||
yield {
|
||||
type: "text",
|
||||
text: chunk.content_block.text,
|
||||
}
|
||||
break
|
||||
}
|
||||
break
|
||||
case "content_block_delta":
|
||||
switch (chunk.delta.type) {
|
||||
case "text_delta":
|
||||
yield {
|
||||
type: "text",
|
||||
text: chunk.delta.text,
|
||||
}
|
||||
break
|
||||
}
|
||||
break
|
||||
case "content_block_stop":
|
||||
break
|
||||
}
|
||||
}
|
||||
yield* this.processStream(stream)
|
||||
}
|
||||
|
||||
getModel(): { id: AnthropicModelId; info: ModelInfo } {
|
||||
override async createModelStream(
|
||||
systemPrompt: string,
|
||||
messages: Anthropic.Messages.MessageParam[],
|
||||
modelId: string,
|
||||
maxTokens: number,
|
||||
): Promise<AnthropicStream<Anthropic.Messages.RawMessageStreamEvent>> {
|
||||
/*
|
||||
The latest message will be the new user message, one before will be the assistant message from a previous request, and the user message before that will be a previously cached user message. So we need to mark the latest user message as ephemeral to cache it for the next request, and mark the second to last user message as ephemeral to let the server know the last message to retrieve from the cache for the current request..
|
||||
*/
|
||||
const userMsgIndices = messages.reduce((acc, msg, index) => (msg.role === "user" ? [...acc, index] : acc), [] as number[])
|
||||
const lastUserMsgIndex = userMsgIndices[userMsgIndices.length - 1] ?? -1
|
||||
const secondLastMsgUserIndex = userMsgIndices[userMsgIndices.length - 2] ?? -1
|
||||
return await this.client.messages.create({
|
||||
model: modelId,
|
||||
max_tokens: maxTokens || AnthropicHandler.DEFAULT_TOKEN_SIZE,
|
||||
temperature: AnthropicHandler.DEFAULT_TEMPERATURE,
|
||||
system: [
|
||||
{
|
||||
text: systemPrompt,
|
||||
type: "text",
|
||||
cache_control: { type: "ephemeral" },
|
||||
},
|
||||
], // setting cache breakpoint for system prompt so new tasks can reuse it
|
||||
messages: messages.map((message, index) =>
|
||||
this.transformMessage(message, index, lastUserMsgIndex, secondLastMsgUserIndex),
|
||||
),
|
||||
// tools, // cache breakpoints go from tools > system > messages, and since tools dont change, we can just set the breakpoint at the end of system (this avoids having to set a breakpoint at the end of tools which by itself does not meet min requirements for haiku caching)
|
||||
// tool_choice: { type: "auto" },
|
||||
// tools: tools,
|
||||
stream: true,
|
||||
})
|
||||
}
|
||||
|
||||
override getModel(): { id: AnthropicModelId; info: ModelInfo } {
|
||||
const modelId = this.options.apiModelId
|
||||
if (modelId && modelId in anthropicModels) {
|
||||
const id = modelId as AnthropicModelId
|
||||
|
|
|
|||
|
|
@ -1,100 +1,128 @@
|
|||
import AnthropicBedrock from "@anthropic-ai/bedrock-sdk"
|
||||
import { Anthropic } from "@anthropic-ai/sdk"
|
||||
import { ApiHandler } from "../"
|
||||
import { ApiHandlerOptions, bedrockDefaultModelId, BedrockModelId, bedrockModels, ModelInfo } from "../../shared/api"
|
||||
import { Stream as AnthropicStream } from "@anthropic-ai/sdk/streaming"
|
||||
import { bedrockDefaultModelId, BedrockModelId, bedrockModels, ModelInfo } from "../../shared/api"
|
||||
import { ApiStream } from "../transform/stream"
|
||||
import { fromNodeProviderChain } from "@aws-sdk/credential-providers"
|
||||
import { fromIni, fromNodeProviderChain } from "@aws-sdk/credential-providers"
|
||||
import { ClaudeStreamingHandler } from "./claude-streaming"
|
||||
|
||||
// https://docs.anthropic.com/en/api/claude-on-amazon-bedrock
|
||||
export class AwsBedrockHandler implements ApiHandler {
|
||||
private options: ApiHandlerOptions
|
||||
/**
|
||||
* Handles interactions with the Anthropic Bedrock service using AWS credentials.
|
||||
*/
|
||||
export class AwsBedrockHandler extends ClaudeStreamingHandler<AnthropicBedrock> {
|
||||
async getClient() {
|
||||
const clientConfig: any = {
|
||||
awsRegion: this.options.awsRegion || "us-west-2",
|
||||
}
|
||||
|
||||
constructor(options: ApiHandlerOptions) {
|
||||
this.options = options
|
||||
try {
|
||||
this.saveClientToNodeProviderChain()
|
||||
// Use AWS profile credentials if specified.
|
||||
if (this.options.awsUseProfile) {
|
||||
const credentials = await fromIni({
|
||||
profile: this.options.awsProfile || "default",
|
||||
ignoreCache: true,
|
||||
})()
|
||||
clientConfig.awsAccessKey = credentials.accessKeyId
|
||||
clientConfig.awsSecretKey = credentials.secretAccessKey
|
||||
clientConfig.awsSessionToken = credentials.sessionToken
|
||||
}
|
||||
// Use provided AWS access key and secret key if specified.
|
||||
else if (this.options.awsAccessKey && this.options.awsSecretKey) {
|
||||
clientConfig.awsAccessKey = this.options.awsAccessKey
|
||||
clientConfig.awsSecretKey = this.options.awsSecretKey
|
||||
if (this.options.awsSessionToken) {
|
||||
clientConfig.awsSessionToken = this.options.awsSessionToken
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
console.error("Failed to initialize Bedrock client:", error)
|
||||
throw error
|
||||
}
|
||||
return new AnthropicBedrock(clientConfig)
|
||||
}
|
||||
|
||||
async *createMessage(systemPrompt: string, messages: Anthropic.Messages.MessageParam[]): ApiStream {
|
||||
// cross region inference requires prefixing the model id with the region
|
||||
let modelId = await this.getModelId()
|
||||
async *createStreamingMessage(systemPrompt: string, messages: Anthropic.Messages.MessageParam[]): ApiStream {
|
||||
const model = this.getModel()
|
||||
const modelId = this.getModelId()
|
||||
|
||||
// create anthropic client, using sessions created or renewed after this handler's
|
||||
// initialization, and allowing for session renewal if necessary as well
|
||||
let client = await this.getClient()
|
||||
let stream: AnthropicStream<Anthropic.Messages.RawMessageStreamEvent>
|
||||
|
||||
const stream = await client.messages.create({
|
||||
if (Object.keys(bedrockModels).includes(modelId)) {
|
||||
stream = await this.createModelStream(
|
||||
systemPrompt,
|
||||
messages,
|
||||
modelId,
|
||||
model.info.maxTokens ?? ClaudeStreamingHandler.DEFAULT_TOKEN_SIZE,
|
||||
)
|
||||
} else {
|
||||
stream = await this.client.messages.create({
|
||||
model: modelId,
|
||||
max_tokens: model.info.maxTokens || ClaudeStreamingHandler.DEFAULT_TOKEN_SIZE,
|
||||
temperature: ClaudeStreamingHandler.DEFAULT_TEMPERATURE,
|
||||
system: systemPrompt,
|
||||
messages,
|
||||
stream: true,
|
||||
})
|
||||
}
|
||||
|
||||
yield* this.processStream(stream)
|
||||
}
|
||||
|
||||
async createModelStream(
|
||||
systemPrompt: string,
|
||||
messages: Anthropic.Messages.MessageParam[],
|
||||
modelId: string,
|
||||
maxTokens: number,
|
||||
): Promise<AnthropicStream<Anthropic.Messages.RawMessageStreamEvent>> {
|
||||
const userMsgIndices = messages.reduce((acc, msg, index) => (msg.role === "user" ? [...acc, index] : acc), [] as number[])
|
||||
|
||||
return await this.client.messages.create({
|
||||
model: modelId,
|
||||
max_tokens: this.getModel().info.maxTokens || 8192,
|
||||
temperature: 0,
|
||||
system: systemPrompt,
|
||||
max_tokens: maxTokens || ClaudeStreamingHandler.DEFAULT_TOKEN_SIZE,
|
||||
temperature: ClaudeStreamingHandler.DEFAULT_TEMPERATURE,
|
||||
system: [
|
||||
{
|
||||
text: systemPrompt,
|
||||
type: "text",
|
||||
cache_control: { type: "ephemeral" },
|
||||
},
|
||||
], // setting cache breakpoint for system prompt so new tasks can reuse it
|
||||
messages,
|
||||
stream: true,
|
||||
})
|
||||
for await (const chunk of stream) {
|
||||
switch (chunk.type) {
|
||||
case "message_start":
|
||||
const usage = chunk.message.usage
|
||||
yield {
|
||||
type: "usage",
|
||||
inputTokens: usage.input_tokens || 0,
|
||||
outputTokens: usage.output_tokens || 0,
|
||||
}
|
||||
break
|
||||
case "message_delta":
|
||||
yield {
|
||||
type: "usage",
|
||||
inputTokens: 0,
|
||||
outputTokens: chunk.usage.output_tokens || 0,
|
||||
}
|
||||
break
|
||||
}
|
||||
|
||||
case "content_block_start":
|
||||
switch (chunk.content_block.type) {
|
||||
case "text":
|
||||
if (chunk.index > 0) {
|
||||
yield {
|
||||
type: "text",
|
||||
text: "\n",
|
||||
}
|
||||
}
|
||||
yield {
|
||||
type: "text",
|
||||
text: chunk.content_block.text,
|
||||
}
|
||||
break
|
||||
}
|
||||
break
|
||||
case "content_block_delta":
|
||||
switch (chunk.delta.type) {
|
||||
case "text_delta":
|
||||
yield {
|
||||
type: "text",
|
||||
text: chunk.delta.text,
|
||||
}
|
||||
break
|
||||
}
|
||||
break
|
||||
protected getModelId(): string {
|
||||
if (this.options.awsUseCrossRegionInference) {
|
||||
const regionPrefix = (this.options.awsRegion || "").slice(0, 3)
|
||||
switch (regionPrefix) {
|
||||
case "us-":
|
||||
return `us.${this.getModel().id}`
|
||||
case "eu-":
|
||||
return `eu.${this.getModel().id}`
|
||||
default:
|
||||
return this.getModel().id
|
||||
}
|
||||
}
|
||||
return this.getModel().id
|
||||
}
|
||||
|
||||
getModel(): { id: BedrockModelId; info: ModelInfo } {
|
||||
const modelId = this.options.apiModelId
|
||||
if (modelId && modelId in bedrockModels) {
|
||||
const id = modelId as BedrockModelId
|
||||
return { id, info: bedrockModels[id] }
|
||||
}
|
||||
return {
|
||||
id: bedrockDefaultModelId,
|
||||
info: bedrockModels[bedrockDefaultModelId],
|
||||
// Return the model information for the specified model ID
|
||||
return { id: modelId as BedrockModelId, info: bedrockModels[modelId as BedrockModelId] }
|
||||
}
|
||||
return { id: bedrockDefaultModelId, info: bedrockModels[bedrockDefaultModelId] }
|
||||
}
|
||||
|
||||
private async getClient(): Promise<AnthropicBedrock> {
|
||||
private async saveClientToNodeProviderChain() {
|
||||
// Create AWS credentials by executing a an AWS provider chain exactly as the
|
||||
// Anthropic SDK does it, by wrapping the default chain into a temporary process
|
||||
// environment.
|
||||
const providerChain = fromNodeProviderChain()
|
||||
const credentials = await AwsBedrockHandler.withTempEnv(
|
||||
await AwsBedrockHandler.withTempEnv(
|
||||
() => {
|
||||
AwsBedrockHandler.setEnv("AWS_REGION", this.options.awsRegion)
|
||||
AwsBedrockHandler.setEnv("AWS_ACCESS_KEY_ID", this.options.awsAccessKey)
|
||||
|
|
@ -104,36 +132,6 @@ export class AwsBedrockHandler implements ApiHandler {
|
|||
},
|
||||
() => providerChain(),
|
||||
)
|
||||
|
||||
// Return an AnthropicBedrock client with the resolved/assumed credentials.
|
||||
//
|
||||
// When AnthropicBedrock creates its AWS client, the chain will execute very
|
||||
// fast as the access/secret keys will already be already provided, and have
|
||||
// a higher precedence than the profiles.
|
||||
return new AnthropicBedrock({
|
||||
awsAccessKey: credentials.accessKeyId,
|
||||
awsSecretKey: credentials.secretAccessKey,
|
||||
awsSessionToken: credentials.sessionToken,
|
||||
awsRegion: this.options.awsRegion || "us-east-1",
|
||||
})
|
||||
}
|
||||
|
||||
private async getModelId(): Promise<string> {
|
||||
if (this.options.awsUseCrossRegionInference) {
|
||||
let regionPrefix = (this.options.awsRegion || "").slice(0, 3)
|
||||
switch (regionPrefix) {
|
||||
case "us-":
|
||||
return `us.${this.getModel().id}`
|
||||
case "eu-":
|
||||
return `eu.${this.getModel().id}`
|
||||
break
|
||||
default:
|
||||
// cross region inference is not supported in this region, falling back to default model
|
||||
return this.getModel().id
|
||||
break
|
||||
}
|
||||
}
|
||||
return this.getModel().id
|
||||
}
|
||||
|
||||
private static async withTempEnv<R>(updateEnv: () => void, fn: () => Promise<R>): Promise<R> {
|
||||
|
|
|
|||
211
src/api/providers/claude-streaming.ts
Normal file
211
src/api/providers/claude-streaming.ts
Normal file
|
|
@ -0,0 +1,211 @@
|
|||
import { Anthropic } from "@anthropic-ai/sdk"
|
||||
import { ApiHandler } from "../"
|
||||
import { ApiHandlerOptions, ModelInfo } from "../../shared/api"
|
||||
import { ApiStream } from "../transform/stream"
|
||||
import { withRetry } from "../retry"
|
||||
import { Stream as AnthropicStream } from "@anthropic-ai/sdk/streaming"
|
||||
import { AnthropicVertex } from "@anthropic-ai/vertex-sdk"
|
||||
import AnthropicBedrock from "@anthropic-ai/bedrock-sdk"
|
||||
|
||||
/**
|
||||
* Abstract base class for Claude-based streaming providers.
|
||||
*
|
||||
* This class provides a standardized framework for handling Claude-based API interactions,
|
||||
* ensuring consistency and reusability. It enforces a contract for subclasses to implement
|
||||
* specific methods, promoting a clear and maintainable architecture.
|
||||
*
|
||||
* The use of generators and yielding allows efficient handling of asynchronous data streams,
|
||||
* enabling real-time data processing without blocking the main thread.
|
||||
*
|
||||
* Generics provide flexibility and type safety for different enterprise clients.
|
||||
*
|
||||
* Caching improves performance by storing frequently accessed data, reducing latency and load
|
||||
* on external services, and optimizing data handling.
|
||||
*
|
||||
* @template ClientType - The type of client. Must be one of Anthropic, AnthropicVertex, or AnthropicBedrock.
|
||||
* @implements ApiHandler
|
||||
*/
|
||||
|
||||
export abstract class ClaudeStreamingHandler<ClientType extends Anthropic | AnthropicVertex | AnthropicBedrock>
|
||||
implements ApiHandler
|
||||
{
|
||||
static readonly DEFAULT_TOKEN_SIZE: number = 8192 // The default token size for message generation.
|
||||
static readonly DEFAULT_TEMPERATURE: number = 0 // The default temperature for message generation.
|
||||
protected options: ApiHandlerOptions // The options for the handler.
|
||||
protected cache: Map<string, ApiStream> // A cache of message streams.
|
||||
protected client!: ClientType // The client.
|
||||
|
||||
/**
|
||||
* Creates a new handler.
|
||||
* @param options - The options for the handler.
|
||||
*/
|
||||
constructor(options: ApiHandlerOptions) {
|
||||
this.options = options
|
||||
this.cache = new Map()
|
||||
this._initialize()
|
||||
}
|
||||
|
||||
/**
|
||||
* Initializes the client.
|
||||
*/
|
||||
private _initialize() {
|
||||
const client = this.getClient()
|
||||
if (client instanceof Promise) {
|
||||
client
|
||||
.then((resolvedClient) => {
|
||||
this.client = resolvedClient
|
||||
})
|
||||
.catch((error) => {
|
||||
throw new Error("Failed to initialize client: " + error)
|
||||
})
|
||||
} else {
|
||||
this.client = client
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a message stream to a Claude model.
|
||||
* @param systemPrompt - The system prompt to initialize the conversation.
|
||||
* @param messages - An array of message parameters.
|
||||
* @returns An asynchronous generator yielding ApiStream events.
|
||||
*/
|
||||
protected abstract createModelStream(
|
||||
systemPrompt: string,
|
||||
messages: Anthropic.Messages.MessageParam[],
|
||||
modelId: string,
|
||||
maxTokens: number,
|
||||
): Promise<AnthropicStream<Anthropic.Messages.RawMessageStreamEvent>>
|
||||
|
||||
/**
|
||||
* Initializes the client.
|
||||
* This method must be implemented by subclasses.
|
||||
*/
|
||||
protected abstract getClient(): ClientType | Promise<ClientType>
|
||||
|
||||
/**
|
||||
* Creates a message stream.
|
||||
* @param systemPrompt - The system prompt to initialize the conversation.
|
||||
* @param messages - An array of message parameters.
|
||||
* @returns An asynchronous generator yielding ApiStream events.
|
||||
*/
|
||||
@withRetry()
|
||||
async *createMessage(systemPrompt: string, messages: Anthropic.Messages.MessageParam[]): ApiStream {
|
||||
try {
|
||||
yield* this.createStreamingMessage(systemPrompt, messages)
|
||||
} catch (error) {
|
||||
this.handleMessageStreamError(error)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Handles an error that occurs during message stream creation. Override to handle provider-specific errors.
|
||||
* @param error - The error that threw during message stream creation.
|
||||
* @throws An error with a message indicating the failure.
|
||||
*/
|
||||
protected handleMessageStreamError(error: any) {
|
||||
throw new Error(`Failed to create message stream: ${error instanceof Error ? error.message : "Unknown error"}`)
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a streaming message stream.
|
||||
* This method must be implemented by subclasses.
|
||||
* @param systemPrompt - The system prompt to initialize the conversation.
|
||||
* @param messages - An array of message parameters.
|
||||
* @returns An asynchronous generator yielding ApiStream events.
|
||||
*/
|
||||
protected abstract createStreamingMessage(systemPrompt: string, messages: Anthropic.Messages.MessageParam[]): ApiStream
|
||||
|
||||
/**
|
||||
* Processes a stream of raw message events.
|
||||
* @param stream - A stream of raw message events.
|
||||
* @returns An asynchronous generator yielding ApiStream events.
|
||||
*/
|
||||
protected async *processStream(stream: AnthropicStream<Anthropic.Messages.RawMessageStreamEvent>): ApiStream {
|
||||
for await (const chunk of stream) {
|
||||
yield* this.processChunk(chunk)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Processes each chunk of the stream and yields the appropriate ApiStream events.
|
||||
* @param chunk - The chunk of data received from the stream.
|
||||
* @returns An asynchronous generator yielding ApiStream events.
|
||||
*/
|
||||
protected async *processChunk(chunk: any): ApiStream {
|
||||
switch (chunk.type) {
|
||||
case "message_start":
|
||||
// tells us cache reads/writes/input/output
|
||||
yield {
|
||||
type: "usage",
|
||||
inputTokens: chunk.message.usage.input_tokens || 0,
|
||||
outputTokens: chunk.message.usage.output_tokens || 0,
|
||||
cacheWriteTokens: chunk.message.usage.cache_creation_input_tokens || undefined,
|
||||
cacheReadTokens: chunk.message.usage.cache_read_input_tokens || undefined,
|
||||
}
|
||||
break
|
||||
case "message_delta":
|
||||
// tells us stop_reason, stop_sequence, and output tokens along the way and at the end of the message
|
||||
|
||||
yield {
|
||||
type: "usage",
|
||||
inputTokens: 0,
|
||||
outputTokens: chunk.usage.output_tokens || 0,
|
||||
}
|
||||
break
|
||||
case "message_stop":
|
||||
// no usage data, just an indicator that the message is done
|
||||
break
|
||||
case "content_block_start":
|
||||
if (chunk.content_block.type === "text") {
|
||||
// we may receive multiple text blocks, in which case just insert a line break between them
|
||||
if (chunk.index > 0) {
|
||||
yield { type: "text", text: "\n" }
|
||||
}
|
||||
yield { type: "text", text: chunk.content_block.text }
|
||||
}
|
||||
break
|
||||
case "content_block_delta":
|
||||
if (chunk.delta.type === "text_delta") {
|
||||
yield { type: "text", text: chunk.delta.text }
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Transforms a message based on its index and user message indices.
|
||||
* @param message - The message to transform.
|
||||
* @param index - The index of the message in the array.
|
||||
* @param lastUserMsgIndex - The index of the last user message.
|
||||
* @param secondLastMsgUserIndex - The index of the second last user message.
|
||||
* @returns The transformed message.
|
||||
*/
|
||||
protected transformMessage(
|
||||
message: Anthropic.Messages.MessageParam,
|
||||
index: number,
|
||||
lastUserMsgIndex: number,
|
||||
secondLastMsgUserIndex: number,
|
||||
): Anthropic.Messages.MessageParam {
|
||||
if (index === lastUserMsgIndex || index === secondLastMsgUserIndex) {
|
||||
return {
|
||||
...message,
|
||||
content:
|
||||
typeof message.content === "string"
|
||||
? [{ type: "text", text: message.content, cache_control: { type: "ephemeral" } }]
|
||||
: message.content.map((content, contentIndex) =>
|
||||
contentIndex === message.content.length - 1
|
||||
? { ...content, cache_control: { type: "ephemeral" } }
|
||||
: content,
|
||||
),
|
||||
}
|
||||
}
|
||||
return message
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the model ID and info for the handler.
|
||||
* This method must be implemented by subclasses.
|
||||
* @returns The model ID and info.
|
||||
*/
|
||||
abstract getModel(): { id: string; info: ModelInfo }
|
||||
}
|
||||
|
|
@ -1,186 +1,88 @@
|
|||
import { Anthropic } from "@anthropic-ai/sdk"
|
||||
import { AnthropicVertex } from "@anthropic-ai/vertex-sdk"
|
||||
import { withRetry } from "../retry"
|
||||
import { ApiHandler } from "../"
|
||||
import { ApiHandlerOptions, ModelInfo, vertexDefaultModelId, VertexModelId, vertexModels } from "../../shared/api"
|
||||
import { Stream as AnthropicStream } from "@anthropic-ai/sdk/streaming"
|
||||
import { ClaudeStreamingHandler } from "./claude-streaming"
|
||||
import { ModelInfo, vertexDefaultModelId, VertexModelId, vertexModels } from "../../shared/api"
|
||||
import { ApiStream } from "../transform/stream"
|
||||
|
||||
// https://docs.anthropic.com/en/api/claude-on-vertex-ai
|
||||
export class VertexHandler implements ApiHandler {
|
||||
private options: ApiHandlerOptions
|
||||
private client: AnthropicVertex
|
||||
|
||||
constructor(options: ApiHandlerOptions) {
|
||||
this.options = options
|
||||
this.client = new AnthropicVertex({
|
||||
/**
|
||||
* Handles interactions with the Anthropic Vertex service.
|
||||
*/
|
||||
export class VertexHandler extends ClaudeStreamingHandler<AnthropicVertex> {
|
||||
getClient() {
|
||||
return new AnthropicVertex({
|
||||
projectId: this.options.vertexProjectId,
|
||||
// https://cloud.google.com/vertex-ai/generative-ai/docs/partner-models/use-claude#regions
|
||||
region: this.options.vertexRegion,
|
||||
})
|
||||
}
|
||||
|
||||
@withRetry()
|
||||
async *createMessage(systemPrompt: string, messages: Anthropic.Messages.MessageParam[]): ApiStream {
|
||||
async *createStreamingMessage(systemPrompt: string, messages: Anthropic.Messages.MessageParam[]): ApiStream {
|
||||
const model = this.getModel()
|
||||
const modelId = model.id
|
||||
let stream: AnthropicStream<Anthropic.Messages.RawMessageStreamEvent>
|
||||
|
||||
let stream
|
||||
switch (modelId) {
|
||||
case "claude-3-7-sonnet@20250219":
|
||||
case "claude-3-5-sonnet-v2@20241022":
|
||||
case "claude-3-5-sonnet@20240620":
|
||||
case "claude-3-5-haiku@20241022":
|
||||
case "claude-3-opus@20240229":
|
||||
case "claude-3-haiku@20240307": {
|
||||
// Find indices of user messages for cache control
|
||||
const userMsgIndices = messages.reduce(
|
||||
(acc, msg, index) => (msg.role === "user" ? [...acc, index] : acc),
|
||||
[] as number[],
|
||||
)
|
||||
const lastUserMsgIndex = userMsgIndices[userMsgIndices.length - 1] ?? -1
|
||||
const secondLastMsgUserIndex = userMsgIndices[userMsgIndices.length - 2] ?? -1
|
||||
if (Object.keys(vertexModels).includes(modelId)) {
|
||||
stream = await this.createModelStream(
|
||||
systemPrompt,
|
||||
messages,
|
||||
modelId,
|
||||
model.info.maxTokens ?? ClaudeStreamingHandler.DEFAULT_TOKEN_SIZE,
|
||||
)
|
||||
} else {
|
||||
stream = this.client.messages.create({
|
||||
model: modelId,
|
||||
max_tokens: model.info.maxTokens || ClaudeStreamingHandler.DEFAULT_TOKEN_SIZE,
|
||||
temperature: ClaudeStreamingHandler.DEFAULT_TEMPERATURE,
|
||||
system: [{ text: systemPrompt, type: "text" }],
|
||||
messages,
|
||||
stream: true,
|
||||
}) as any
|
||||
}
|
||||
|
||||
stream = await this.client.beta.messages.create(
|
||||
{
|
||||
model: modelId,
|
||||
max_tokens: model.info.maxTokens || 8192,
|
||||
temperature: 0,
|
||||
system: [
|
||||
{
|
||||
text: systemPrompt,
|
||||
type: "text",
|
||||
cache_control: { type: "ephemeral" },
|
||||
},
|
||||
],
|
||||
messages: messages.map((message, index) => {
|
||||
if (index === lastUserMsgIndex || index === secondLastMsgUserIndex) {
|
||||
return {
|
||||
...message,
|
||||
content:
|
||||
typeof message.content === "string"
|
||||
? [
|
||||
{
|
||||
type: "text",
|
||||
text: message.content,
|
||||
cache_control: {
|
||||
type: "ephemeral",
|
||||
},
|
||||
},
|
||||
]
|
||||
: message.content.map((content, contentIndex) =>
|
||||
contentIndex === message.content.length - 1
|
||||
? {
|
||||
...content,
|
||||
cache_control: {
|
||||
type: "ephemeral",
|
||||
},
|
||||
}
|
||||
: content,
|
||||
),
|
||||
}
|
||||
}
|
||||
return {
|
||||
...message,
|
||||
content:
|
||||
typeof message.content === "string"
|
||||
? [
|
||||
{
|
||||
type: "text",
|
||||
text: message.content,
|
||||
},
|
||||
]
|
||||
: message.content,
|
||||
}
|
||||
}),
|
||||
stream: true,
|
||||
},
|
||||
{
|
||||
headers: {},
|
||||
},
|
||||
)
|
||||
break
|
||||
}
|
||||
default: {
|
||||
stream = await this.client.beta.messages.create({
|
||||
model: modelId,
|
||||
max_tokens: model.info.maxTokens || 8192,
|
||||
temperature: 0,
|
||||
system: [
|
||||
{
|
||||
text: systemPrompt,
|
||||
type: "text",
|
||||
},
|
||||
],
|
||||
messages: messages.map((message) => ({
|
||||
...message,
|
||||
content:
|
||||
typeof message.content === "string"
|
||||
? [
|
||||
{
|
||||
type: "text",
|
||||
text: message.content,
|
||||
},
|
||||
]
|
||||
: message.content,
|
||||
})),
|
||||
stream: true,
|
||||
})
|
||||
break
|
||||
}
|
||||
}
|
||||
for await (const chunk of stream) {
|
||||
switch (chunk.type) {
|
||||
case "message_start":
|
||||
const usage = chunk.message.usage
|
||||
yield {
|
||||
type: "usage",
|
||||
inputTokens: usage.input_tokens || 0,
|
||||
outputTokens: usage.output_tokens || 0,
|
||||
cacheWriteTokens: usage.cache_creation_input_tokens || undefined,
|
||||
cacheReadTokens: usage.cache_read_input_tokens || undefined,
|
||||
}
|
||||
break
|
||||
case "message_delta":
|
||||
yield {
|
||||
type: "usage",
|
||||
inputTokens: 0,
|
||||
outputTokens: chunk.usage.output_tokens || 0,
|
||||
}
|
||||
break
|
||||
case "message_stop":
|
||||
break
|
||||
case "content_block_start":
|
||||
switch (chunk.content_block.type) {
|
||||
case "text":
|
||||
if (chunk.index > 0) {
|
||||
yield {
|
||||
type: "text",
|
||||
text: "\n",
|
||||
}
|
||||
}
|
||||
yield {
|
||||
type: "text",
|
||||
text: chunk.content_block.text,
|
||||
}
|
||||
break
|
||||
}
|
||||
break
|
||||
case "content_block_delta":
|
||||
switch (chunk.delta.type) {
|
||||
case "text_delta":
|
||||
yield {
|
||||
type: "text",
|
||||
text: chunk.delta.text,
|
||||
}
|
||||
break
|
||||
}
|
||||
break
|
||||
case "content_block_stop":
|
||||
break
|
||||
}
|
||||
yield* this.processStream(stream)
|
||||
}
|
||||
|
||||
protected override handleMessageStreamError(error: any): void {
|
||||
if (error.error === "invalid_grant" && error.error_subtype === "invalid_rapt") {
|
||||
// Handle reauthentication-related error (invalid_rapt)
|
||||
console.error("Reauthentication-related error (invalid_rapt): ", error.error_description)
|
||||
console.info(
|
||||
`To resolve this issue, please visit the following link for reauthentication instructions: ${error.error_uri}, or execute \`gcloud auth application-default login\`.`,
|
||||
)
|
||||
throw new Error(`Invalid grant: Please visit ${error.error_uri} or reauthenticate via the gcloud CLI to proceed.`)
|
||||
} else {
|
||||
super.handleMessageStreamError(error)
|
||||
}
|
||||
}
|
||||
|
||||
async createModelStream(
|
||||
systemPrompt: string,
|
||||
messages: Anthropic.Messages.MessageParam[],
|
||||
modelId: string,
|
||||
maxTokens: number,
|
||||
): Promise<AnthropicStream<Anthropic.Messages.RawMessageStreamEvent>> {
|
||||
const userMsgIndices = messages.reduce((acc, msg, index) => (msg.role === "user" ? [...acc, index] : acc), [] as number[])
|
||||
const lastUserMsgIndex = userMsgIndices[userMsgIndices.length - 1] ?? -1
|
||||
const secondLastMsgUserIndex = userMsgIndices[userMsgIndices.length - 2] ?? -1
|
||||
|
||||
return this.client.messages.create({
|
||||
model: modelId,
|
||||
max_tokens: maxTokens || ClaudeStreamingHandler.DEFAULT_TOKEN_SIZE,
|
||||
temperature: ClaudeStreamingHandler.DEFAULT_TEMPERATURE,
|
||||
system: [
|
||||
{
|
||||
text: systemPrompt,
|
||||
type: "text",
|
||||
cache_control: { type: "ephemeral" },
|
||||
},
|
||||
], // setting cache breakpoint for system prompt so new tasks can reuse it
|
||||
messages: messages.map((message, index) =>
|
||||
this.transformMessage(message, index, lastUserMsgIndex, secondLastMsgUserIndex),
|
||||
),
|
||||
stream: true,
|
||||
})
|
||||
}
|
||||
|
||||
getModel(): { id: VertexModelId; info: ModelInfo } {
|
||||
const modelId = this.options.apiModelId
|
||||
if (modelId && modelId in vertexModels) {
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@ export function convertAnthropicContentToGemini(
|
|||
| Anthropic.Messages.ImageBlockParam
|
||||
| Anthropic.Messages.ToolUseBlockParam
|
||||
| Anthropic.Messages.ToolResultBlockParam
|
||||
| Anthropic.Messages.DocumentBlockParam
|
||||
>,
|
||||
): Part[] {
|
||||
if (typeof content === "string") {
|
||||
|
|
@ -38,6 +39,16 @@ export function convertAnthropicContentToGemini(
|
|||
mimeType: block.source.media_type,
|
||||
},
|
||||
} as InlineDataPart
|
||||
case "document":
|
||||
if (block.source.type !== "base64") {
|
||||
throw new Error("Unsupported document source type")
|
||||
}
|
||||
return {
|
||||
inlineData: {
|
||||
data: block.source.data,
|
||||
mimeType: block.source.media_type,
|
||||
},
|
||||
} as InlineDataPart
|
||||
case "tool_use":
|
||||
return {
|
||||
functionCall: {
|
||||
|
|
@ -133,7 +144,7 @@ export function convertGeminiResponseToAnthropic(response: EnhancedGenerateConte
|
|||
// Add the main text response
|
||||
const text = response.text()
|
||||
if (text) {
|
||||
content.push({ type: "text", text })
|
||||
content.push({ type: "text", text, citations: [] })
|
||||
}
|
||||
|
||||
// Add function calls as tool_use blocks
|
||||
|
|
@ -183,6 +194,8 @@ export function convertGeminiResponseToAnthropic(response: EnhancedGenerateConte
|
|||
usage: {
|
||||
input_tokens: response.usageMetadata?.promptTokenCount ?? 0,
|
||||
output_tokens: response.usageMetadata?.candidatesTokenCount ?? 0,
|
||||
cache_creation_input_tokens: null,
|
||||
cache_read_input_tokens: null,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -376,6 +376,7 @@ export function convertO1ResponseToAnthropicMessage(
|
|||
{
|
||||
type: "text",
|
||||
text: normalText,
|
||||
citations: [],
|
||||
},
|
||||
],
|
||||
model: completion.model,
|
||||
|
|
@ -396,6 +397,8 @@ export function convertO1ResponseToAnthropicMessage(
|
|||
usage: {
|
||||
input_tokens: completion.usage?.prompt_tokens || 0,
|
||||
output_tokens: completion.usage?.completion_tokens || 0,
|
||||
cache_creation_input_tokens: null,
|
||||
cache_read_input_tokens: null,
|
||||
},
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -161,6 +161,7 @@ export function convertToAnthropicMessage(completion: OpenAI.Chat.Completions.Ch
|
|||
{
|
||||
type: "text",
|
||||
text: openAiMessage.content || "",
|
||||
citations: [], // Add an empty array or appropriate citations
|
||||
},
|
||||
],
|
||||
model: completion.model,
|
||||
|
|
@ -181,6 +182,8 @@ export function convertToAnthropicMessage(completion: OpenAI.Chat.Completions.Ch
|
|||
usage: {
|
||||
input_tokens: completion.usage?.prompt_tokens || 0,
|
||||
output_tokens: completion.usage?.completion_tokens || 0,
|
||||
cache_creation_input_tokens: null,
|
||||
cache_read_input_tokens: null,
|
||||
},
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -175,6 +175,7 @@ export async function convertToAnthropicMessage(
|
|||
return {
|
||||
type: "text",
|
||||
text: part.value,
|
||||
citations: [], // Add an empty array or appropriate citations here
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -195,6 +196,8 @@ export async function convertToAnthropicMessage(
|
|||
usage: {
|
||||
input_tokens: 0,
|
||||
output_tokens: 0,
|
||||
cache_creation_input_tokens: null,
|
||||
cache_read_input_tokens: null,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -59,6 +59,7 @@ import { formatResponse } from "./prompts/responses"
|
|||
import { addUserInstructions, SYSTEM_PROMPT } from "./prompts/system"
|
||||
import { getNextTruncationRange, getTruncatedMessages } from "./sliding-window"
|
||||
import { ClineProvider, GlobalFileNames } from "./webview/ClineProvider"
|
||||
import { ImageBlockParam, TextBlockParam, ToolResultBlockParam, ToolUseBlockParam } from "@anthropic-ai/sdk/resources/index.mjs"
|
||||
import { DEFAULT_LANGUAGE_SETTINGS, getLanguageKey, LanguageDisplay, LanguageKey } from "../shared/Languages"
|
||||
import { telemetryService } from "../services/telemetry/TelemetryService"
|
||||
|
||||
|
|
@ -959,7 +960,10 @@ export class Cline {
|
|||
existingApiConversationHistory[existingApiConversationHistory.length - 2]
|
||||
|
||||
const existingUserContent: UserContent = Array.isArray(lastMessage.content)
|
||||
? lastMessage.content
|
||||
? lastMessage.content.filter(
|
||||
(block): block is TextBlockParam | ImageBlockParam | ToolUseBlockParam | ToolResultBlockParam =>
|
||||
block.type !== "document",
|
||||
)
|
||||
: [{ type: "text", text: lastMessage.content }]
|
||||
if (previousAssistantMessage && previousAssistantMessage.role === "assistant") {
|
||||
const assistantContent = Array.isArray(previousAssistantMessage.content)
|
||||
|
|
|
|||
|
|
@ -36,13 +36,18 @@ export async function downloadTask(dateTs: number, conversationHistory: Anthropi
|
|||
|
||||
if (saveUri) {
|
||||
// Write content to the selected location
|
||||
await vscode.workspace.fs.writeFile(saveUri, Buffer.from(markdownContent))
|
||||
await vscode.workspace.fs.writeFile(saveUri, new Uint8Array(Buffer.from(markdownContent)))
|
||||
vscode.window.showTextDocument(saveUri, { preview: true })
|
||||
}
|
||||
}
|
||||
|
||||
export function formatContentBlockToMarkdown(
|
||||
block: Anthropic.TextBlockParam | Anthropic.ImageBlockParam | Anthropic.ToolUseBlockParam | Anthropic.ToolResultBlockParam,
|
||||
block:
|
||||
| Anthropic.TextBlockParam
|
||||
| Anthropic.ImageBlockParam
|
||||
| Anthropic.ToolUseBlockParam
|
||||
| Anthropic.ToolResultBlockParam
|
||||
| Anthropic.DocumentBlockParam,
|
||||
// messages: Anthropic.MessageParam[]
|
||||
): string {
|
||||
switch (block.type) {
|
||||
|
|
|
|||
|
|
@ -245,7 +245,7 @@ export const vertexModels = {
|
|||
cacheWritesPrice: 3.75,
|
||||
cacheReadsPrice: 0.3,
|
||||
},
|
||||
"claude-3-5-sonnet@20240620": {
|
||||
"claude-3-sonnet@20240229": {
|
||||
maxTokens: 8192,
|
||||
contextWindow: 200_000,
|
||||
supportsImages: true,
|
||||
|
|
|
|||
|
|
@ -24,16 +24,16 @@ describe("Announcement", () => {
|
|||
|
||||
it("renders the mcp server improvements announcement", () => {
|
||||
render(<Announcement version="2.0.0" hideAnnouncement={hideAnnouncement} />)
|
||||
expect(screen.getByText(/MCP server improvements:/)).toBeInTheDocument()
|
||||
expect(screen.getByText(/Introducing MCP Marketplace/)).toBeInTheDocument()
|
||||
})
|
||||
|
||||
it("renders the 'See new changes' button feature", () => {
|
||||
render(<Announcement version="2.0.0" hideAnnouncement={hideAnnouncement} />)
|
||||
expect(screen.getByText(/See it in action here./)).toBeInTheDocument()
|
||||
expect(screen.getByText(/See a demo of the changes here!/)).toBeInTheDocument()
|
||||
})
|
||||
|
||||
it("renders the demo link", () => {
|
||||
render(<Announcement version="2.0.0" hideAnnouncement={hideAnnouncement} />)
|
||||
expect(screen.getByText(/See a demo here./)).toBeInTheDocument()
|
||||
expect(screen.getByText(/Join us on/)).toBeInTheDocument()
|
||||
})
|
||||
})
|
||||
|
|
|
|||
|
|
@ -603,17 +603,25 @@ const ApiOptions = ({ showModelOptions, apiErrorMessage, modelIdErrorMessage, is
|
|||
marginTop: "5px",
|
||||
color: "var(--vscode-descriptionForeground)",
|
||||
}}>
|
||||
To use Google Cloud Vertex AI, you need to
|
||||
<VSCodeLink
|
||||
href="https://cloud.google.com/vertex-ai/generative-ai/docs/partner-models/use-claude#before_you_begin"
|
||||
style={{ display: "inline", fontSize: "inherit" }}>
|
||||
{"1) create a Google Cloud account › enable the Vertex AI API › enable the desired Claude models,"}
|
||||
</VSCodeLink>{" "}
|
||||
<VSCodeLink
|
||||
href="https://cloud.google.com/docs/authentication/provide-credentials-adc#google-idp"
|
||||
style={{ display: "inline", fontSize: "inherit" }}>
|
||||
{"2) install the Google Cloud CLI › configure Application Default Credentials."}
|
||||
</VSCodeLink>
|
||||
To use Google Cloud Vertex AI:
|
||||
<ol>
|
||||
<li>
|
||||
<VSCodeLink
|
||||
href="https://cloud.google.com/vertex-ai/generative-ai/docs/partner-models/use-claude#before_you_begin"
|
||||
style={{ display: "inline", fontSize: "inherit" }}>
|
||||
{
|
||||
"Create a Google Cloud account › enable the Vertex AI API › enable the desired Claude models."
|
||||
}
|
||||
</VSCodeLink>
|
||||
</li>
|
||||
<li>
|
||||
<VSCodeLink
|
||||
href="https://cloud.google.com/docs/authentication/provide-credentials-adc#google-idp"
|
||||
style={{ display: "inline", fontSize: "inherit" }}>
|
||||
{"Install the Google Cloud CLI › configure Application Default Credentials."}
|
||||
</VSCodeLink>
|
||||
</li>
|
||||
</ol>
|
||||
</p>
|
||||
</div>
|
||||
)}
|
||||
|
|
|
|||
|
|
@ -1,123 +1,87 @@
|
|||
import { render, screen, fireEvent } from "@testing-library/react"
|
||||
import { describe, it, expect, vi } from "vitest"
|
||||
import { fireEvent, render, screen } from "@testing-library/react"
|
||||
import userEvent from "@testing-library/user-event"
|
||||
import ApiOptions from "../ApiOptions"
|
||||
import { ExtensionStateContextProvider } from "../../../context/ExtensionStateContext"
|
||||
import React from "react"
|
||||
|
||||
vi.mock("../../../context/ExtensionStateContext", async (importOriginal) => {
|
||||
const actual = await importOriginal()
|
||||
return {
|
||||
...actual,
|
||||
// your mocked methods
|
||||
useExtensionState: vi.fn(() => ({
|
||||
apiConfiguration: {
|
||||
apiProvider: "requesty",
|
||||
requestyApiKey: "",
|
||||
requestyModelId: "",
|
||||
},
|
||||
setApiConfiguration: vi.fn(),
|
||||
uriScheme: "vscode",
|
||||
})),
|
||||
}
|
||||
})
|
||||
vi.mock("../src/utils/vscode", () => ({
|
||||
vscode: {
|
||||
postMessage: vi.fn(),
|
||||
},
|
||||
}))
|
||||
|
||||
describe("ApiOptions Component", () => {
|
||||
vi.clearAllMocks()
|
||||
const mockPostMessage = vi.fn()
|
||||
|
||||
beforeEach(() => {
|
||||
global.vscode = { postMessage: mockPostMessage } as any
|
||||
})
|
||||
|
||||
it("renders Requesty API Key input", () => {
|
||||
render(
|
||||
<ExtensionStateContextProvider>
|
||||
<ApiOptions showModelOptions={true} />
|
||||
</ExtensionStateContextProvider>,
|
||||
)
|
||||
const apiKeyInput = screen.getByPlaceholderText("Enter API Key...")
|
||||
expect(apiKeyInput).toBeInTheDocument()
|
||||
})
|
||||
|
||||
it("renders Requesty Model ID input", () => {
|
||||
render(
|
||||
<ExtensionStateContextProvider>
|
||||
<ApiOptions showModelOptions={true} />
|
||||
</ExtensionStateContextProvider>,
|
||||
)
|
||||
const modelIdInput = screen.getByPlaceholderText("Enter Model ID...")
|
||||
expect(modelIdInput).toBeInTheDocument()
|
||||
})
|
||||
})
|
||||
|
||||
vi.mock("../../../context/ExtensionStateContext", async (importOriginal) => {
|
||||
const actual = await importOriginal()
|
||||
return {
|
||||
...actual,
|
||||
// your mocked methods
|
||||
useExtensionState: vi.fn(() => ({
|
||||
apiConfiguration: {
|
||||
apiProvider: "together",
|
||||
requestyApiKey: "",
|
||||
requestyModelId: "",
|
||||
},
|
||||
setApiConfiguration: vi.fn(),
|
||||
uriScheme: "vscode",
|
||||
})),
|
||||
}
|
||||
})
|
||||
|
||||
describe("ApiOptions Component", () => {
|
||||
vi.clearAllMocks()
|
||||
const mockPostMessage = vi.fn()
|
||||
|
||||
beforeEach(() => {
|
||||
global.vscode = { postMessage: mockPostMessage } as any
|
||||
})
|
||||
|
||||
it("renders Together API Key input", () => {
|
||||
render(
|
||||
<ExtensionStateContextProvider>
|
||||
<ApiOptions showModelOptions={true} />
|
||||
</ExtensionStateContextProvider>,
|
||||
)
|
||||
const apiKeyInput = screen.getByPlaceholderText("Enter API Key...")
|
||||
expect(apiKeyInput).toBeInTheDocument()
|
||||
})
|
||||
|
||||
it("renders Together Model ID input", () => {
|
||||
render(
|
||||
<ExtensionStateContextProvider>
|
||||
<ApiOptions showModelOptions={true} />
|
||||
</ExtensionStateContextProvider>,
|
||||
)
|
||||
const modelIdInput = screen.getByPlaceholderText("Enter Model ID...")
|
||||
expect(modelIdInput).toBeInTheDocument()
|
||||
})
|
||||
})
|
||||
|
||||
vi.mock("../../../context/ExtensionStateContext", async (importOriginal) => {
|
||||
const actual = await importOriginal()
|
||||
return {
|
||||
...actual,
|
||||
// your mocked methods
|
||||
useExtensionState: vi.fn(() => ({
|
||||
apiConfiguration: {
|
||||
apiProvider: "openai",
|
||||
requestyApiKey: "",
|
||||
requestyModelId: "",
|
||||
},
|
||||
setApiConfiguration: vi.fn(),
|
||||
uriScheme: "vscode",
|
||||
})),
|
||||
}
|
||||
})
|
||||
|
||||
describe("OpenApiInfoOptions", () => {
|
||||
const mockPostMessage = vi.fn()
|
||||
const renderComponent = (props = {}) => {
|
||||
return render(
|
||||
<ExtensionStateContextProvider>
|
||||
<ApiOptions showModelOptions={true} {...props} />
|
||||
</ExtensionStateContextProvider>,
|
||||
)
|
||||
}
|
||||
|
||||
describe("ApiOptions", () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks()
|
||||
global.vscode = { postMessage: mockPostMessage }
|
||||
Element.prototype.scrollIntoView = vi.fn()
|
||||
})
|
||||
|
||||
it("renders API Provider dropdown", () => {
|
||||
renderComponent()
|
||||
expect(screen.getByLabelText(/API Provider/i)).toBeInTheDocument()
|
||||
})
|
||||
|
||||
it("renders OpenAI API Key input when OpenAI provider is selected", async () => {
|
||||
renderComponent()
|
||||
const dropdown = screen.getAllByRole("combobox")[0]
|
||||
await userEvent.click(dropdown)
|
||||
const option = screen.getByRole("option", { name: "OpenAI" })
|
||||
await userEvent.click(option)
|
||||
expect(screen.getByPlaceholderText(/Enter API Key/i)).toBeInTheDocument()
|
||||
})
|
||||
|
||||
it("renders error message when apiErrorMessage is provided", () => {
|
||||
renderComponent({ apiErrorMessage: "API Error" })
|
||||
expect(screen.getByText(/API Error/i)).toBeInTheDocument()
|
||||
})
|
||||
|
||||
it("renders model dropdown for selected provider", async () => {
|
||||
renderComponent()
|
||||
const dropdown = screen.getAllByRole("combobox")[0]
|
||||
await userEvent.click(dropdown)
|
||||
const option = screen.getByRole("option", { name: "Anthropic" })
|
||||
await userEvent.click(option)
|
||||
expect(screen.getByPlaceholderText(/Enter API Key/i)).toBeInTheDocument()
|
||||
})
|
||||
|
||||
it("renders OpenRouter API Key input when OpenRouter provider is selected", async () => {
|
||||
renderComponent()
|
||||
const dropdown = screen.getAllByRole("combobox")[0]
|
||||
await userEvent.click(dropdown)
|
||||
const option = screen.getByRole("option", { name: "OpenRouter" })
|
||||
await userEvent.click(option)
|
||||
expect(screen.getByPlaceholderText(/Enter API Key/i)).toBeInTheDocument()
|
||||
})
|
||||
|
||||
it("renders AWS credentials inputs when AWS Bedrock provider is selected", async () => {
|
||||
renderComponent()
|
||||
const dropdown = screen.getAllByRole("combobox")[0]
|
||||
await userEvent.click(dropdown)
|
||||
const option = screen.getByRole("option", { name: "AWS Bedrock" })
|
||||
await userEvent.click(option)
|
||||
expect(screen.getByPlaceholderText(/Enter Access Key/i)).toBeInTheDocument()
|
||||
expect(screen.getByPlaceholderText(/Enter Secret Key/i)).toBeInTheDocument()
|
||||
expect(screen.getByPlaceholderText(/Enter Session Token/i)).toBeInTheDocument()
|
||||
const toggle = screen.getByText("AWS Profile")
|
||||
await userEvent.click(toggle)
|
||||
expect(screen.getByPlaceholderText(/Enter profile name/i)).toBeInTheDocument()
|
||||
})
|
||||
|
||||
it("renders Google Cloud Project ID input when Vertex provider is selected", async () => {
|
||||
renderComponent()
|
||||
const dropdown = screen.getAllByRole("combobox")[0]
|
||||
await userEvent.click(dropdown)
|
||||
const option = screen.getByRole("option", { name: "GCP Vertex AI" })
|
||||
await userEvent.click(option)
|
||||
expect(screen.getByPlaceholderText(/Enter Project ID/i)).toBeInTheDocument()
|
||||
})
|
||||
|
||||
it("renders OpenAI Supports Images input", () => {
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue