From e2af6818d7fe11a2eede92b04bd6b806ab423d11 Mon Sep 17 00:00:00 2001 From: Kevin Taylor Date: Tue, 29 Jul 2025 16:07:07 -0700 Subject: [PATCH] Add Cerebras as a provider - add support for 4 Cerebras models: llama-3.3-70b qwen-3-32b qwen-3-235b-a22b qwen-3-235b-a22b-instruct-2507 - System for filtering out thinking tokens from Cerebras reasoning model input --- packages/types/src/global-settings.ts | 1 + packages/types/src/provider-settings.ts | 7 + packages/types/src/providers/cerebras.ts | 46 +++ packages/types/src/providers/index.ts | 1 + src/api/index.ts | 3 + src/api/providers/cerebras.ts | 324 ++++++++++++++++++ src/api/providers/index.ts | 1 + .../src/components/settings/ApiOptions.tsx | 7 + .../src/components/settings/constants.ts | 3 + .../settings/providers/Cerebras.tsx | 50 +++ .../components/settings/providers/index.ts | 1 + .../components/ui/hooks/useSelectedModel.ts | 2 +- webview-ui/src/i18n/locales/en/settings.json | 2 + webview-ui/src/utils/validate.ts | 5 + 14 files changed, 452 insertions(+), 1 deletion(-) create mode 100644 packages/types/src/providers/cerebras.ts create mode 100644 src/api/providers/cerebras.ts create mode 100644 webview-ui/src/components/settings/providers/Cerebras.tsx diff --git a/packages/types/src/global-settings.ts b/packages/types/src/global-settings.ts index dc5a9e6744..33a17bd510 100644 --- a/packages/types/src/global-settings.ts +++ b/packages/types/src/global-settings.ts @@ -172,6 +172,7 @@ export const SECRET_STATE_KEYS = [ "openAiApiKey", "geminiApiKey", "openAiNativeApiKey", + "cerebrasApiKey", "deepSeekApiKey", "moonshotApiKey", "mistralApiKey", diff --git a/packages/types/src/provider-settings.ts b/packages/types/src/provider-settings.ts index 8cdb5296b2..eacad87836 100644 --- a/packages/types/src/provider-settings.ts +++ b/packages/types/src/provider-settings.ts @@ -33,6 +33,7 @@ export const providerNames = [ "chutes", "litellm", "huggingface", + "cerebras", ] as const export const providerNamesSchema = z.enum(providerNames) @@ -241,6 +242,10 @@ const litellmSchema = baseProviderSettingsSchema.extend({ litellmUsePromptCache: z.boolean().optional(), }) +const cerebrasSchema = apiModelIdProviderModelSchema.extend({ + cerebrasApiKey: z.string().optional(), +}) + const defaultSchema = z.object({ apiProvider: z.undefined(), }) @@ -271,6 +276,7 @@ export const providerSettingsSchemaDiscriminated = z.discriminatedUnion("apiProv huggingFaceSchema.merge(z.object({ apiProvider: z.literal("huggingface") })), chutesSchema.merge(z.object({ apiProvider: z.literal("chutes") })), litellmSchema.merge(z.object({ apiProvider: z.literal("litellm") })), + cerebrasSchema.merge(z.object({ apiProvider: z.literal("cerebras") })), defaultSchema, ]) @@ -301,6 +307,7 @@ export const providerSettingsSchema = z.object({ ...huggingFaceSchema.shape, ...chutesSchema.shape, ...litellmSchema.shape, + ...cerebrasSchema.shape, ...codebaseIndexProviderSchema.shape, }) diff --git a/packages/types/src/providers/cerebras.ts b/packages/types/src/providers/cerebras.ts new file mode 100644 index 0000000000..cab3bab41d --- /dev/null +++ b/packages/types/src/providers/cerebras.ts @@ -0,0 +1,46 @@ +import type { ModelInfo } from "../model.js" + +// https://inference-docs.cerebras.ai/api-reference/chat-completions +export type CerebrasModelId = keyof typeof cerebrasModels + +export const cerebrasDefaultModelId: CerebrasModelId = "qwen-3-235b-a22b-instruct-2507" + +export const cerebrasModels = { + "llama-3.3-70b": { + maxTokens: 64000, + contextWindow: 64000, + supportsImages: false, + supportsPromptCache: false, + inputPrice: 0, + outputPrice: 0, + description: "Smart model with ~2600 tokens/s", + }, + "qwen-3-32b": { + maxTokens: 64000, + contextWindow: 64000, + supportsImages: false, + supportsPromptCache: false, + inputPrice: 0, + outputPrice: 0, + description: "SOTA coding performance with ~2500 tokens/s", + }, + "qwen-3-235b-a22b": { + maxTokens: 40000, + contextWindow: 40000, + supportsImages: false, + supportsPromptCache: false, + inputPrice: 0, + outputPrice: 0, + description: "SOTA performance with ~1400 tokens/s", + }, + "qwen-3-235b-a22b-instruct-2507": { + maxTokens: 64000, + contextWindow: 640000, + supportsImages: false, + supportsPromptCache: false, + inputPrice: 0, + outputPrice: 0, + description: "SOTA performance with ~1400 tokens/s", + supportsReasoningEffort: true, + }, +} as const satisfies Record diff --git a/packages/types/src/providers/index.ts b/packages/types/src/providers/index.ts index f5061f152c..f3651431c8 100644 --- a/packages/types/src/providers/index.ts +++ b/packages/types/src/providers/index.ts @@ -1,5 +1,6 @@ export * from "./anthropic.js" export * from "./bedrock.js" +export * from "./cerebras.js" export * from "./chutes.js" export * from "./claude-code.js" export * from "./deepseek.js" diff --git a/src/api/index.ts b/src/api/index.ts index bda390848c..feec42b533 100644 --- a/src/api/index.ts +++ b/src/api/index.ts @@ -8,6 +8,7 @@ import { GlamaHandler, AnthropicHandler, AwsBedrockHandler, + CerebrasHandler, OpenRouterHandler, VertexHandler, AnthropicVertexHandler, @@ -115,6 +116,8 @@ export function buildApiHandler(configuration: ProviderSettings): ApiHandler { return new ChutesHandler(options) case "litellm": return new LiteLLMHandler(options) + case "cerebras": + return new CerebrasHandler(options) default: apiProvider satisfies "gemini-cli" | undefined return new AnthropicHandler(options) diff --git a/src/api/providers/cerebras.ts b/src/api/providers/cerebras.ts new file mode 100644 index 0000000000..e2621ccef1 --- /dev/null +++ b/src/api/providers/cerebras.ts @@ -0,0 +1,324 @@ +import { Anthropic } from "@anthropic-ai/sdk" + +import { type CerebrasModelId, cerebrasDefaultModelId, cerebrasModels } from "@roo-code/types" + +import type { ApiHandlerOptions } from "../../shared/api" +import { calculateApiCostOpenAI } from "../../shared/cost" +import { ApiStream } from "../transform/stream" +import { convertToOpenAiMessages } from "../transform/openai-format" +import { XmlMatcher } from "../../utils/xml-matcher" + +import type { ApiHandlerCreateMessageMetadata, SingleCompletionHandler } from "../index" +import { BaseProvider } from "./base-provider" + +const CEREBRAS_BASE_URL = "https://api.cerebras.ai/v1" +const CEREBRAS_DEFAULT_TEMPERATURE = 0 + +/** + * Removes thinking tokens from text to prevent model confusion when processing conversation history. + * This is crucial because models can get confused by their own thinking tokens in input. + */ +function stripThinkingTokens(text: string): string { + // Remove ... blocks entirely, including nested ones + return text.replace(/[\s\S]*?<\/think>/g, "").trim() +} + +/** + * Flattens OpenAI message content to simple strings that Cerebras can handle. + * Cerebras doesn't support complex content arrays like OpenAI does. + */ +function flattenMessageContent(content: any): string { + if (typeof content === "string") { + return content + } + + if (Array.isArray(content)) { + return content + .map((part) => { + if (typeof part === "string") { + return part + } + if (part.type === "text") { + return part.text || "" + } + if (part.type === "image_url") { + return "[Image]" // Placeholder for images since Cerebras doesn't support images + } + return "" + }) + .filter(Boolean) + .join("\n") + } + + // Fallback for any other content types + return String(content || "") +} + +/** + * Converts OpenAI messages to Cerebras-compatible format with simple string content. + * Also strips thinking tokens from assistant messages to prevent model confusion. + */ +function convertToCerebrasMessages(openaiMessages: any[]): Array<{ role: string; content: string }> { + return openaiMessages + .map((msg) => { + let content = flattenMessageContent(msg.content) + + // Strip thinking tokens from assistant messages to prevent confusion + if (msg.role === "assistant") { + content = stripThinkingTokens(content) + } + + return { + role: msg.role, + content, + } + }) + .filter((msg) => msg.content.trim() !== "") // Remove empty messages +} + +export class CerebrasHandler extends BaseProvider implements SingleCompletionHandler { + private apiKey: string + private providerModels: typeof cerebrasModels + private defaultProviderModelId: CerebrasModelId + private options: ApiHandlerOptions + private lastUsage: { inputTokens: number; outputTokens: number } = { inputTokens: 0, outputTokens: 0 } + + constructor(options: ApiHandlerOptions) { + super() + this.options = options + this.apiKey = options.cerebrasApiKey || "" + this.providerModels = cerebrasModels + this.defaultProviderModelId = cerebrasDefaultModelId + + if (!this.apiKey) { + throw new Error("Cerebras API key is required") + } + } + + getModel(): { id: CerebrasModelId; info: (typeof cerebrasModels)[CerebrasModelId] } { + const modelId = (this.options.apiModelId as CerebrasModelId) || this.defaultProviderModelId + return { + id: modelId, + info: this.providerModels[modelId], + } + } + + async *createMessage( + systemPrompt: string, + messages: Anthropic.Messages.MessageParam[], + metadata?: ApiHandlerCreateMessageMetadata, + ): ApiStream { + const { + id: model, + info: { maxTokens: max_tokens }, + } = this.getModel() + const temperature = this.options.modelTemperature ?? CEREBRAS_DEFAULT_TEMPERATURE + + // Convert Anthropic messages to OpenAI format, then flatten for Cerebras + // This will automatically strip thinking tokens from assistant messages + const openaiMessages = convertToOpenAiMessages(messages) + const cerebrasMessages = convertToCerebrasMessages(openaiMessages) + + // Prepare request body following Cerebras API specification exactly + const requestBody = { + model, + messages: [{ role: "system", content: systemPrompt }, ...cerebrasMessages], + stream: true, + // Use max_completion_tokens (Cerebras-specific parameter) + ...(max_tokens && max_tokens > 0 && max_tokens <= 32768 ? { max_completion_tokens: max_tokens } : {}), + // Clamp temperature to Cerebras range (0 to 1.5) + ...(temperature !== undefined && temperature !== CEREBRAS_DEFAULT_TEMPERATURE + ? { + temperature: Math.max(0, Math.min(1.5, temperature)), + } + : {}), + } + + console.log("[CEREBRAS DEBUG] Request URL:", `${CEREBRAS_BASE_URL}/chat/completions`) + console.log("[CEREBRAS DEBUG] Request body:", JSON.stringify(requestBody, null, 2)) + console.log("[CEREBRAS DEBUG] API key present:", !!this.apiKey) + console.log("[CEREBRAS DEBUG] Message conversion:") + console.log(" - Original messages:", messages.length) + console.log(" - OpenAI messages:", openaiMessages.length) + console.log(" - Cerebras messages:", cerebrasMessages.length) + console.log( + " - All content is strings:", + cerebrasMessages.every((msg) => typeof msg.content === "string"), + ) + console.log( + " - Thinking tokens stripped from assistant messages:", + cerebrasMessages.filter((msg) => msg.role === "assistant").length > 0 ? "✅" : "N/A", + ) + + try { + const response = await fetch(`${CEREBRAS_BASE_URL}/chat/completions`, { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: `Bearer ${this.apiKey}`, + "User-Agent": "roo-cline/1.0.0", + }, + body: JSON.stringify(requestBody), + }) + + console.log("[CEREBRAS DEBUG] Response status:", response.status) + const headersObj: Record = {} + response.headers.forEach((value, key) => { + headersObj[key] = value + }) + console.log("[CEREBRAS DEBUG] Response headers:", headersObj) + + if (!response.ok) { + const errorText = await response.text() + console.error("[CEREBRAS DEBUG] Error response body:", errorText) + + let errorDetails = "Unknown error" + try { + const errorJson = JSON.parse(errorText) + errorDetails = JSON.stringify(errorJson, null, 2) + } catch { + errorDetails = errorText || `HTTP ${response.status}` + } + + throw new Error(`Cerebras API Error: ${response.status} - ${errorDetails}`) + } + + if (!response.body) { + throw new Error("Cerebras API Error: No response body") + } + + // Initialize XmlMatcher to parse ... tags + const matcher = new XmlMatcher( + "think", + (chunk) => + ({ + type: chunk.matched ? "reasoning" : "text", + text: chunk.data, + }) as const, + ) + + const reader = response.body.getReader() + const decoder = new TextDecoder() + let buffer = "" + let inputTokens = 0 + let outputTokens = 0 + + try { + while (true) { + const { done, value } = await reader.read() + if (done) break + + buffer += decoder.decode(value, { stream: true }) + const lines = buffer.split("\n") + buffer = lines.pop() || "" // Keep the last incomplete line in the buffer + + for (const line of lines) { + if (line.trim() === "") continue + + try { + if (line.startsWith("data: ")) { + const jsonStr = line.slice(6).trim() + if (jsonStr === "[DONE]") { + continue + } + + const parsed = JSON.parse(jsonStr) + + // Handle text content - parse for thinking tokens + if (parsed.choices?.[0]?.delta?.content) { + const content = parsed.choices[0].delta.content + + // Use XmlMatcher to parse ... tags + for (const chunk of matcher.update(content)) { + yield chunk + } + } + + // Handle usage information if available + if (parsed.usage) { + inputTokens = parsed.usage.prompt_tokens || 0 + outputTokens = parsed.usage.completion_tokens || 0 + } + } + } catch (error) { + console.error("[CEREBRAS DEBUG] Failed to parse streaming data:", error, "Line:", line) + } + } + } + } finally { + reader.releaseLock() + } + + // Process any remaining content in the matcher + for (const chunk of matcher.final()) { + yield chunk + } + + // Provide token usage estimate if not available from API + if (inputTokens === 0 || outputTokens === 0) { + const inputText = systemPrompt + cerebrasMessages.map((m) => m.content).join("") + inputTokens = inputTokens || Math.ceil(inputText.length / 4) // Rough estimate: 4 chars per token + outputTokens = outputTokens || Math.ceil((max_tokens || 1000) / 10) // Rough estimate + } + + // Store usage for cost calculation + this.lastUsage = { inputTokens, outputTokens } + + yield { + type: "usage", + inputTokens, + outputTokens, + } + } catch (error) { + console.error("[CEREBRAS] Streaming error:", error) + + if (error instanceof Error) { + throw new Error(`Cerebras API error: ${error.message}`) + } + throw error + } + } + + async completePrompt(prompt: string): Promise { + const { id: model } = this.getModel() + + // Prepare request body for non-streaming completion + const requestBody = { + model, + messages: [{ role: "user", content: prompt }], + stream: false, + } + + try { + const response = await fetch(`${CEREBRAS_BASE_URL}/chat/completions`, { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: `Bearer ${this.apiKey}`, + "User-Agent": "roo-cline/1.0.0", + }, + body: JSON.stringify(requestBody), + }) + + if (!response.ok) { + const errorText = await response.text() + throw new Error(`Cerebras API Error: ${response.status} - ${errorText}`) + } + + const result = await response.json() + return result.choices?.[0]?.message?.content || "" + } catch (error) { + if (error instanceof Error) { + throw new Error(`Cerebras completion error: ${error.message}`) + } + throw error + } + } + + getApiCost(metadata: ApiHandlerCreateMessageMetadata): number { + const { info } = this.getModel() + // Use actual token usage from the last request + const { inputTokens, outputTokens } = this.lastUsage + return calculateApiCostOpenAI(info, inputTokens, outputTokens) + } +} diff --git a/src/api/providers/index.ts b/src/api/providers/index.ts index 1cefd0616b..4a1b74833f 100644 --- a/src/api/providers/index.ts +++ b/src/api/providers/index.ts @@ -1,6 +1,7 @@ export { AnthropicVertexHandler } from "./anthropic-vertex" export { AnthropicHandler } from "./anthropic" export { AwsBedrockHandler } from "./bedrock" +export { CerebrasHandler } from "./cerebras" export { ChutesHandler } from "./chutes" export { ClaudeCodeHandler } from "./claude-code" export { DeepSeekHandler } from "./deepseek" diff --git a/webview-ui/src/components/settings/ApiOptions.tsx b/webview-ui/src/components/settings/ApiOptions.tsx index 38d2ceebd3..7f0fed72d6 100644 --- a/webview-ui/src/components/settings/ApiOptions.tsx +++ b/webview-ui/src/components/settings/ApiOptions.tsx @@ -22,6 +22,7 @@ import { mistralDefaultModelId, xaiDefaultModelId, groqDefaultModelId, + cerebrasDefaultModelId, chutesDefaultModelId, bedrockDefaultModelId, vertexDefaultModelId, @@ -53,6 +54,7 @@ import { import { Anthropic, Bedrock, + Cerebras, Chutes, ClaudeCode, DeepSeek, @@ -286,6 +288,7 @@ const ApiOptions = ({ requesty: { field: "requestyModelId", default: requestyDefaultModelId }, litellm: { field: "litellmModelId", default: litellmDefaultModelId }, anthropic: { field: "apiModelId", default: anthropicDefaultModelId }, + cerebras: { field: "apiModelId", default: cerebrasDefaultModelId }, "claude-code": { field: "apiModelId", default: claudeCodeDefaultModelId }, "openai-native": { field: "apiModelId", default: openAiNativeDefaultModelId }, gemini: { field: "apiModelId", default: geminiDefaultModelId }, @@ -492,6 +495,10 @@ const ApiOptions = ({ )} + {selectedProvider === "cerebras" && ( + + )} + {selectedProvider === "chutes" && ( )} diff --git a/webview-ui/src/components/settings/constants.ts b/webview-ui/src/components/settings/constants.ts index 995f591034..94aa3e5e9c 100644 --- a/webview-ui/src/components/settings/constants.ts +++ b/webview-ui/src/components/settings/constants.ts @@ -3,6 +3,7 @@ import { type ModelInfo, anthropicModels, bedrockModels, + cerebrasModels, claudeCodeModels, deepSeekModels, moonshotModels, @@ -19,6 +20,7 @@ export const MODELS_BY_PROVIDER: Partial void +} + +export const Cerebras = ({ apiConfiguration, setApiConfigurationField }: CerebrasProps) => { + const { t } = useAppTranslation() + + const handleInputChange = useCallback( + ( + field: K, + transform: (event: E) => ProviderSettings[K] = inputEventTransform, + ) => + (event: E | Event) => { + setApiConfigurationField(field, transform(event as E)) + }, + [setApiConfigurationField], + ) + + return ( + <> + + + +
+ {t("settings:providers.apiKeyStorageNotice")} +
+ {!apiConfiguration?.cerebrasApiKey && ( + + {t("settings:providers.getCerebrasApiKey")} + + )} + + ) +} diff --git a/webview-ui/src/components/settings/providers/index.ts b/webview-ui/src/components/settings/providers/index.ts index 6c6fdddaee..e6615d183d 100644 --- a/webview-ui/src/components/settings/providers/index.ts +++ b/webview-ui/src/components/settings/providers/index.ts @@ -1,5 +1,6 @@ export { Anthropic } from "./Anthropic" export { Bedrock } from "./Bedrock" +export { Cerebras } from "./Cerebras" export { Chutes } from "./Chutes" export { ClaudeCode } from "./ClaudeCode" export { DeepSeek } from "./DeepSeek" diff --git a/webview-ui/src/components/ui/hooks/useSelectedModel.ts b/webview-ui/src/components/ui/hooks/useSelectedModel.ts index 8dceb6e117..49fb6e8af7 100644 --- a/webview-ui/src/components/ui/hooks/useSelectedModel.ts +++ b/webview-ui/src/components/ui/hooks/useSelectedModel.ts @@ -228,7 +228,7 @@ function getSelectedModel({ // case "human-relay": // case "fake-ai": default: { - provider satisfies "anthropic" | "gemini-cli" | "human-relay" | "fake-ai" + provider satisfies "anthropic" | "gemini-cli" | "human-relay" | "fake-ai" | "cerebras" const id = apiConfiguration.apiModelId ?? anthropicDefaultModelId const info = anthropicModels[id as keyof typeof anthropicModels] return { id, info } diff --git a/webview-ui/src/i18n/locales/en/settings.json b/webview-ui/src/i18n/locales/en/settings.json index 7c58e679c6..13db5dc112 100644 --- a/webview-ui/src/i18n/locales/en/settings.json +++ b/webview-ui/src/i18n/locales/en/settings.json @@ -249,6 +249,8 @@ "anthropicApiKey": "Anthropic API Key", "getAnthropicApiKey": "Get Anthropic API Key", "anthropicUseAuthToken": "Pass Anthropic API Key as Authorization header instead of X-Api-Key", + "cerebrasApiKey": "Cerebras API Key", + "getCerebrasApiKey": "Get Cerebras API Key", "chutesApiKey": "Chutes API Key", "getChutesApiKey": "Get Chutes API Key", "deepSeekApiKey": "DeepSeek API Key", diff --git a/webview-ui/src/utils/validate.ts b/webview-ui/src/utils/validate.ts index ed546cccc7..3b85ef9919 100644 --- a/webview-ui/src/utils/validate.ts +++ b/webview-ui/src/utils/validate.ts @@ -110,6 +110,11 @@ function validateModelsAndKeysProvided(apiConfiguration: ProviderSettings): stri return i18next.t("settings:validation.modelId") } break + case "cerebras": + if (!apiConfiguration.cerebrasApiKey) { + return i18next.t("settings:validation.apiKey") + } + break } return undefined