diff --git a/packages/types/src/providers/groq.ts b/packages/types/src/providers/groq.ts index cab0c69900..a977a66654 100644 --- a/packages/types/src/providers/groq.ts +++ b/packages/types/src/providers/groq.ts @@ -22,90 +22,100 @@ export const groqModels = { maxTokens: 8192, contextWindow: 131072, supportsImages: false, - supportsPromptCache: false, + supportsPromptCache: true, inputPrice: 0.05, outputPrice: 0.08, + cacheReadsPrice: 0.01, // 80% discount on cached tokens description: "Meta Llama 3.1 8B Instant model, 128K context.", }, "llama-3.3-70b-versatile": { maxTokens: 8192, contextWindow: 131072, supportsImages: false, - supportsPromptCache: false, + supportsPromptCache: true, inputPrice: 0.59, outputPrice: 0.79, + cacheReadsPrice: 0.118, // 80% discount on cached tokens description: "Meta Llama 3.3 70B Versatile model, 128K context.", }, "meta-llama/llama-4-scout-17b-16e-instruct": { maxTokens: 8192, contextWindow: 131072, supportsImages: false, - supportsPromptCache: false, + supportsPromptCache: true, inputPrice: 0.11, outputPrice: 0.34, + cacheReadsPrice: 0.022, // 80% discount on cached tokens description: "Meta Llama 4 Scout 17B Instruct model, 128K context.", }, "meta-llama/llama-4-maverick-17b-128e-instruct": { maxTokens: 8192, contextWindow: 131072, supportsImages: false, - supportsPromptCache: false, + supportsPromptCache: true, inputPrice: 0.2, outputPrice: 0.6, + cacheReadsPrice: 0.04, // 80% discount on cached tokens description: "Meta Llama 4 Maverick 17B Instruct model, 128K context.", }, "mistral-saba-24b": { maxTokens: 8192, contextWindow: 32768, supportsImages: false, - supportsPromptCache: false, + supportsPromptCache: true, inputPrice: 0.79, outputPrice: 0.79, + cacheReadsPrice: 0.158, // 80% discount on cached tokens description: "Mistral Saba 24B model, 32K context.", }, "qwen-qwq-32b": { maxTokens: 8192, contextWindow: 131072, supportsImages: false, - supportsPromptCache: false, + supportsPromptCache: true, inputPrice: 0.29, outputPrice: 0.39, + cacheReadsPrice: 0.058, // 80% discount on cached tokens description: "Alibaba Qwen QwQ 32B model, 128K context.", }, "qwen/qwen3-32b": { maxTokens: 8192, contextWindow: 131072, supportsImages: false, - supportsPromptCache: false, + supportsPromptCache: true, inputPrice: 0.29, outputPrice: 0.59, + cacheReadsPrice: 0.058, // 80% discount on cached tokens description: "Alibaba Qwen 3 32B model, 128K context.", }, "deepseek-r1-distill-llama-70b": { maxTokens: 8192, contextWindow: 131072, supportsImages: false, - supportsPromptCache: false, + supportsPromptCache: true, inputPrice: 0.75, outputPrice: 0.99, + cacheReadsPrice: 0.15, // 80% discount on cached tokens description: "DeepSeek R1 Distill Llama 70B model, 128K context.", }, "moonshotai/kimi-k2-instruct": { maxTokens: 16384, contextWindow: 131072, supportsImages: false, - supportsPromptCache: false, + supportsPromptCache: true, inputPrice: 1.0, outputPrice: 3.0, + cacheReadsPrice: 0.2, // 80% discount on cached tokens description: "Moonshot AI Kimi K2 Instruct 1T model, 128K context.", }, "openai/gpt-oss-120b": { maxTokens: 32766, contextWindow: 131072, supportsImages: false, - supportsPromptCache: false, + supportsPromptCache: true, inputPrice: 0.15, outputPrice: 0.75, + cacheReadsPrice: 0.03, // 80% discount on cached tokens description: "GPT-OSS 120B is OpenAI's flagship open source model, built on a Mixture-of-Experts (MoE) architecture with 20 billion parameters and 128 experts.", }, @@ -113,9 +123,10 @@ export const groqModels = { maxTokens: 32768, contextWindow: 131072, supportsImages: false, - supportsPromptCache: false, + supportsPromptCache: true, inputPrice: 0.1, outputPrice: 0.5, + cacheReadsPrice: 0.02, // 80% discount on cached tokens description: "GPT-OSS 20B is OpenAI's flagship open source model, built on a Mixture-of-Experts (MoE) architecture with 20 billion parameters and 32 experts.", }, diff --git a/src/api/providers/__tests__/groq.spec.ts b/src/api/providers/__tests__/groq.spec.ts index a943e84daa..e53d99f68d 100644 --- a/src/api/providers/__tests__/groq.spec.ts +++ b/src/api/providers/__tests__/groq.spec.ts @@ -42,6 +42,8 @@ describe("GroqHandler", () => { const model = handler.getModel() expect(model.id).toBe(groqDefaultModelId) expect(model.info).toEqual(groqModels[groqDefaultModelId]) + // Verify prompt caching is enabled + expect(model.info.supportsPromptCache).toBe(true) }) it("should return specified model when valid model is provided", () => { @@ -50,6 +52,8 @@ describe("GroqHandler", () => { const model = handlerWithModel.getModel() expect(model.id).toBe(testModelId) expect(model.info).toEqual(groqModels[testModelId]) + // Verify prompt caching is enabled + expect(model.info.supportsPromptCache).toBe(true) }) it("completePrompt method should return text from Groq API", async () => { @@ -108,7 +112,13 @@ describe("GroqHandler", () => { const firstChunk = await stream.next() expect(firstChunk.done).toBe(false) - expect(firstChunk.value).toEqual({ type: "usage", inputTokens: 10, outputTokens: 20 }) + expect(firstChunk.value).toEqual({ + type: "usage", + inputTokens: 10, + outputTokens: 20, + cacheWriteTokens: 0, + cacheReadTokens: 0, + }) }) it("createMessage should pass correct parameters to Groq client", async () => { @@ -221,4 +231,94 @@ describe("GroqHandler", () => { undefined, ) }) + + it("createMessage should handle cached tokens from Groq API", async () => { + const testContent = "This is test content from Groq stream" + const cachedTokens = 50 + + mockCreate.mockImplementationOnce(() => { + return { + [Symbol.asyncIterator]: () => ({ + next: vitest + .fn() + .mockResolvedValueOnce({ + done: false, + value: { choices: [{ delta: { content: testContent } }] }, + }) + .mockResolvedValueOnce({ + done: false, + value: { + choices: [{ delta: {} }], + usage: { + prompt_tokens: 100, + completion_tokens: 20, + prompt_tokens_details: { + cached_tokens: cachedTokens, + }, + }, + }, + }) + .mockResolvedValueOnce({ done: true }), + }), + } + }) + + const stream = handler.createMessage("system prompt", []) + const chunks = [] + for await (const chunk of stream) { + chunks.push(chunk) + } + + // Should have text chunk and usage chunk + expect(chunks).toHaveLength(2) + expect(chunks[0]).toEqual({ type: "text", text: testContent }) + + // Usage chunk should properly handle cached tokens + expect(chunks[1]).toEqual({ + type: "usage", + inputTokens: 50, // 100 total - 50 cached = 50 non-cached + outputTokens: 20, + cacheWriteTokens: 0, // Groq doesn't track cache writes + cacheReadTokens: 50, + }) + }) + + it("createMessage should handle missing cache information gracefully", async () => { + mockCreate.mockImplementationOnce(() => { + return { + [Symbol.asyncIterator]: () => ({ + next: vitest + .fn() + .mockResolvedValueOnce({ + done: false, + value: { + choices: [{ delta: {} }], + usage: { + prompt_tokens: 100, + completion_tokens: 20, + // No prompt_tokens_details + }, + }, + }) + .mockResolvedValueOnce({ done: true }), + }), + } + }) + + const stream = handler.createMessage("system prompt", []) + const chunks = [] + for await (const chunk of stream) { + chunks.push(chunk) + } + + // Should handle missing cache information gracefully + expect(chunks).toHaveLength(1) + expect(chunks[0]).toEqual({ + type: "usage", + inputTokens: 100, // No cached tokens, so all are non-cached + outputTokens: 20, + cacheWriteTokens: 0, + cacheReadTokens: 0, // Default to 0 when not provided + }) + }) }) diff --git a/src/api/providers/groq.ts b/src/api/providers/groq.ts index 7583edc51c..2c0b354980 100644 --- a/src/api/providers/groq.ts +++ b/src/api/providers/groq.ts @@ -1,6 +1,10 @@ +import { Anthropic } from "@anthropic-ai/sdk" + import { type GroqModelId, groqDefaultModelId, groqModels } from "@roo-code/types" import type { ApiHandlerOptions } from "../../shared/api" +import type { ApiHandlerCreateMessageMetadata } from "../index" +import { ApiStream } from "../transform/stream" import { BaseOpenAiCompatibleProvider } from "./base-openai-compatible-provider" @@ -16,4 +20,42 @@ export class GroqHandler extends BaseOpenAiCompatibleProvider { defaultTemperature: 0.5, }) } + + // Override to handle Groq's usage metrics, including caching + override async *createMessage( + systemPrompt: string, + messages: Anthropic.Messages.MessageParam[], + metadata?: ApiHandlerCreateMessageMetadata, + ): ApiStream { + const stream = await this.createStream(systemPrompt, messages, metadata) + + for await (const chunk of stream) { + const delta = chunk.choices[0]?.delta + + if (delta?.content) { + yield { + type: "text", + text: delta.content, + } + } + + if (chunk.usage) { + // Groq includes cached token information in prompt_tokens_details + const promptTokens = chunk.usage.prompt_tokens || 0 + const completionTokens = chunk.usage.completion_tokens || 0 + const cachedTokens = (chunk.usage as any).prompt_tokens_details?.cached_tokens || 0 + + // Calculate non-cached input tokens + const nonCachedInputTokens = Math.max(0, promptTokens - cachedTokens) + + yield { + type: "usage", + inputTokens: nonCachedInputTokens, + outputTokens: completionTokens, + cacheWriteTokens: 0, // Groq doesn't track cache writes + cacheReadTokens: cachedTokens, + } + } + } + } }