diff --git a/litellm/constants.py b/litellm/constants.py index bbeb4846e27..ad13d57aa42 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -210,6 +210,7 @@ LITELLM_UI_ALLOW_HEADERS: Final = [ "x-litellm-guardrail-scan-id", "x-litellm-guardrail-scan-metadata", "x-litellm-cache-key", + "x-litellm-response-cost", ] # Gemini model-specific minimal thinking budget constants diff --git a/tests/test_litellm/proxy/test_proxy_server.py b/tests/test_litellm/proxy/test_proxy_server.py index 950a6cc3c40..394118dd880 100644 --- a/tests/test_litellm/proxy/test_proxy_server.py +++ b/tests/test_litellm/proxy/test_proxy_server.py @@ -103,6 +103,13 @@ def test_cors_exposes_cache_key_header_to_browser_js(): assert "x-litellm-cache-key" in cors_middleware.kwargs["expose_headers"] +def test_cors_exposes_response_cost_header_to_browser_js(): + from fastapi.middleware.cors import CORSMiddleware + + cors_middleware = next(m for m in app.user_middleware if m.cls is CORSMiddleware) + assert "x-litellm-response-cost" in cors_middleware.kwargs["expose_headers"] + + def test_login_v2_returns_redirect_url_and_sets_cookie(monkeypatch): mock_login_result = {"user_id": "test-user"} mock_prisma_client = MagicMock() diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CostOptimizationView.activity.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CostOptimizationView.activity.test.tsx index 03250e3e53b..b3ba951783f 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CostOptimizationView.activity.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CostOptimizationView.activity.test.tsx @@ -37,9 +37,14 @@ vi.mock("@/components/shared/charts", () => ({ SEQUENTIAL_COLOR_RAMP: ["indigo"], })); -vi.mock("@/app/(dashboard)/router-settings/_components/general_settings", () => ({ - PromptCachingPanel: () =>
, -})); +vi.mock("@/app/(dashboard)/router-settings/_components/general_settings", async (importOriginal) => { + const original = + await importOriginal(); + return { + ...original, + PromptCachingPanel: () =>
, + }; +}); vi.mock("./PromptCompressionTab", () => ({ __esModule: true, default: () =>
})); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/PromptCachingTab.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/PromptCachingTab.test.tsx index 66db347e70f..9bba3d5495a 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/PromptCachingTab.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/PromptCachingTab.test.tsx @@ -7,11 +7,17 @@ vi.mock("@/components/networking", () => ({ getGeneralSettingsCall: (...args: unknown[]) => mockGetGeneralSettingsCall(...args), })); -vi.mock("@/app/(dashboard)/router-settings/_components/general_settings", () => ({ - PromptCachingPanel: () =>
, -})); +vi.mock("@/app/(dashboard)/router-settings/_components/general_settings", async (importOriginal) => { + const original = + await importOriginal(); + return { + ...original, + PromptCachingPanel: () =>
, + }; +}); const mockCacheLeakageCard = vi.fn(); +const mockPromptCachingTestCard = vi.fn(); vi.mock("./CacheLeakageCard", () => ({ __esModule: true, @@ -21,27 +27,72 @@ vi.mock("./CacheLeakageCard", () => ({ }, })); +vi.mock("./PromptCachingTestCard", () => ({ + __esModule: true, + default: (props: unknown) => { + mockPromptCachingTestCard(props); + return
; + }, +})); + import PromptCachingTab from "./PromptCachingTab"; +const activity = { + dateValue: {}, + onDateChange: vi.fn(), + results: [], + loading: false, + isFetchingMore: false, + progress: { currentPage: 1, totalPages: 1 }, + cancelled: false, + failed: false, + cancel: vi.fn(), +}; + describe("PromptCachingTab", () => { it("renders the cache leakage table alongside the caching settings", async () => { mockGetGeneralSettingsCall.mockResolvedValue([]); - const activity = { - dateValue: {}, - onDateChange: vi.fn(), - results: [], - loading: false, - isFetchingMore: false, - progress: { currentPage: 1, totalPages: 1 }, - cancelled: false, - failed: false, - cancel: vi.fn(), - }; render(); expect(screen.getByTestId("caching-settings")).toBeInTheDocument(); expect(screen.getByTestId("cache-leakage-card")).toBeInTheDocument(); await waitFor(() => expect(mockCacheLeakageCard).toHaveBeenCalledWith(expect.objectContaining({ activity }))); }); + + it("passes enabled=true to the test card when the caching setting is on", async () => { + mockGetGeneralSettingsCall.mockResolvedValue([ + { + field_name: "enable_anthropic_prompt_caching", + field_type: "Boolean", + field_value: true, + field_description: "", + stored_in_db: true, + }, + ]); + + render(); + + await waitFor(() => + expect(mockPromptCachingTestCard).toHaveBeenCalledWith(expect.objectContaining({ enabled: true })), + ); + }); + + it("passes enabled=false to the test card when the caching setting is off", async () => { + mockGetGeneralSettingsCall.mockResolvedValue([ + { + field_name: "enable_anthropic_prompt_caching", + field_type: "Boolean", + field_value: false, + field_description: "", + stored_in_db: true, + }, + ]); + + render(); + + await waitFor(() => + expect(mockPromptCachingTestCard).toHaveBeenCalledWith(expect.objectContaining({ enabled: false })), + ); + }); }); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/PromptCachingTab.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/PromptCachingTab.tsx index 59b38f272e0..dd541d7563b 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/PromptCachingTab.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/PromptCachingTab.tsx @@ -5,10 +5,13 @@ import React, { useCallback, useEffect, useState } from "react"; import { getGeneralSettingsCall } from "@/components/networking"; import { toast } from "@/lib/toast"; import { + ENABLE_ANTHROPIC_PROMPT_CACHING, PromptCachingPanel, generalSettingsItem, + isOn, } from "@/app/(dashboard)/router-settings/_components/general_settings"; import CacheLeakageCard from "./CacheLeakageCard"; +import PromptCachingTestCard from "./PromptCachingTestCard"; import { DailyActivityRange } from "./useDailyActivityRange"; interface PromptCachingTabProps { @@ -45,9 +48,12 @@ const PromptCachingTab: React.FC = ({ accessToken, activi return null; } + const enableSetting = settings.find((s) => s.field_name === ENABLE_ANTHROPIC_PROMPT_CACHING); + return (
+
); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/PromptCachingTestCard.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/PromptCachingTestCard.test.tsx new file mode 100644 index 00000000000..3bb1045c8c1 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/PromptCachingTestCard.test.tsx @@ -0,0 +1,83 @@ +import { fireEvent, render, screen } from "@testing-library/react"; +import { describe, expect, it, vi } from "vitest"; + +vi.mock("@/components/networking", () => ({ + getProxyBaseUrl: vi.fn(() => "http://proxy:4000"), +})); + +vi.mock("@/components/common_components/ModelSelector", () => ({ + __esModule: true, + default: ({ onChange }: { onChange?: (value: string | null) => void }) => ( + + {running && Sending call {runningCall} of 2...} +
+ + {error && ( + + + Test failed + {error} + + )} + + {outcome && ( +
+ + + + + Prompt tokens + Cache write + Cache read + Cost + Latency + + + + + + +
+ {verdict && ( + + {outcome.verdict === "injected" || outcome.verdict === "cache_hit_only" ? ( + + ) : ( + + )} + {verdict.title && {verdict.title}} + {verdict.text(outcome)} + + )} +
+ )} + + + ); +}; + +export default PromptCachingTestCard; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/promptCachingTest.test.ts b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/promptCachingTest.test.ts new file mode 100644 index 00000000000..649dce7cce3 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/promptCachingTest.test.ts @@ -0,0 +1,126 @@ +import { describe, expect, it, vi } from "vitest"; + +import { + PROMPT_CACHING_TEST_MIN_SYSTEM_TOKENS, + PromptCachingCallResult, + buildTestSystemPrompt, + judgePromptCachingTest, + runPromptCachingTest, +} from "./promptCachingTest"; + +const callResult = (overrides: Partial = {}): PromptCachingCallResult => ({ + cacheCreationTokens: 0, + cacheReadTokens: 0, + promptTokens: 5000, + responseCost: null, + model: null, + durationMs: 10, + ...overrides, +}); + +const jsonResponse = (body: unknown, init: { status?: number; headers?: Record } = {}) => + new Response(JSON.stringify(body), { + status: init.status ?? 200, + headers: { "Content-Type": "application/json", ...init.headers }, + }); + +const anthropicUsage = { + prompt_tokens: 5010, + cache_creation_input_tokens: 5000, + cache_read_input_tokens: 0, +}; + +describe("buildTestSystemPrompt", () => { + it("is comfortably above Anthropic's largest minimum cacheable size and differs by nonce", () => { + const first = buildTestSystemPrompt("run-a"); + const second = buildTestSystemPrompt("run-b"); + + expect(first.length).toBeGreaterThan(20000); + expect(first.length).toBeGreaterThan(PROMPT_CACHING_TEST_MIN_SYSTEM_TOKENS * 4); + expect(first).not.toBe(second); + expect(first.startsWith("Test run run-a.")).toBe(true); + expect(first.slice(first.indexOf("\n"))).toBe(second.slice(second.indexOf("\n"))); + }); +}); + +describe("judgePromptCachingTest", () => { + it("returns injected when call 1 writes and call 2 reads", () => { + expect( + judgePromptCachingTest(callResult({ cacheCreationTokens: 5000 }), callResult({ cacheReadTokens: 5000 })), + ).toBe("injected"); + }); + + it("returns cache_hit_only when neither call writes but a call reads", () => { + expect(judgePromptCachingTest(callResult(), callResult({ cacheReadTokens: 5000 }))).toBe("cache_hit_only"); + expect(judgePromptCachingTest(callResult({ cacheReadTokens: 5000 }), callResult({ cacheReadTokens: 5000 }))).toBe( + "cache_hit_only", + ); + }); + + it("returns injected_no_read when call 1 writes but call 2 reads nothing", () => { + expect(judgePromptCachingTest(callResult({ cacheCreationTokens: 5000 }), callResult())).toBe("injected_no_read"); + }); + + it("returns not_injected with no cache activity at all", () => { + expect(judgePromptCachingTest(callResult(), callResult())).toBe("not_injected"); + }); +}); + +describe("runPromptCachingTest", () => { + it("sends two sequential identical requests with no cache_control and parses usage", async () => { + const seenBodies: unknown[] = []; + const responses = [ + jsonResponse( + { model: "claude-haiku-4-5", usage: anthropicUsage }, + { headers: { "x-litellm-response-cost": "0.0042" } }, + ), + jsonResponse({ + model: "claude-haiku-4-5", + usage: { prompt_tokens: 5010, prompt_tokens_details: { cached_tokens: 5000 } }, + }), + ]; + const fetchImpl = vi.fn(async (_url: unknown, init?: RequestInit) => { + seenBodies.push(JSON.parse(String(init?.body))); + return responses[seenBodies.length - 1]; + }); + + const testOptions = { + accessToken: "token-123", + model: "claude-haiku-4-5", + baseUrl: "http://proxy:4000", + fetchImpl, + customHeaders: { "x-custom-auth": "abc", Authorization: "Bearer wrong" }, + }; + const result = await runPromptCachingTest(testOptions); + + expect(fetchImpl).toHaveBeenCalledTimes(2); + const [url, init] = fetchImpl.mock.calls[0] as unknown as [string, RequestInit]; + expect(url).toBe("http://proxy:4000/v1/chat/completions"); + expect((init.headers as Record).Authorization).toBe("Bearer token-123"); + expect((init.headers as Record)["x-custom-auth"]).toBe("abc"); + expect((init.headers as Record)["x-litellm-tags"]).toBe("prompt-caching-test"); + + const serialized = JSON.stringify(seenBodies); + expect(serialized).not.toContain("cache_control"); + expect((seenBodies[0] as { messages: { content: string }[] }).messages[0].content).toBe( + (seenBodies[1] as { messages: { content: string }[] }).messages[0].content, + ); + + expect(result.first.cacheCreationTokens).toBe(5000); + expect(result.first.responseCost).toBeCloseTo(0.0042); + expect(result.second.cacheReadTokens).toBe(5000); + expect(result.verdict).toBe("injected"); + }); + + it("rejects with status and error message on non-2xx", async () => { + const fetchImpl = vi.fn(async () => jsonResponse({ error: { message: "boom" } }, { status: 400 })); + + const testOptions = { + accessToken: "token-123", + model: "claude-haiku-4-5", + baseUrl: "http://proxy:4000", + fetchImpl, + }; + await expect(runPromptCachingTest(testOptions)).rejects.toThrow(/400.*boom|boom.*400/); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/promptCachingTest.ts b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/promptCachingTest.ts new file mode 100644 index 00000000000..f0c2102523d --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/promptCachingTest.ts @@ -0,0 +1,178 @@ +import { z } from "zod"; + +import { CustomHeaders, buildPlaygroundHeaders, withRequiredHeaders } from "@/components/llm_calls/request_headers"; +import { extractPromptCacheTokens } from "@/utils/promptCacheUsage"; + +export const PROMPT_CACHING_TEST_MIN_SYSTEM_TOKENS = 4096; + +const FILLER_LINE_COUNT = 600; + +export const buildTestSystemPrompt = (nonce: string): string => { + const filler = Array.from( + { length: FILLER_LINE_COUNT }, + (_, index) => `Reference item ${index + 1}: the gateway forwards requests to the configured deployment.`, + ).join("\n"); + return `Test run ${nonce}.\n${filler}`; +}; + +export interface PromptCachingTestRequestBody { + readonly model: string; + readonly messages: readonly { readonly role: string; readonly content: string }[]; + readonly max_tokens: number; + readonly temperature: number; +} + +export const buildTestRequestBody = (model: string, systemPrompt: string): PromptCachingTestRequestBody => ({ + model, + messages: [ + { role: "system", content: systemPrompt }, + { role: "user", content: "Reply with the single word OK." }, + ], + max_tokens: 5, + temperature: 0, +}); + +export interface PromptCachingCallResult { + cacheCreationTokens: number; + cacheReadTokens: number; + promptTokens: number; + responseCost: number | null; + model: string | null; + durationMs: number; +} + +export type PromptCachingVerdict = "injected" | "injected_no_read" | "not_injected" | "cache_hit_only"; + +export const judgePromptCachingTest = ( + first: PromptCachingCallResult, + second: PromptCachingCallResult, +): PromptCachingVerdict => { + if (first.cacheCreationTokens > 0 && second.cacheReadTokens > 0) { + return "injected"; + } + if (first.cacheCreationTokens > 0 && second.cacheReadTokens === 0) { + return "injected_no_read"; + } + if (first.cacheCreationTokens === 0 && second.cacheCreationTokens === 0) { + if (first.cacheReadTokens > 0 || second.cacheReadTokens > 0) { + return "cache_hit_only"; + } + } + return "not_injected"; +}; + +const tokenDetailsSchema = z + .object({ + cached_tokens: z.number().nullish(), + cache_write_tokens: z.number().nullish(), + }) + .nullish(); + +const usageFields = { + prompt_tokens: z.number().nullish(), + cache_creation_input_tokens: z.number().nullish(), + cache_read_input_tokens: z.number().nullish(), + prompt_tokens_details: tokenDetailsSchema, + input_tokens_details: tokenDetailsSchema, +}; + +const usageSchema = z.object(usageFields).nullish(); + +const chatCompletionResponseSchema = z.object({ + model: z.string().nullish(), + usage: usageSchema, +}); + +const errorBodySchema = z.object({ + error: z.object({ message: z.string().optional() }).optional(), +}); + +const parseResponseCost = (headers: Headers): number | null => { + const raw = headers.get("x-litellm-response-cost"); + if (raw === null) { + return null; + } + const cost = Number.parseFloat(raw); + return Number.isFinite(cost) ? cost : null; +}; + +interface SendTestCallOptions { + readonly accessToken: string; + readonly baseUrl: string; + readonly body: PromptCachingTestRequestBody; + readonly fetchImpl: typeof fetch; + readonly customHeaders: CustomHeaders; +} + +const sendTestCall = async ({ + accessToken, + baseUrl, + body, + fetchImpl, + customHeaders, +}: SendTestCallOptions): Promise => { + const startedAt = Date.now(); + const headers = withRequiredHeaders(buildPlaygroundHeaders(["prompt-caching-test"], customHeaders), { + Authorization: `Bearer ${accessToken}`, + "Content-Type": "application/json", + }); + const response = await fetchImpl(`${baseUrl}/v1/chat/completions`, { + method: "POST", + headers, + body: JSON.stringify(body), + }); + const durationMs = Date.now() - startedAt; + + const json: unknown = await response.json(); + if (!response.ok) { + const parsedError = errorBodySchema.safeParse(json); + const detail = parsedError.success ? parsedError.data.error?.message : undefined; + throw new Error(`Request failed with status ${response.status}${detail ? `: ${detail}` : ""}`); + } + + const parsed = chatCompletionResponseSchema.parse(json); + const cacheTokens = extractPromptCacheTokens(parsed.usage); + + return { + cacheCreationTokens: cacheTokens.cacheCreationTokens ?? 0, + cacheReadTokens: cacheTokens.cacheReadTokens ?? 0, + promptTokens: parsed.usage?.prompt_tokens ?? 0, + responseCost: parseResponseCost(response.headers), + model: parsed.model ?? null, + durationMs, + }; +}; + +export interface RunPromptCachingTestOptions { + readonly accessToken: string; + readonly model: string; + readonly baseUrl: string; + readonly fetchImpl?: typeof fetch; + readonly onCallStart?: (callIndex: 1 | 2) => void; + readonly customHeaders?: CustomHeaders; +} + +export const runPromptCachingTest = async ({ + accessToken, + model, + baseUrl, + fetchImpl = fetch, + onCallStart, + customHeaders = {}, +}: RunPromptCachingTestOptions): Promise<{ + first: PromptCachingCallResult; + second: PromptCachingCallResult; + verdict: PromptCachingVerdict; +}> => { + const systemPrompt = buildTestSystemPrompt(`${Date.now()}-${Math.random().toString(36).slice(2)}`); + const body = buildTestRequestBody(model, systemPrompt); + + const callOptions = { accessToken, baseUrl, body, fetchImpl, customHeaders }; + + onCallStart?.(1); + const first = await sendTestCall(callOptions); + onCallStart?.(2); + const second = await sendTestCall(callOptions); + + return { first, second, verdict: judgePromptCachingTest(first, second) }; +}; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/router-settings/_components/general_settings.tsx b/ui/litellm-dashboard/src/app/(dashboard)/router-settings/_components/general_settings.tsx index 3d0a2e438ac..aae84399751 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/router-settings/_components/general_settings.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/router-settings/_components/general_settings.tsx @@ -16,11 +16,11 @@ import Fallbacks from "@/components/Settings/RouterSettings/Fallbacks/Fallbacks" import RoutingGroups from "@/components/routing_groups"; const PROMPT_CACHING_TAB = "prompt_caching"; -const ENABLE_ANTHROPIC_PROMPT_CACHING = "enable_anthropic_prompt_caching"; +export const ENABLE_ANTHROPIC_PROMPT_CACHING = "enable_anthropic_prompt_caching"; const ANTHROPIC_PROMPT_CACHING_TTL = "anthropic_prompt_caching_ttl"; const OPENAI_SYSTEM_MESSAGES_FIRST = "openai_system_messages_first"; -const isOn = (value: unknown) => value === true || value === "true"; +export const isOn = (value: unknown) => value === true || value === "true"; interface GeneralSettingsPageProps { accessToken: string | null;